free-coding-models 0.3.5 → 0.3.9

package/src/testfcm.js

@@ -0,0 +1,451 @@
+/**
+ * @file src/testfcm.js
+ * @description Shared helpers for the AI-driven `/testfcm` workflow.
+ *
+ * @details
+ *   📖 These helpers stay side-effect free on purpose so the reporting logic can
+ *   📖 be unit-tested without spawning a PTY or touching the user's machine.
+ *
+ *   📖 The runner in `scripts/testfcm-runner.mjs` handles the live terminal work:
+ *   📖 copying config into an isolated HOME, driving the TUI, launching a tool,
+ *   📖 sending a prompt, collecting logs, and writing the final Markdown report.
+ *   📖 This module focuses on the pieces that should remain stable and reusable:
+ *   📖 tool metadata, transcript classification, JSON extraction, and report text.
+ *
+ * @functions
+ *   → `normalizeTestfcmToolName` — map aliases like `claude` to canonical FCM tool modes
+ *   → `resolveTestfcmToolSpec` — return the runner metadata for one tool mode
+ *   → `hasConfiguredKey` — decide whether a config entry really contains an API key
+ *   → `createTestfcmRunId` — build a stable timestamp-based run id for artifacts
+ *   → `extractJsonPayload` — recover JSON mode output even when logs prefix stdout
+ *   → `detectTranscriptFindings` — map raw tool output to actionable failure findings
+ *   → `classifyToolTranscript` — classify a run as passed, failed, or inconclusive
+ *   → `buildFixTasks` — convert findings into concrete follow-up work items
+ *   → `buildTestfcmReport` — render the final Markdown report written under `task/`
+ *
+ * @exports TESTFCM_TOOL_SPECS, normalizeTestfcmToolName, resolveTestfcmToolSpec
+ * @exports hasConfiguredKey, createTestfcmRunId, extractJsonPayload
+ * @exports detectTranscriptFindings, classifyToolTranscript, buildFixTasks
+ * @exports buildTestfcmReport
+ */
+
+export const TESTFCM_TOOL_SPECS = {
+  crush: {
+    mode: 'crush',
+    label: 'Crush',
+    command: 'crush',
+    flag: '--crush',
+    prefersProxy: true,
+    configPaths: ['.config/crush/crush.json'],
+  },
+  codex: {
+    mode: 'codex',
+    label: 'Codex CLI',
+    command: 'codex',
+    flag: '--codex',
+    prefersProxy: true,
+    configPaths: [],
+  },
+  'claude-code': {
+    mode: 'claude-code',
+    label: 'Claude Code',
+    command: 'claude',
+    flag: '--claude-code',
+    prefersProxy: true,
+    configPaths: [],
+  },
+  gemini: {
+    mode: 'gemini',
+    label: 'Gemini CLI',
+    command: 'gemini',
+    flag: '--gemini',
+    prefersProxy: true,
+    configPaths: ['.gemini/settings.json'],
+  },
+  goose: {
+    mode: 'goose',
+    label: 'Goose',
+    command: 'goose',
+    flag: '--goose',
+    prefersProxy: true,
+    configPaths: ['.config/goose/config.yaml'],
+  },
+  aider: {
+    mode: 'aider',
+    label: 'Aider',
+    command: 'aider',
+    flag: '--aider',
+    prefersProxy: false,
+    configPaths: ['.aider.conf.yml'],
+  },
+  qwen: {
+    mode: 'qwen',
+    label: 'Qwen Code',
+    command: 'qwen',
+    flag: '--qwen',
+    prefersProxy: false,
+    configPaths: ['.qwen/settings.json'],
+  },
+  amp: {
+    mode: 'amp',
+    label: 'Amp',
+    command: 'amp',
+    flag: '--amp',
+    prefersProxy: false,
+    configPaths: ['.config/amp/settings.json'],
+  },
+  pi: {
+    mode: 'pi',
+    label: 'Pi',
+    command: 'pi',
+    flag: '--pi',
+    prefersProxy: false,
+    configPaths: ['.pi/agent/models.json', '.pi/agent/settings.json'],
+  },
+  opencode: {
+    mode: 'opencode',
+    label: 'OpenCode CLI',
+    command: 'opencode',
+    flag: '--opencode',
+    prefersProxy: false,
+    configPaths: ['.config/opencode/opencode.json'],
+  },
+  openhands: {
+    mode: 'openhands',
+    label: 'OpenHands',
+    command: 'openhands',
+    flag: '--openhands',
+    prefersProxy: false,
+    configPaths: [],
+  },
+}
+
+const TESTFCM_TOOL_ALIASES = {
+  claude: 'claude-code',
+  claudecode: 'claude-code',
+  codexcli: 'codex',
+  opencodecli: 'opencode',
+}
+
+const TRANSCRIPT_FINDING_RULES = [
+  {
+    id: 'tool_missing',
+    title: 'Tool binary missing',
+    severity: 'high',
+    regex: /could not find "[^"]+" in path|command not found|enoent/i,
+    task: 'Install the requested tool binary or pass `--tool-bin-dir` so FCM can launch it during `/testfcm`.',
+  },
+  {
+    id: 'invalid_api_key',
+    title: 'Invalid or missing API auth',
+    severity: 'high',
+    regex: /invalid api|bad api key|incorrect api key|authentication failed|unauthorized|forbidden|missing api key|no api key|anthropic_auth_token|401\b|403\b/i,
+    task: 'Validate the provider key used by the selected model, then re-run `/testfcm` and inspect the proxy request log for the failing request.',
+  },
+  {
+    id: 'rate_limited',
+    title: 'Provider rate limited',
+    severity: 'medium',
+    regex: /rate limit|too many requests|quota exceeded|429\b/i,
+    task: 'Retry with another configured provider or inspect the retry-after and cooldown handling in the proxy/tool launch flow.',
+  },
+  {
+    id: 'proxy_failure',
+    title: 'Proxy startup or routing failure',
+    severity: 'high',
+    regex: /failed to start proxy|proxy mode .* required|selected model may not exist|routing reload is taking longer than expected|proxy launch is blocked/i,
+    task: 'Inspect `request-log.jsonl`, proxy startup messages, and the isolated config to verify the tool can reach the local FCM proxy.',
+  },
+  {
+    id: 'tool_launch_failed',
+    title: 'Tool launch failed',
+    severity: 'high',
+    regex: /failed to launch|failed to start|process exited with code 1|syntaxerror|traceback|fatal:/i,
+    task: 'Inspect the tool transcript and generated tool config under the isolated HOME to find the exact launcher failure.',
+  },
+]
+
+const SUCCESS_PATTERNS = [
+  /hello[,! ]/i,
+  /how can i help/i,
+  /how may i help/i,
+  /how can i assist/i,
+  /ready to help/i,
+]
+
+/**
+ * 📖 Normalize a user/tool alias to the canonical FCM tool mode.
+ *
+ * @param {string | null | undefined} value
+ * @returns {string | null}
+ */
+export function normalizeTestfcmToolName(value) {
+  if (typeof value !== 'string' || value.trim().length === 0) return null
+  const normalized = value.trim().toLowerCase()
+  return TESTFCM_TOOL_ALIASES[normalized] || normalized
+}
+
+/**
+ * 📖 Resolve one `/testfcm` tool spec from user input.
+ *
+ * @param {string | null | undefined} value
+ * @returns {typeof TESTFCM_TOOL_SPECS[keyof typeof TESTFCM_TOOL_SPECS] | null}
+ */
+export function resolveTestfcmToolSpec(value) {
+  const normalized = normalizeTestfcmToolName(value)
+  if (!normalized) return null
+  return TESTFCM_TOOL_SPECS[normalized] || null
+}
+
+/**
+ * 📖 Treat both string and multi-key array config entries as "configured" when at
+ * 📖 least one non-empty key is present.
+ *
+ * @param {unknown} value
+ * @returns {boolean}
+ */
+export function hasConfiguredKey(value) {
+  if (typeof value === 'string') return value.trim().length > 0
+  if (Array.isArray(value)) return value.some((entry) => typeof entry === 'string' && entry.trim().length > 0)
+  return false
+}
+
+/**
+ * 📖 Build an artifact-friendly run id such as `20260316-184512`.
+ *
+ * @param {Date} [date]
+ * @returns {string}
+ */
+export function createTestfcmRunId(date = new Date()) {
+  const iso = date.toISOString()
+  return iso
+    .replace(/\.\d{3}Z$/, '')
+    .replace(/:/g, '')
+    .replace(/-/g, '')
+    .replace('T', '-')
+}
+
+/**
+ * 📖 Extract the first valid JSON array payload from mixed stdout text.
+ *
+ * @param {string} text
+ * @returns {Array<object> | null}
+ */
+export function extractJsonPayload(text) {
+  const source = String(text || '')
+  let offset = source.indexOf('[')
+  while (offset !== -1) {
+    const candidate = source.slice(offset).trim()
+    try {
+      const parsed = JSON.parse(candidate)
+      return Array.isArray(parsed) ? parsed : null
+    } catch {
+      offset = source.indexOf('[', offset + 1)
+    }
+  }
+  return null
+}
+
+/**
+ * 📖 Detect known failure patterns in the raw tool transcript.
+ *
+ * @param {string} output
+ * @returns {Array<{ id: string, title: string, severity: string, task: string, excerpt: string }>}
+ */
+export function detectTranscriptFindings(output) {
+  const transcript = String(output || '')
+  const findings = []
+
+  for (const rule of TRANSCRIPT_FINDING_RULES) {
+    const match = transcript.match(rule.regex)
+    if (!match) continue
+
+    findings.push({
+      id: rule.id,
+      title: rule.title,
+      severity: rule.severity,
+      task: rule.task,
+      excerpt: match[0],
+    })
+  }
+
+  return findings
+}
+
+/**
+ * 📖 Decide whether the tool transcript proves success, proves failure, or stays
+ * 📖 too ambiguous to trust.
+ *
+ * @param {string} output
+ * @returns {{ status: 'passed' | 'failed' | 'inconclusive', findings: Array<{ id: string, title: string, severity: string, task: string, excerpt: string }>, matchedSuccess: string | null }}
+ */
+export function classifyToolTranscript(output) {
+  const transcript = String(output || '')
+  const matchedSuccess = SUCCESS_PATTERNS.find((pattern) => pattern.test(transcript))
+
+  if (matchedSuccess) {
+    return {
+      status: 'passed',
+      findings: [],
+      matchedSuccess: matchedSuccess.source,
+    }
+  }
+
+  const findings = detectTranscriptFindings(transcript)
+  if (findings.length > 0) {
+    return {
+      status: 'failed',
+      findings,
+      matchedSuccess: null,
+    }
+  }
+
+  return {
+    status: 'inconclusive',
+    findings: [],
+    matchedSuccess: null,
+  }
+}
+
+/**
+ * 📖 Collapse findings into unique human-readable follow-up tasks.
+ *
+ * @param {Array<{ task: string }>} findings
+ * @returns {string[]}
+ */
+export function buildFixTasks(findings) {
+  const tasks = new Set()
+  for (const finding of findings) {
+    if (typeof finding?.task === 'string' && finding.task.trim().length > 0) {
+      tasks.add(finding.task.trim())
+    }
+  }
+  return [...tasks]
+}
+
+/**
+ * 📖 Render the final Markdown report saved under `task/reports/`.
+ *
+ * @param {{
+ *   runId: string,
+ *   status: 'passed' | 'failed' | 'blocked',
+ *   startedAt: string,
+ *   finishedAt: string,
+ *   toolLabel: string,
+ *   toolMode: string,
+ *   prompt: string,
+ *   configuredProviders: string[],
+ *   toolBinaryPath: string | null,
+ *   isolatedHome: string,
+ *   preflightSummary: string,
+ *   findings: Array<{ id: string, title: string, severity: string, excerpt: string }>,
+ *   tasks: string[],
+ *   evidenceFiles: string[],
+ *   requestLogSummary: string[],
+ *   notes: string[],
+ *   transcriptExcerpt: string
+ * }} input
+ * @returns {string}
+ */
+export function buildTestfcmReport(input) {
+  const lines = []
+  const findings = Array.isArray(input.findings) ? input.findings : []
+  const tasks = Array.isArray(input.tasks) ? input.tasks : []
+  const evidenceFiles = Array.isArray(input.evidenceFiles) ? input.evidenceFiles : []
+  const requestLogSummary = Array.isArray(input.requestLogSummary) ? input.requestLogSummary : []
+  const notes = Array.isArray(input.notes) ? input.notes : []
+  const configuredProviders = Array.isArray(input.configuredProviders) ? input.configuredProviders : []
+  const transcriptExcerpt = String(input.transcriptExcerpt || '').trim()
+
+  lines.push(`# /testfcm Report - ${input.runId}`)
+  lines.push('')
+  lines.push(`- Status: **${input.status.toUpperCase()}**`)
+  lines.push(`- Started: ${input.startedAt}`)
+  lines.push(`- Finished: ${input.finishedAt}`)
+  lines.push(`- Tool: ${input.toolLabel} (${input.toolMode})`)
+  lines.push(`- Prompt sent: \`${input.prompt}\``)
+  lines.push(`- Configured providers in isolated run: ${configuredProviders.length > 0 ? configuredProviders.join(', ') : '(none)'}`)
+  lines.push(`- Tool binary: ${input.toolBinaryPath || '(not found on PATH)'}`)
+  lines.push(`- Isolated HOME: \`${input.isolatedHome}\``)
+  lines.push('')
+  lines.push('## Summary')
+  lines.push('')
+  lines.push(input.preflightSummary)
+  lines.push('')
+
+  if (findings.length > 0) {
+    lines.push('## Bugs Found')
+    lines.push('')
+    for (const finding of findings) {
+      lines.push(`- [${finding.severity}] ${finding.title} - evidence: \`${finding.excerpt}\``)
+    }
+    lines.push('')
+  } else {
+    lines.push('## Bugs Found')
+    lines.push('')
+    lines.push('- No blocking bug pattern matched the captured transcript in this run.')
+    lines.push('')
+  }
+
+  lines.push('## Tasks To Resolve')
+  lines.push('')
+  if (tasks.length > 0) {
+    for (const task of tasks) {
+      lines.push(`- ${task}`)
+    }
+  } else {
+    lines.push('- No follow-up task was generated from the captured evidence.')
+  }
+  lines.push('')
+
+  lines.push('## Evidence')
+  lines.push('')
+  if (evidenceFiles.length > 0) {
+    for (const file of evidenceFiles) {
+      lines.push(`- ${file}`)
+    }
+  } else {
+    lines.push('- No artifact file was captured.')
+  }
+  lines.push('')
+
+  lines.push('## Request Log Summary')
+  lines.push('')
+  if (requestLogSummary.length > 0) {
+    for (const entry of requestLogSummary) {
+      lines.push(`- ${entry}`)
+    }
+  } else {
+    lines.push('- No proxy request log entry was captured for this run.')
+  }
+  lines.push('')
+
+  lines.push('## Notes')
+  lines.push('')
+  if (notes.length > 0) {
+    for (const note of notes) {
+      lines.push(`- ${note}`)
+    }
+  } else {
+    lines.push('- No extra notes.')
+  }
+  lines.push('')
+
+  lines.push('## Transcript Excerpt')
+  lines.push('')
+  if (transcriptExcerpt) {
+    lines.push('```text')
+    lines.push(transcriptExcerpt)
+    lines.push('```')
+  } else {
+    lines.push('```text')
+    lines.push('(empty transcript excerpt)')
+    lines.push('```')
+  }
+  lines.push('')
+  lines.push('## Next Step')
+  lines.push('')
+  lines.push('Ask the AI to read this report, summarize the blockers, and propose or apply the fixes.')
+
+  return lines.join('\n')
+}

package/src/token-usage-reader.js

@@ -26,28 +26,70 @@
  * @see src/render-table.js
  */
 
+import { readFileSync, existsSync } from 'node:fs'
+import { join } from 'node:path'
+import { homedir } from 'node:os'
 import { loadRecentLogs } from './log-reader.js'
 
+const DEFAULT_DATA_DIR = join(homedir(), '.free-coding-models')
+const STATS_FILE = join(DEFAULT_DATA_DIR, 'token-stats.json')
+
 // 📖 buildProviderModelTokenKey keeps provider-scoped totals isolated even when
 // 📖 multiple Origins expose the same model ID.
 export function buildProviderModelTokenKey(providerKey, modelId) {
   return `${providerKey}::${modelId}`
 }
 
-// 📖 loadTokenUsageByProviderModel reads the full bounded log history available
-// 📖 through log-reader and sums tokens per exact provider+model pair.
-export function loadTokenUsageByProviderModel({ logFile, limit = 50_000 } = {}) {
-  const rows = loadRecentLogs({ logFile, limit })
+// 📖 loadTokenUsageByProviderModel prioritizes token-stats.json for accurate
+// 📖 historical totals. If missing, it falls back to parsing the bounded log history.
+export function loadTokenUsageByProviderModel({ logFile, statsFile = STATS_FILE, limit = 50_000 } = {}) {
+  // 📖 If a custom logFile is provided (Test Mode), ONLY use that file.
+  if (logFile) {
+    const testTotals = {}
+    const rows = loadRecentLogs({ logFile, limit })
+    for (const row of rows) {
+      const key = buildProviderModelTokenKey(row.provider, row.model)
+      testTotals[key] = (testTotals[key] || 0) + (Number(row.tokens) || 0)
+    }
+    return testTotals
+  }
+
   const totals = {}
 
-  for (const row of rows) {
-    const providerKey = typeof row.provider === 'string' ? row.provider : 'unknown'
-    const modelId = typeof row.model === 'string' ? row.model : 'unknown'
-    const tokens = Number(row.tokens) || 0
-    if (tokens <= 0) continue
+  // 📖 Phase 1: Try to load from the aggregated stats file (canonical source for totals)
+  try {
+    if (existsSync(statsFile)) {
+      const stats = JSON.parse(readFileSync(statsFile, 'utf8'))
+      // 📖 Aggregate byAccount entries (which use providerKey/slug/keyIdx as ID)
+      // 📖 into providerKey::modelId buckets.
+      if (stats.byAccount && typeof stats.byAccount === 'object') {
+        for (const [accountId, acct] of Object.entries(stats.byAccount)) {
+          const tokens = Number(acct.tokens) || 0
+          if (tokens <= 0) continue
+
+          // 📖 Extract providerKey and modelId from accountId (provider/model/index)
+          const parts = accountId.split('/')
+          if (parts.length >= 2) {
+            const providerKey = parts[0]
+            const modelId = parts[1]
+            const key = buildProviderModelTokenKey(providerKey, modelId)
+            totals[key] = (totals[key] || 0) + tokens
+          }
+        }
+      }
+    }
+  } catch (err) {
+    // 📖 Silently fall back to log parsing if stats file is corrupt or unreadable
+  }
 
-    const key = buildProviderModelTokenKey(providerKey, modelId)
-    totals[key] = (totals[key] || 0) + tokens
+  // 📖 Phase 2: Supplement with recent log entries if totals are still empty
+  // 📖 (e.g. fresh install or token-stats.json deleted)
+  if (Object.keys(totals).length === 0) {
+    const rows = loadRecentLogs({ limit })
+    for (const row of rows) {
+      const key = buildProviderModelTokenKey(row.provider, row.model)
+      totals[key] = (totals[key] || 0) + (Number(row.tokens) || 0)
+    }
   }
 
   return totals

package/src/utils.js

@@ -414,10 +414,29 @@ export function parseArgs(argv) {
     ? profileIdx + 1
     : -1
 
+  // New value flags
+  const sortIdx = args.findIndex(a => a.toLowerCase() === '--sort')
+  const sortValueIdx = (sortIdx !== -1 && args[sortIdx + 1] && !args[sortIdx + 1].startsWith('--'))
+    ? sortIdx + 1
+    : -1
+
+  const originIdx = args.findIndex(a => a.toLowerCase() === '--origin')
+  const originValueIdx = (originIdx !== -1 && args[originIdx + 1] && !args[originIdx + 1].startsWith('--'))
+    ? originIdx + 1
+    : -1
+
+  const pingIntervalIdx = args.findIndex(a => a.toLowerCase() === '--ping-interval')
+  const pingIntervalValueIdx = (pingIntervalIdx !== -1 && args[pingIntervalIdx + 1] && !args[pingIntervalIdx + 1].startsWith('--'))
+    ? pingIntervalIdx + 1
+    : -1
+
   // 📖 Set of arg indices that are values for flags (not API keys)
   const skipIndices = new Set()
   if (tierValueIdx !== -1) skipIndices.add(tierValueIdx)
   if (profileValueIdx !== -1) skipIndices.add(profileValueIdx)
+  if (sortValueIdx !== -1) skipIndices.add(sortValueIdx)
+  if (originValueIdx !== -1) skipIndices.add(originValueIdx)
+  if (pingIntervalValueIdx !== -1) skipIndices.add(pingIntervalValueIdx)
 
   for (const [i, arg] of args.entries()) {
     if (arg.startsWith('--') || arg === '-h') {
@@ -448,8 +467,20 @@ export function parseArgs(argv) {
   const cleanProxyMode = flags.includes('--clean-proxy') || flags.includes('--proxy-clean')
   const jsonMode = flags.includes('--json')
   const helpMode = flags.includes('--help') || flags.includes('-h')
+  const premiumMode = flags.includes('--premium')
+
+  // New boolean flags
+  const sortDesc = flags.includes('--desc')
+  const sortAscFlag = flags.includes('--asc')
+  const hideUnconfigured = flags.includes('--hide-unconfigured')
+  const showUnconfigured = flags.includes('--show-unconfigured')
+  const disableWidthsWarning = flags.includes('--disable-widths-warning')
 
   let tierFilter = tierValueIdx !== -1 ? args[tierValueIdx].toUpperCase() : null
+  let sortColumn = sortValueIdx !== -1 ? args[sortValueIdx].toLowerCase() : null
+  let originFilter = originValueIdx !== -1 ? args[originValueIdx] : null
+  let pingInterval = pingIntervalValueIdx !== -1 ? parseInt(args[pingIntervalValueIdx], 10) : null
+  let sortDirection = sortDesc ? 'desc' : (sortAscFlag ? 'asc' : null)
 
   const profileName = profileValueIdx !== -1 ? args[profileValueIdx] : null
 
@@ -478,6 +509,14 @@ export function parseArgs(argv) {
     jsonMode,
     helpMode,
     tierFilter,
+    sortColumn,
+    sortDirection,
+    originFilter,
+    pingInterval,
+    hideUnconfigured,
+    showUnconfigured,
+    disableWidthsWarning,
+    premiumMode,
     profileName,
     recommendMode
   }