@lawaty/lacode 2.0.0-beta.0 → 2.2.0-beta.0

package/scripts/score-models.mjs

@@ -12,7 +12,8 @@
 //   npm run score -- --no-write         print only, touch nothing
 //
 // WHAT IT MEASURES
-//   context      limit.context from `opencode models --verbose`
+//   context      limit.context from `opencode models --verbose`; a known limit
+//                below --min-context is excluded from ranking, not scored low
 //   output       limit.output  (the token ceiling for one reply)
 //   speed        wall-clock ms for one tiny completion, one request per model
 //   reliability  did the model actually answer? A 429/401/404 is recorded as the
@@ -21,10 +22,14 @@
 // WHAT IT DOES NOT DO
 //   * It never invents a latency or a score. A model it could not reach is shown
 //     as unreachable with the reason and is NOT ranked.
+//   * It never ranks a model whose known context is below --min-context, or whose
+//     measured score is below --min-score: those are listed as excluded instead.
 //   * It never writes pool.txt. It prints a SUGGESTED table for you to paste; the
 //     pool stays yours to decide.
-//   * It never probes a paid model. Only cost 0/0 models from the provider's
-//     catalog are candidates (non-chat/media/safety/meta-router models are skipped).
+//   * opencode and opencode-go stay free-only. Every other provider is probed
+//     even when its catalogue lists a price, because provider credits cover it;
+//     --free-only restores the cost-0 rule everywhere. Non-chat/media/safety/
+//     meta-router models are always skipped.
 //
 // OUTPUT
 //   a per-provider ranked table, an overall ranking, a suggested pool.txt block,
@@ -35,11 +40,14 @@ import { execFileSync, spawn } from "node:child_process"
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"
 import { homedir } from "node:os"
 import { dirname, join } from "node:path"
+import { MIN_CONTEXT, MIN_SCORE, retryAfterMs, selectPool } from "./pool-selection.mjs"
 
 const AUTH = join(homedir(), ".local", "share", "opencode", "auth.json")
 const OUT = join(homedir(), ".config", "lacode", "model-scores.json")
-const DEFAULT_PROVIDERS = ["opencode", "opencode-go", "openrouter", "console-openrouter"]
+// No default provider list: `opencode models --verbose` with no argument returns
+// every provider opencode knows about, and that is what we scan by default.
 const SESSION_PROVIDERS = new Set(["opencode", "opencode-go"])
+const FREE_ONLY_PROVIDERS = new Set(["opencode", "opencode-go"])
 
 // ── scoring weights ─────────────────────────────────────────────────────────
 // Mechanical work prizes speed and availability over raw capability. Context is
@@ -50,6 +58,9 @@ const FAST_MS = 300 // at or below this, speed scores 1.0
 const SLOW_MS = 8_000 // at or above this, speed scores 0
 const PROMPT = "Reply with the single word: ok"
 const MAX_TOKENS = 8
+const PROBE_ATTEMPTS = 2 // one retry for a timeout or a dropped connection
+const isTransientProbe = (reason) =>
+  /timeout|timed out|ECONNRESET|ETIMEDOUT|fetch failed|socket hang up|no answer/i.test(String(reason ?? ""))
 
 // ── args ────────────────────────────────────────────────────────────────────
 const argv = process.argv.slice(2)
@@ -63,27 +74,37 @@ const value = (flag, fallback) => {
   if (has("--help") || has("-h")) {
     console.log(`Usage: npm run score -- [options]
 
-  --providers a,b   providers to scan (default: ${DEFAULT_PROVIDERS.join(",")})
-  --max N           probe at most N free models per provider (default: 12)
-  --timeout MS      per-request deadline in ms (default: 20000)
+  --providers a,b   restrict the scan to these providers (default: all configured)
+  --max N           probe at most N candidate models per provider (default: 50)
+  --min-context N   exclude models whose context limit is below N (default: ${MIN_CONTEXT})
+  --min-score N     do not rank models scoring below N (default: ${MIN_SCORE})
+  --free-only        only probe catalogue cost 0/0 models on every provider
+  --timeout MS      per-request deadline in ms (default: 45000)
   --delay MS        pause between requests in ms (default: 300)
   --no-write        do not write ~/.config/lacode/model-scores.json
   -h, --help        this message
 
-Only cost 0/0 concrete text chat/completion models are candidates
-(non-chat/media generators, safety classifiers, and OpenRouter meta-routers are skipped).
+opencode and opencode-go stay free-only; every other provider is probed even
+when its catalogue lists a price, because provider credits cover it. Use
+--free-only to apply the cost-0 rule everywhere. Non-chat/media generators,
+safety classifiers, and OpenRouter meta-routers are always skipped.
 Credentials come from ~/.local/share/opencode/auth.json; a provider with no
 entry falls back to the "opencode" key (which is how the console-openrouter
 connection authenticates).
-opencode/opencode-go models are probed through a temporary local opencode serve --pure session, while the other providers are probed directly.`)
+opencode/opencode-go models are probed through a temporary local opencode serve --pure session, while the other providers are probed directly.
+Providers with no direct HTTP endpoint (or an unexpanded \${...} URL) are probed through the same local opencode session as opencode/opencode-go.`)
     process.exit(0)
   }
 
-const providers = String(value("--providers", DEFAULT_PROVIDERS.join(","))).split(",").map((s) => s.trim()).filter(Boolean)
-const maxPerProvider = Math.max(1, Number(value("--max", "12")) || 12)
-const timeoutMs = Math.max(1_000, Number(value("--timeout", "20000")) || 20_000)
+const providerArg = value("--providers", null)
+const providers = providerArg ? String(providerArg).split(",").map((s) => s.trim()).filter(Boolean) : null
+const maxPerProvider = Math.max(1, Number(value("--max", "50")) || 50)
+const minContext = Math.max(0, Number(value("--min-context", String(MIN_CONTEXT))) || MIN_CONTEXT)
+const minScore = Math.max(0, Number(value("--min-score", String(MIN_SCORE))) || MIN_SCORE)
+const timeoutMs = Math.max(1_000, Number(value("--timeout", "45000")) || 45_000)
 const delayMs = Math.max(0, Number(value("--delay", "300")) || 300)
 const writeResults = !has("--no-write")
+const freeOnlyFlag = has("--free-only")
 
 // ── discovery ───────────────────────────────────────────────────────────────
 /** Parse `opencode models --verbose <provider>`: alternating id line + JSON block. */
@@ -136,21 +157,11 @@ function isChatModel(model) {
   return true
 }
 
-function freeModels(provider) {
-  let text
-  try {
-    text = execFileSync("opencode", ["models", "--verbose", provider], {
-      encoding: "utf8",
-      maxBuffer: 64 * 1024 * 1024,
-      stdio: ["ignore", "pipe", "ignore"],
-    })
-  } catch (error) {
-    return { provider, error: `opencode models failed: ${String(error?.message ?? error)}`, models: [] }
-  }
-  const rows = parseVerbose(text)
+function buildProviderEntry(provider, rows) {
+  const freeOnly = freeOnlyFlag || FREE_ONLY_PROVIDERS.has(provider)
   const models = rows
-    .filter(({ model }) => model?.api?.url && model?.api?.id)
-    .filter(({ model }) => (model.cost?.input ?? 1) === 0 && (model.cost?.output ?? 1) === 0)
+    .filter(({ model }) => model?.api?.id)
+    .filter(({ model }) => !freeOnly || ((model.cost?.input ?? 1) === 0 && (model.cost?.output ?? 1) === 0))
     .filter(({ model }) => !model.status || model.status === "active")
     .filter(({ model }) => isChatModel(model))
     .map(({ id, model }) => ({
@@ -161,7 +172,57 @@ function freeModels(provider) {
       output: model.limit?.output ?? null,
       name: model.name ?? id,
     }))
-  return { provider, error: null, models }
+  const excluded = models
+    .filter((m) => m.context != null && m.context < minContext)
+    .map((m) => ({
+      model: m.model,
+      context: m.context,
+      output: m.output,
+      reason: `context ${m.context} < minimum ${minContext}`,
+    }))
+  return {
+    provider,
+    error: null,
+    models: models.filter((m) => m.context == null || m.context >= minContext),
+    excluded,
+  }
+}
+
+function freeModels(provider) {
+  let text
+  try {
+    text = execFileSync("opencode", ["models", "--verbose", provider], {
+      encoding: "utf8",
+      maxBuffer: 64 * 1024 * 1024,
+      stdio: ["ignore", "pipe", "ignore"],
+    })
+  } catch (error) {
+    return { provider, error: `opencode models failed: ${String(error?.message ?? error)}`, models: [], excluded: [] }
+  }
+  return buildProviderEntry(provider, parseVerbose(text))
+}
+
+function freeModelsAll() {
+  let text
+  try {
+    text = execFileSync("opencode", ["models", "--verbose"], {
+      encoding: "utf8",
+      maxBuffer: 64 * 1024 * 1024,
+      stdio: ["ignore", "pipe", "ignore"],
+    })
+  } catch (error) {
+    return [{ provider: "all", error: `opencode models failed: ${String(error?.message ?? error)}`, models: [], excluded: [] }]
+  }
+  const groups = new Map()
+  for (const row of parseVerbose(text)) {
+    const provider = String(row.model?.providerID ?? row.id.split("/")[0] ?? "")
+    if (!provider) continue
+    if (!groups.has(provider)) groups.set(provider, [])
+    groups.get(provider).push(row)
+  }
+  return [...groups.entries()]
+    .sort(([a], [b]) => a.localeCompare(b))
+    .map(([provider, rows]) => buildProviderEntry(provider, rows))
 }
 
 // ── credentials ─────────────────────────────────────────────────────────────
@@ -322,10 +383,17 @@ async function probeDirect(candidate, provider) {
             : response.status === 404
               ? "not served"
               : `http ${response.status}`
-      return { ok: false, ms, reason: `${kind}: ${detail}` }
+      const retryAfter =
+        kind === "rate-limited" || /credit|402/i.test(detail)
+          ? retryAfterMs(body, response.headers.get("retry-after"))
+          : undefined
+      return { ok: false, ms, reason: `${kind}: ${detail}`, retryAfter }
     }
     // A 200 can still carry an error object (OpenRouter does this).
-    if (parsed?.error) return { ok: false, ms, reason: `${parsed.error.message ?? "error"} `.trim() }
+    if (parsed?.error) {
+      const reason = `${parsed.error.message ?? "error"} `.trim()
+      return { ok: false, ms, reason, retryAfter: retryAfterMs(reason) }
+    }
     if (!Array.isArray(parsed?.choices) || parsed.choices.length === 0) {
       return { ok: false, ms, reason: "no choices in response" }
     }
@@ -359,7 +427,7 @@ async function probeViaSession(candidate, provider) {
     if (!createRes.ok) {
       const detail = createdJson?.error?.message ?? createdJson?.message ?? createText.slice(0, 160)
       const ms = Math.round(performance.now() - started)
-      return { ok: false, ms, reason: `session create http ${createRes.status}: ${detail}` }
+      return { ok: false, ms, reason: `session create http ${createRes.status}: ${detail}`, retryAfter: retryAfterMs(detail) }
     }
     id = createdJson?.id ?? createdJson?.data?.id ?? null
     if (!id) {
@@ -385,11 +453,12 @@ async function probeViaSession(candidate, provider) {
     const ms = Math.round(performance.now() - (promptStarted ?? started))
     if (!msgRes.ok) {
       const detail = parsed?.error?.message ?? parsed?.message ?? msgText.slice(0, 160)
-      return { ok: false, ms, reason: `http ${msgRes.status}: ${detail}` }
+      return { ok: false, ms, reason: `http ${msgRes.status}: ${detail}`, retryAfter: retryAfterMs(detail) }
     }
     const failure = parsed?.error ?? parsed?.info?.error
     if (failure) {
-      return { ok: false, ms, reason: failureText(failure).slice(0, 240) }
+      const reason = failureText(failure).slice(0, 240)
+      return { ok: false, ms, reason, retryAfter: retryAfterMs(reason) }
     }
     if (!parsed?.info) {
       return { ok: false, ms, reason: "no response" }
@@ -400,7 +469,8 @@ async function probeViaSession(candidate, provider) {
     if (error?.name === "TimeoutError") {
       return { ok: false, ms, reason: `timeout after ${timeoutMs}ms` }
     }
-    return { ok: false, ms, reason: failureText(error).slice(0, 240) }
+    const reason = failureText(error).slice(0, 240)
+    return { ok: false, ms, reason, retryAfter: retryAfterMs(reason) }
   } finally {
     if (id && server) {
       try {
@@ -410,9 +480,22 @@ async function probeViaSession(candidate, provider) {
   }
 }
 
+function usesSession(provider, candidate) {
+  if (SESSION_PROVIDERS.has(provider)) return true
+  const url = String(candidate.url ?? "")
+  return url.length === 0 || url.includes("${")
+}
+
 async function probe(candidate, provider) {
-  if (SESSION_PROVIDERS.has(provider)) return probeViaSession(candidate, provider)
-  return probeDirect(candidate, provider)
+  if (usesSession(provider, candidate)) return probeViaSession(candidate, provider)
+  const outcome = await probeDirect(candidate, provider)
+  if (outcome.ok) return outcome
+  // A provider whose direct URL is misconfigured can still be reachable
+  // through opencode's own provider resolution.
+  if (/credential|no credential|not served|invalid request path|could not route/i.test(String(outcome.reason ?? ""))) {
+    return probeViaSession(candidate, provider)
+  }
+  return outcome
 }
 
 // ── scoring ─────────────────────────────────────────────────────────────────
@@ -436,32 +519,51 @@ const pad = (s, n) => String(s).padEnd(n)
 const padL = (s, n) => String(s).padStart(n)
 
 async function main() {
-  const scanned = providers.map(freeModels)
+  const scanned = providers ? providers.map((provider) => freeModels(provider)) : freeModelsAll()
   const results = []
 
   for (const entry of scanned) {
     if (entry.error) {
       console.log(`\n${entry.provider}: ${entry.error}`)
-      results.push({ provider: entry.provider, error: entry.error, rows: [] })
+      results.push({ provider: entry.provider, error: entry.error, rows: [], excluded: [] })
       continue
     }
     const candidates = entry.models.slice(0, maxPerProvider)
-    console.log(`\n${entry.provider}: probing ${candidates.length} free model(s)${entry.models.length > candidates.length ? ` of ${entry.models.length}` : ""}`)
+    const excludedNote = entry.excluded.length ? `, ${entry.excluded.length} excluded (context < ${minContext})` : ""
+    console.log(`\n${entry.provider}: probing ${candidates.length} candidate model(s)${entry.models.length > candidates.length ? ` of ${entry.models.length}` : ""}${excludedNote}`)
+    if (entry.excluded.length) {
+      console.log(`  excluded — context below ${minContext}:`)
+      for (const x of entry.excluded) console.log(`    ${pad(x.model.replace(`${entry.provider}/`, ""), 52)} ${padL(x.context, 9)}`)
+    }
     if (candidates.length === 0) {
-      results.push({ provider: entry.provider, error: null, rows: [] })
+      results.push({ provider: entry.provider, error: null, rows: [], excluded: entry.excluded })
       continue
     }
     const rows = []
     for (const candidate of candidates) {
-      const outcome = await probe(candidate, entry.provider)
+      let outcome = await probe(candidate, entry.provider)
+      for (let attempt = 1; attempt < PROBE_ATTEMPTS && !outcome.ok && isTransientProbe(outcome.reason); attempt++) {
+        console.log(`  retrying ${candidate.model.replace(`${entry.provider}/`, "")} after ${String(outcome.reason).slice(0, 60)}`)
+        await sleep(delayMs)
+        outcome = await probe(candidate, entry.provider)
+      }
       const row = { provider: entry.provider, ...candidate, ...outcome, score: null }
       row.score = score(row)
+      if (row.score && row.score.value < minScore) {
+        entry.excluded.push({
+          model: row.model,
+          context: row.context,
+          output: row.output,
+          score: row.score.value,
+          reason: `score ${row.score.value} < minimum ${minScore}`,
+        })
+      }
       rows.push(row)
       const verdict = outcome.ok ? `${outcome.ms}ms` : outcome.reason.slice(0, 60)
       console.log(`  ${pad(candidate.model.replace(`${entry.provider}/`, ""), 52)} ${padL(verdict, 16)}`)
       await sleep(delayMs)
     }
-    results.push({ provider: entry.provider, error: null, rows })
+    results.push({ provider: entry.provider, error: null, rows, excluded: entry.excluded })
   }
   stopSessionServer()
 
@@ -470,7 +572,7 @@ async function main() {
     if (entry.error || entry.rows.length === 0) continue
     console.log(`\n${entry.provider} — ranked`)
     console.log(`  ${pad("#", 3)}${pad("model", 52)}${padL("ctx", 9)}${padL("out", 9)}${padL("ms", 7)}${padL("score", 7)}`)
-    const ranked = [...entry.rows].filter((r) => r.score).sort((a, b) => b.score.value - a.score.value)
+    const ranked = [...entry.rows].filter((r) => r.score && r.score.value >= minScore).sort((a, b) => b.score.value - a.score.value)
     ranked.forEach((r, i) => {
       console.log(`  ${pad(i + 1, 3)}${pad(r.model.replace(`${entry.provider}/`, ""), 52)}${padL(r.context ?? "?", 9)}${padL(r.output ?? "?", 9)}${padL(r.ms ?? "?", 7)}${padL(r.score.value, 7)}`)
     })
@@ -481,32 +583,52 @@ async function main() {
     }
   }
 
+  // Models dropped for a too-small context or a too-low score are reported, never ranked.
+  const excluded = results.flatMap((e) => e.excluded ?? [])
+  if (excluded.length) {
+    console.log(`\nexcluded — below the context or score floor (not ranked)`)
+    for (const x of excluded) console.log(`  ${pad(x.model, 58)}${padL(x.context ?? "?", 9)}  ${x.reason}`)
+  }
+
   // Overall ranking across providers.
-  const all = results.flatMap((e) => e.rows).filter((r) => r.score)
+  const all = results.flatMap((e) => e.rows).filter((r) => r.score && r.score.value >= minScore)
   all.sort((a, b) => b.score.value - a.score.value)
   if (all.length) {
-    console.log(`\noverall — best free models right now`)
+    console.log(`\noverall — best candidate models right now`)
     all.slice(0, 12).forEach((r, i) => {
       console.log(`  ${pad(i + 1, 3)}${pad(r.model, 58)}${padL(`${r.ms}ms`, 8)}${padL(r.score.value, 7)}`)
     })
   }
 
-  // A SUGGESTED pool: the best model from each provider, weights 4/3/2/1 by rank.
-  // Printed, never written — pool.txt stays the user's decision.
-  const bestPerProvider = []
-  for (const entry of results) {
-    const top = entry.rows.filter((r) => r.score).sort((a, b) => b.score.value - a.score.value)[0]
-    if (top) bestPerProvider.push(top)
-  }
-  bestPerProvider.sort((a, b) => b.score.value - a.score.value)
-  if (bestPerProvider.length) {
+  // Previous scores let a transient outage keep the pool it had.
+  const previousScores = (() => {
+    try {
+      return JSON.parse(readFileSync(OUT, "utf8"))
+    } catch {
+      return null
+    }
+  })()
+  const { selection, warnings, retry } = selectPool({
+    providers: results.map((entry) => ({
+      provider: entry.provider,
+      error: entry.error,
+      rows: entry.rows.map((row) => ({ ...row, score: row.score?.value ?? null })),
+      excluded: entry.excluded,
+    })),
+    previousProviders: previousScores?.providers,
+    minContext,
+    minScore,
+  })
+
+  if (selection.length) {
     console.log(`\nsuggested pool.txt (paste if it matches your judgement — nothing was changed)`)
-    console.log(`# scored ${new Date().toISOString()} — one best free model per provider`)
-    bestPerProvider.forEach((r, i) => {
-      const weight = Math.max(1, 4 - i)
-      console.log(`${pad(r.model, 60)}${weight}`)
-    })
+    console.log(`# scored ${new Date().toISOString()} — opencode-go first (weight 4), small-context providers (weight 2), large-context providers (weight 1), opencode last (max 1)`)
+    for (const entry of selection) {
+      const bucket = entry.bucket ? `  b:${entry.bucket}` : ""
+      console.log(`${pad(entry.model, 60)}${entry.weight}${bucket}`)
+    }
   }
+  if (retry) console.log(`\nlimited: ${retry.models.length} model(s); retry after ${retry.after}`)
 
   if (writeResults) {
     try {
@@ -514,9 +636,15 @@ async function main() {
       const payload = {
         generatedAt: new Date().toISOString(),
         weights: W,
+        minContext,
+        minScore,
+        selection,
+        warnings,
+        retry,
         providers: results.map((e) => ({
           provider: e.provider,
           error: e.error,
+          excluded: e.excluded ?? [],
           rows: e.rows.map((r) => ({
             model: r.model,
             context: r.context,
@@ -525,6 +653,7 @@ async function main() {
             ms: r.ms,
             reason: r.reason,
             score: r.score?.value ?? null,
+            retryAfter: r.retryAfter ?? null,
           })),
         })),
       }