@lawaty/lacode 2.3.0-beta.4 → 2.3.0-beta.6

package/scripts/score-models.mjs

@@ -46,13 +46,15 @@ import {
   classifyFailure,
   isTransientFailure,
   isTransientKind,
+  parsePoolDirectives,
+  parsePoolSlots,
   providerRank,
   retryAfterMs,
   selectPool,
 } from "./pool-selection.mjs"
 
 const AUTH = join(homedir(), ".local", "share", "opencode", "auth.json")
-const OUT = join(homedir(), ".config", "lacode", "model-scores.json")
+const DEFAULT_OUT = join(homedir(), ".config", "lacode", "model-scores.json")
 // No default provider list: `opencode models --verbose` with no argument returns
 // every provider opencode knows about, and that is what we scan by default.
 const SESSION_PROVIDERS = new Set(["opencode", "opencode-go"])
@@ -100,6 +102,13 @@ const value = (flag, fallback) => {
     console.log(`Usage: npm run score -- [options]
 
   --providers a,b   restrict the scan to these providers (default: all configured)
+  --probe-models a,b probe exactly these provider/model ids (canary: skips the
+                    catalogue scan for everything else; unscanned providers keep
+                    their previous rows)
+  --deprioritize-providers a,b probe these providers last (a live status hint;
+                    never excludes anything, ordering only)
+  --current-pool PATH feed pool.txt as the current pool so selection retains
+                    members this run could not reach (default: no retention)
   --max N           probe at most N candidate models per provider (default: 50)
   --min-context N   exclude models whose context limit is below N (default: ${MIN_CONTEXT})
   --min-score N     do not rank models scoring below N (default: ${MIN_SCORE})
@@ -126,6 +135,16 @@ Providers with no direct HTTP endpoint (or an unexpanded \${...} URL) are probed
 
 const providerArg = value("--providers", null)
 const providers = providerArg ? String(providerArg).split(",").map((s) => s.trim()).filter(Boolean) : null
+const probeModelsArg = value("--probe-models", null)
+const probeModels = probeModelsArg ? new Set(String(probeModelsArg).split(",").map((s) => s.trim()).filter(Boolean)) : null
+const deprioritizeArg = value("--deprioritize-providers", null)
+  const deprioritizedProviders = new Set(
+    deprioritizeArg ? String(deprioritizeArg).split(",").map((s) => s.trim()).filter(Boolean) : [],
+  )
+  // probe --probe-models ids absent from every catalogue. Filled during the
+  // catalogue filter above; emitted as synthetic not-served rows after probing.
+  const missingProbeModels = []
+const currentPoolPath = value("--current-pool", null)
 const maxPerProvider = Math.max(1, Number(value("--max", "50")) || 50)
 const minContext = Math.max(0, Number(value("--min-context", String(MIN_CONTEXT))) || MIN_CONTEXT)
 const minScore = Math.max(0, Number(value("--min-score", String(MIN_SCORE))) || MIN_SCORE)
@@ -133,6 +152,7 @@ const timeoutMs = Math.max(1_000, Number(value("--timeout", "12000")) || 12_000)
 const delayMs = Math.max(0, Number(value("--delay", "300")) || 300)
 const budgetMin = Math.max(1, Number(value("--budget-min", "30")) || 30)
 const budgetMs = budgetMin * 60_000
+const OUT = value("--out", DEFAULT_OUT)
 const writeResults = !has("--no-write")
 const freeOnlyFlag = has("--free-only")
 
@@ -570,12 +590,43 @@ const pad = (s, n) => String(s).padEnd(n)
 const padL = (s, n) => String(s).padStart(n)
 
 async function main() {
-  const scanned = providers ? providers.map((provider) => freeModels(provider)) : freeModelsAll()
+  const currentPoolText = (() => {
+    if (!currentPoolPath) return ""
+    try {
+      return readFileSync(currentPoolPath, "utf8")
+    } catch {
+      return ""
+    }
+  })()
+  const excludedProviders = parsePoolDirectives(currentPoolText)
+  let scanned = providers ? providers.map((provider) => freeModels(provider)) : freeModelsAll()
+  if (excludedProviders.size > 0) {
+    scanned = scanned.filter((entry) => !excludedProviders.has(entry.provider))
+    if (scanned.length === 0) console.log(`every provider was excluded by pool.txt directives; nothing to probe`)
+  }
   // Core providers first, then the free-only middle tier, then the slow paid
-  // catalogues. Ordering only; every provider is still probed if time allows.
+  // catalogues — and explicitly deprioritized providers last. Ordering only;
+  // every provider is still probed if time allows.
   scanned.sort(
-    (a, b) => providerRank(a.provider) - providerRank(b.provider) || String(a.provider).localeCompare(String(b.provider)),
+    (a, b) =>
+      Number(deprioritizedProviders.has(a.provider)) - Number(deprioritizedProviders.has(b.provider)) ||
+      providerRank(a.provider) - providerRank(b.provider) ||
+      String(a.provider).localeCompare(String(b.provider)),
   )
+  const currentSlots = parsePoolSlots(currentPoolText).filter(
+    (slot) => !excludedProviders.has(String(slot.model).split("/")[0]),
+  )
+  if (probeModels && probeModels.size > 0) {
+    const seen = new Set()
+    scanned = scanned.flatMap((entry) => {
+      if (entry.error) return []
+      const models = (entry.models ?? []).filter((candidate) => probeModels.has(candidate.model))
+      models.forEach((candidate) => seen.add(candidate.model))
+      if (models.length === 0) return []
+      return [{ ...entry, models }]
+    })
+    missingProbeModels.push(...[...probeModels].filter((model) => !seen.has(model)))
+  }
   const results = []
   const deadline = Date.now() + budgetMs
   const skippedProviders = []
@@ -620,6 +671,12 @@ async function main() {
         outcome = await probe(candidate, entry.provider)
       }
       const kind = outcome.kind ?? classifyFailure({ status: outcome.status, reason: outcome.reason })
+      // Only cheap-failure kinds that also stall the scan count toward the
+      // breaker. A rate limit is deliberately NOT included: a 429 answers in
+      // milliseconds, so truncating on it saves little, and it would leave the
+      // provider's pool members unscanned -- which means no current failure row,
+      // so their consecutive-failure streak could never advance and they could
+      // never retire. Probe every model and record the evidence instead.
       const providerFault = !outcome.ok && ["timeout", "network", "server"].includes(kind)
       consecutiveTransient = providerFault ? consecutiveTransient + 1 : 0
       const row = { provider: entry.provider, ...candidate, ...outcome, score: null }
@@ -655,6 +712,32 @@ async function main() {
   }
   stopSessionServer()
 
+  // A canary model absent from every catalogue is not "unscanned" — it is
+  // gone. A synthetic not-served row makes the selector drop it (hard-dead)
+  // instead of retaining it blindly.
+  for (const model of missingProbeModels) {
+    const slash = model.indexOf("/")
+    const provider = slash === -1 ? "unknown" : model.slice(0, slash)
+    const row = {
+      provider,
+      model,
+      url: "",
+      apiId: slash === -1 ? model : model.slice(slash + 1),
+      context: null,
+      output: null,
+      ok: false,
+      ms: null,
+      reason: "not in provider catalogue (model-missing)",
+      status: 404,
+      kind: "not-served",
+      score: null,
+      retryAfter: null,
+    }
+    const entry = results.find((candidate) => candidate.provider === provider)
+    if (entry) entry.rows.push(row)
+    else results.push({ provider, error: null, rows: [row], excluded: [], truncated: false, degraded: false })
+  }
+
   // Per-provider ranked tables.
   for (const entry of results) {
     if (entry.error || entry.rows.length === 0) continue
@@ -696,15 +779,20 @@ async function main() {
       return null
     }
   })()
-  const { selection, warnings, retry } = selectPool({
-    providers: results.map((entry) => ({
-      provider: entry.provider,
-      error: entry.error,
-      truncated: entry.truncated,
-      rows: entry.rows.map((row) => ({ ...row, score: row.score?.value ?? null })),
-      excluded: entry.excluded,
-    })),
-    previousProviders: previousScores?.providers,
+  const { selection, fallback, warnings, retry } = selectPool({
+    providers: results
+      .filter((entry) => !excludedProviders.has(entry.provider))
+      .map((entry) => ({
+        provider: entry.provider,
+        error: entry.error,
+        truncated: entry.truncated,
+        rows: entry.rows.map((row) => ({ ...row, score: row.score?.value ?? null })),
+        excluded: entry.excluded,
+      })),
+    previousProviders: (previousScores?.providers ?? []).filter(
+      (entry) => !excludedProviders.has(String(entry?.provider ?? "")),
+    ),
+    currentSlots,
     minContext,
     minScore,
   })
@@ -716,6 +804,14 @@ async function main() {
     console.log(`\nsuggested pool.txt (paste if it matches your judgement — nothing was changed)`)
     console.log(`# scored ${new Date().toISOString()} — opencode-go first (weight 4), small-context providers (weight 2), large-context providers (weight 1), opencode last (max 1)`)
     for (const entry of selection) {
+      const ceiling = Number.isFinite(entry.ceiling) && entry.ceiling > entry.weight ? `  ceil:${entry.ceiling}` : ""
+      const bucket = entry.bucket ? `  b:${entry.bucket}` : ""
+      console.log(`${pad(entry.model, 60)}${entry.weight}${ceiling}${bucket}`)
+    }
+  }
+  if (fallback.length) {
+    console.log(`\nfallback section (used only when every primary slot is unavailable)`)
+    for (const entry of fallback) {
       const bucket = entry.bucket ? `  b:${entry.bucket}` : ""
       console.log(`${pad(entry.model, 60)}${entry.weight}${bucket}`)
     }
@@ -729,7 +825,9 @@ async function main() {
       // entry forward verbatim so the next run still sees it and selectPool can
       // keep its rankable models.
       const scannedNames = new Set(results.map((e) => e.provider))
-      const carried = (previousScores?.providers ?? []).filter((p) => !scannedNames.has(String(p?.provider ?? "")))
+      const carried = (previousScores?.providers ?? []).filter(
+        (p) => !scannedNames.has(String(p?.provider ?? "")) && !excludedProviders.has(String(p?.provider ?? "")),
+      )
       const previousByModel = new Map()
       for (const provider of previousScores?.providers ?? []) {
         for (const row of provider?.rows ?? []) {
@@ -742,6 +840,7 @@ async function main() {
         minContext,
         minScore,
         selection,
+        fallback,
         warnings,
         retry,
         providers: [
@@ -763,14 +862,21 @@ async function main() {
                 kind: r.kind ?? null,
                 score: r.score?.value ?? null,
                 retryAfter: r.retryAfter ?? null,
+                failStreak: 0,
               }
               // A transient failure must not erase the last measured ranking:
               // the next run needs it to keep a previously-good model while a
-              // gateway is down. Keep both rows; the selector prefers the
-              // rankable one and still schedules the failed one for retry.
+              // gateway is down. But a model that keeps failing scan after scan
+              // is not having a bad moment, and the selector must be able to see
+              // that: carry a consecutive-failure count forward on both rows so
+              // it can retire the slot after MAX_TRANSIENT_STREAK scans. A
+              // successful scan emits a fresh row with a zero streak.
               if (r.ok === false && isTransientFailure(r)) {
                 const prior = previousByModel.get(r.model)
-                if (prior?.ok === true && typeof prior.score === "number") return [current, prior]
+                const failStreak = Number(prior?.failStreak ?? 0) + 1
+                const failed = { ...current, failStreak }
+                if (prior?.ok === true && typeof prior.score === "number") return [failed, { ...prior, failStreak }]
+                return [failed]
               }
               return [current]
             }),