lynkr 9.9.0 → 9.10.0

package/scripts/mine-difficulty-anchors.js

@@ -0,0 +1,288 @@
+#!/usr/bin/env node
+/**
+ * Mine empirically-labeled difficulty anchors from public router datasets.
+ *
+ * Sources (fetched via the HF datasets-server JSON API, no deps):
+ *  - RouteWorks/RouterArena (full split): per-query empirical difficulty
+ *    bands — easy (≥20/42 models correct), medium (5–19/42), hard (≤4/42).
+ *  - routellm/gpt4_dataset: per-prompt mixtral_score 1–5 (GPT-4-judged
+ *    quality of the weak model's answer).
+ *
+ * Label recipe (config B: local → GLM → Claude):
+ *  - substantive (45, MEDIUM/ollama):  RouterArena easy  + mixtral_score 5
+ *  - heavyweight (68, COMPLEX/GLM):    RouterArena medium + mixtral_score ≤2
+ *  - frontier    (85, REASONING/Claude): RouterArena hard + hand examples
+ *  - trivial stays hand-curated — benchmarks have no conversational prompts.
+ *
+ * Output: data/difficulty-anchors.json (gitignored, wins over config/) and
+ * data/difficulty-anchors.provenance.json. The bundled 13-anchor default in
+ * config/ is untouched.
+ *
+ * Requires local Ollama with nomic-embed-text (for diversity selection).
+ * Usage: node scripts/mine-difficulty-anchors.js [--dry-run]
+ */
+
+const fs = require("fs");
+const path = require("path");
+
+const DS_SERVER = "https://datasets-server.huggingface.co/rows";
+const OLLAMA_EMBED = process.env.OLLAMA_EMBEDDINGS_ENDPOINT || "http://localhost:11434/api/embeddings";
+const EMBED_MODEL = process.env.OLLAMA_EMBEDDINGS_MODEL || "nomic-embed-text";
+
+const OUT_ANCHORS = path.join(__dirname, "../data/difficulty-anchors.json");
+const OUT_PROVENANCE = path.join(__dirname, "../data/difficulty-anchors.provenance.json");
+const CONFIG_ANCHORS = path.join(__dirname, "../config/difficulty-anchors.json");
+
+// Per-class targets after diversity selection (mined only, hand anchors extra)
+const TARGETS = { substantive: 130, heavyweight: 130, frontier: 100 };
+
+// Hand-written frontier examples: engineering-flavored deep-reasoning asks.
+// RouterArena's hard band skews competition-academic; real Lynkr frontier
+// traffic looks like these. Both kinds go in.
+const HAND_FRONTIER = [
+  "Prove the correctness of this lock-free queue implementation and identify any ABA hazards",
+  "Design a novel conflict-free replicated data type for collaborative rich-text editing and argue its convergence from first principles",
+  "Derive the optimal cache eviction policy for this access distribution and prove its competitive ratio",
+  "Diagnose this heisenbug: a race that only reproduces under load across three services, given these interleaved logs",
+  "Design a Byzantine-fault-tolerant consensus protocol variant that tolerates f faults with 2f+1 replicas by weakening liveness, and analyze exactly which guarantees are lost",
+  "Formally verify that this state machine can never deadlock, or produce a counterexample trace",
+  "Given these profiler traces, determine whether the tail latency is queueing-theoretic or GC-driven, and prove which by constructing a discriminating experiment",
+  "Reason through the security of this key-rotation scheme under an adversary who can observe but not modify traffic, and find the weakest assumption it relies on",
+  "Work out the exact memory ordering constraints this concurrent hashmap needs on ARM, and justify each barrier from the C++ memory model",
+  "Given this failing distributed transaction trace, determine whether the anomaly is write skew or lost update, and design the minimal isolation-level change that eliminates it",
+  "Derive the amortized complexity of this self-adjusting tree under this adversarial access pattern, and prove the bound is tight",
+  "Analyze whether this migration can be run online without violating any invariant the application relies on, enumerating the interleavings that could corrupt state",
+  "Reverse-engineer why this JIT deoptimizes on this hot path and design a fix that preserves the semantics, with reasoning about hidden class transitions",
+  "Prove this rate limiter is fair under concurrent refill, or construct the starvation schedule that breaks it",
+  "Design an exactly-once delivery layer on top of an at-least-once queue without idempotency keys, or prove it is impossible under these constraints",
+  "Given these heap dumps over time, reason to the retention path causing the leak and rule out the two most plausible alternative explanations",
+  "Reconcile these conflicting benchmark results by identifying the confound in the experimental setup, and design the experiment that isolates it",
+  "Determine the weakest consistency model under which this caching layer is still linearizable from the client's perspective, with proof",
+  "Plan a zero-downtime re-sharding of this 2TB database under sustained writes, reasoning through every failure mode and its recovery path",
+  "Explain step by step why this floating-point summation loses precision at scale and derive the compensated algorithm that bounds the error",
+];
+
+function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
+
+async function fetchRows(dataset, config, split, offset, length) {
+  const url = `${DS_SERVER}?dataset=${encodeURIComponent(dataset)}&config=${config}&split=${split}&offset=${offset}&length=${length}`;
+  for (let attempt = 0; attempt < 6; attempt++) {
+    try {
+      const res = await fetch(url);
+      if (res.status === 429) { await sleep(2500 * (attempt + 1)); continue; }
+      if (!res.ok) throw new Error(`HTTP ${res.status}`);
+      const json = await res.json();
+      return { rows: json.rows?.map(r => r.row) || [], total: json.num_rows_total ?? null };
+    } catch (err) {
+      if (attempt === 5) {
+        console.warn(`  page offset=${offset} failed after retries (${err.message}) — skipping`);
+        return { rows: [], total: null, failed: true };
+      }
+      await sleep(1500 * (attempt + 1));
+    }
+  }
+  return { rows: [], total: null, failed: true };
+}
+
+// --- candidate filters -------------------------------------------------------
+
+function cleanCandidate(text) {
+  if (typeof text !== "string") return null;
+  const t = text.replace(/\s+/g, " ").trim();
+  if (t.length < 15 || t.length > 400) return null;
+  // Cloze / MCQ scaffolding reads nothing like a typed ask
+  if (/\(\s*\)/.test(t) || /_{3,}/.test(t)) return null;
+  if (/\b[A-D]\)\s/.test(t) || /which of the following/i.test(t)) return null;
+  // NLI/benchmark scaffolding (gpt4_dataset carries FLAN-style tasks)
+  if (/\b(premise|hypothesis)\b/i.test(t) && /\b(entail|inference|contradict)\b/i.test(t)) return null;
+  if (/natural language inference|answer with (yes|no)|options: -/i.test(t)) return null;
+  // Anonymization tokens (gpt4_dataset) and template junk
+  if (/NAME_\d|\{\{.*\}\}/.test(t)) return null;
+  // Mostly-English only: embeddings + our traffic are English
+  const nonAscii = (t.match(/[^\x20-\x7E]/g) || []).length;
+  if (nonAscii > t.length * 0.1) return null;
+  return t;
+}
+
+// Quiz-bowl clue style: leading demonstrative riddles and pronunciation
+// guides. "Hard" trivia measures obscure recall, not reasoning depth — it
+// would anchor the frontier class to the wrong concept entirely.
+function isQuizbowlStyle(t) {
+  return /^(this |one |a \d{4} (book|work|paper)|fictional |in one (work|section))/i.test(t) ||
+    /\([A-Za-z]+-[A-Z][A-Za-z-]+\)/.test(t);
+}
+
+const TECH_DOMAIN_RE = /computer science|technology|engineering|mathematic|science/i;
+// SuperGLUE tasks are benchmark scaffolding, not natural asks.
+const DATASET_EXCLUDE_RE = /superglue|cloze|entailment/i;
+
+// --- diversity selection (greedy k-center / max-min) -------------------------
+
+async function embed(text) {
+  const res = await fetch(OLLAMA_EMBED, {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify({ model: EMBED_MODEL, prompt: text }),
+  });
+  if (!res.ok) throw new Error(`embed HTTP ${res.status}`);
+  const json = await res.json();
+  return json.embedding;
+}
+
+function cosine(a, b) {
+  let dot = 0, na = 0, nb = 0;
+  for (let i = 0; i < a.length; i++) { dot += a[i] * b[i]; na += a[i] * a[i]; nb += b[i] * b[i]; }
+  const d = Math.sqrt(na) * Math.sqrt(nb);
+  return d > 0 ? dot / d : 0;
+}
+
+async function diversitySelect(candidates, target, label) {
+  if (candidates.length <= target) return candidates.map(c => c.text);
+  process.stdout.write(`  embedding ${candidates.length} ${label} candidates...`);
+  const embedded = [];
+  for (const c of candidates) {
+    try {
+      embedded.push({ ...c, vec: await embed(c.text) });
+    } catch { /* skip failures */ }
+    if (embedded.length % 200 === 0) process.stdout.write(".");
+  }
+  console.log(` ${embedded.length} embedded`);
+
+  // Greedy max-min: always keep the candidate farthest from the selected set.
+  const selected = [embedded[0]];
+  const minSim = new Array(embedded.length).fill(-1).map((_, i) => cosine(embedded[i].vec, embedded[0].vec));
+  while (selected.length < target) {
+    let bestIdx = -1, bestVal = Infinity;
+    for (let i = 0; i < embedded.length; i++) {
+      if (minSim[i] === Infinity) continue; // already selected
+      if (minSim[i] < bestVal) { bestVal = minSim[i]; bestIdx = i; }
+    }
+    if (bestIdx === -1) break;
+    const pick = embedded[bestIdx];
+    selected.push(pick);
+    minSim[bestIdx] = Infinity;
+    for (let i = 0; i < embedded.length; i++) {
+      if (minSim[i] === Infinity) continue;
+      const s = cosine(embedded[i].vec, pick.vec);
+      if (s > minSim[i]) minSim[i] = s;
+    }
+  }
+  return selected.map(s => s.text);
+}
+
+// --- main --------------------------------------------------------------------
+
+async function main() {
+  const dryRun = process.argv.includes("--dry-run");
+  const pools = { substantive: [], heavyweight: [], frontier: [] };
+  const provenance = {};
+
+  // RouterArena: full split, paginate everything (~8.4k rows)
+  console.log("Fetching RouterArena (full split)...");
+  let raCount = 0;
+  let raTotal = null;
+  for (let offset = 0; raTotal === null || offset < raTotal; offset += 100) {
+    const { rows, total } = await fetchRows("RouteWorks/RouterArena", "default", "full", offset, 100);
+    if (total !== null) raTotal = total;
+    if (rows.length === 0 && raTotal === null) break;
+    raCount += rows.length;
+    for (const row of rows) {
+      if (row.Context && String(row.Context).trim()) continue; // passage-dependent
+      if (Array.isArray(row.Options) && row.Options.length > 0) continue; // MCQ
+      if (DATASET_EXCLUDE_RE.test(String(row["Dataset name"] || ""))) continue;
+      const text = cleanCandidate(row.Question);
+      if (!text || isQuizbowlStyle(text)) continue;
+      const technical = TECH_DOMAIN_RE.test(String(row.Domain || ""));
+      const diff = String(row.Difficulty || "").toLowerCase();
+      // hard→frontier and medium→heavyweight only from technical domains:
+      // non-technical "hard" is dominated by obscure-recall trivia, which
+      // anchors the expensive tiers to the wrong concept.
+      let cls = null;
+      if (diff === "easy") cls = "substantive";
+      else if (diff === "medium" && technical) cls = "heavyweight";
+      else if (diff === "hard" && technical) cls = "frontier";
+      if (!cls) continue;
+      pools[cls].push({ text, source: "RouterArena", label: `${row.Difficulty}/${row["Dataset name"]}`, technical });
+    }
+    if (offset % 2000 === 0 && offset > 0) console.log(`  ...${raCount}/${raTotal ?? "?"} rows scanned`);
+    await sleep(250); // stay friendly to the API
+  }
+  console.log(`RouterArena: ${raCount} rows → pools: sub=${pools.substantive.length} heavy=${pools.heavyweight.length} frontier=${pools.frontier.length}`);
+
+  // gpt4_dataset: sample pages spread across the 109k-row train split
+  console.log("Sampling routellm/gpt4_dataset...");
+  const PAGES = 60;
+  const SPAN = 109000;
+  for (let p = 0; p < PAGES; p++) {
+    const offset = Math.floor((p * SPAN) / PAGES);
+    const { rows } = await fetchRows("routellm/gpt4_dataset", "default", "train", offset, 100);
+    for (const row of rows) {
+      const text = cleanCandidate(row.prompt);
+      if (!text) continue;
+      const score = Number(row.mixtral_score);
+      const technical = /code|function|script|debug|compile|regex|sql|api|server|deploy|git|test/i.test(text);
+      if (score === 5) {
+        pools.substantive.push({ text, source: "gpt4_dataset", label: `mixtral_score=${score}`, technical });
+      } else if (score <= 2 && score >= 1 && text.length >= 60) {
+        // ≥60 chars: low judge scores on one-liners are label noise
+        // ("What is the capital of Massachusetts?" scored 1), not difficulty.
+        pools.heavyweight.push({ text, source: "gpt4_dataset", label: `mixtral_score=${score}`, technical });
+      }
+    }
+    await sleep(250);
+  }
+  console.log(`After gpt4_dataset: sub=${pools.substantive.length} heavy=${pools.heavyweight.length} frontier=${pools.frontier.length}`);
+
+  // Bias ~60% technical where the pool allows: sort technical-first, then
+  // interleave so diversity selection sees both kinds.
+  for (const cls of Object.keys(pools)) {
+    const tech = pools[cls].filter(c => c.technical);
+    const gen = pools[cls].filter(c => !c.technical);
+    const mixed = [];
+    let ti = 0, gi = 0;
+    while (ti < tech.length || gi < gen.length) {
+      // 3 technical : 2 general cadence ≈ 60/40
+      for (let k = 0; k < 3 && ti < tech.length; k++) mixed.push(tech[ti++]);
+      for (let k = 0; k < 2 && gi < gen.length; k++) mixed.push(gen[gi++]);
+    }
+    pools[cls] = mixed;
+  }
+
+  if (dryRun) {
+    for (const cls of Object.keys(pools)) {
+      console.log(`\n=== ${cls} (${pools[cls].length} candidates), first 5:`);
+      for (const c of pools[cls].slice(0, 5)) console.log(`  [${c.source}/${c.label}] ${c.text.slice(0, 100)}`);
+    }
+    return;
+  }
+
+  // Diversity-select down to targets
+  const selected = {};
+  for (const [cls, target] of Object.entries(TARGETS)) {
+    selected[cls] = await diversitySelect(pools[cls], target, cls);
+    for (const text of selected[cls]) {
+      const cand = pools[cls].find(c => c.text === text);
+      provenance[text] = { class: cls, source: cand?.source, label: cand?.label };
+    }
+  }
+
+  // Merge with the bundled hand anchors (they always survive)
+  const hand = JSON.parse(fs.readFileSync(CONFIG_ANCHORS, "utf8"));
+  const out = {
+    _comment: `Mined ${new Date().toISOString().slice(0, 10)} by scripts/mine-difficulty-anchors.js from RouteWorks/RouterArena (empirical difficulty bands) and routellm/gpt4_dataset (mixtral_score). Hand anchors from config/difficulty-anchors.json retained. Local only (data/ is gitignored and excluded from the npm tarball).`,
+    trivial: hand.trivial,
+    substantive: [...hand.substantive, ...selected.substantive],
+    heavyweight: [...hand.heavyweight, ...selected.heavyweight],
+    frontier: [...HAND_FRONTIER, ...selected.frontier],
+  };
+  for (const t of HAND_FRONTIER) provenance[t] = { class: "frontier", source: "hand", label: "engineering-frontier" };
+
+  fs.writeFileSync(OUT_ANCHORS, JSON.stringify(out, null, 2));
+  fs.writeFileSync(OUT_PROVENANCE, JSON.stringify(provenance, null, 2));
+  console.log(`\nWrote ${OUT_ANCHORS}:`);
+  for (const cls of ["trivial", "substantive", "heavyweight", "frontier"]) {
+    console.log(`  ${cls}: ${out[cls].length} anchors`);
+  }
+}
+
+main().catch(err => { console.error(err); process.exit(1); });

package/scripts/validate-difficulty-classifier.js

@@ -0,0 +1,144 @@
+#!/usr/bin/env node
+/**
+ * Validate the LLM difficulty classifier against the eval set.
+ *
+ * Runs data/difficulty-eval.jsonl through src/routing/difficulty-classifier.js
+ * (the SIMPLE tier model — currently minimax-m2.5:cloud via ollama).
+ * Reports overall + per-tier accuracy, confusion matrix, and lists the
+ * misclassifications so we can eyeball whether classifier or label is wrong.
+ *
+ * Bar to ship: ≥85% overall, zero MEDIUM→REASONING false positives.
+ *
+ * Usage: node scripts/validate-difficulty-classifier.js
+ */
+
+const fs = require("fs");
+const path = require("path");
+const { classifyDifficulty, _clearCacheForTests } = require("../src/routing/difficulty-classifier");
+
+const EVAL_FILE = path.join(__dirname, "../data/difficulty-eval.jsonl");
+const TIERS = ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"];
+
+async function main() {
+  // --file=data/xyz.jsonl runs an alternate eval set (e.g. the follow-up
+  // slice with conversation context). Results land next to the input file.
+  const fileArg = process.argv.find((a) => a.startsWith("--file="));
+  const evalFile = fileArg ? path.resolve(fileArg.slice(7)) : EVAL_FILE;
+  const resultsFile = evalFile.replace(/\.jsonl$/, "-results.jsonl");
+
+  _clearCacheForTests();
+  const lines = fs.readFileSync(evalFile, "utf8").split("\n").filter(Boolean);
+  const rows = lines.map(l => JSON.parse(l));
+  console.log(`Loaded ${rows.length} eval rows from ${path.basename(evalFile)}`);
+
+  // Persist incrementally so a crash mid-run preserves partial data.
+  const resultsFd = fs.openSync(resultsFile, "w");
+  const results = [];
+  const t0 = Date.now();
+  let done = 0;
+  for (const row of rows) {
+    // context is threaded the same way production does (router window loop
+    // → scoreIntent → classifyDifficulty); absent for standalone prompts.
+    const r = await classifyDifficulty(row.text, { context: row.context });
+    const record = {
+      ...row,
+      predicted: r?.tier ?? null,
+      confidence: r?.confidence ?? null,
+    };
+    results.push(record);
+    fs.writeSync(resultsFd, JSON.stringify(record) + "\n");
+    done++;
+    if (done % 25 === 0) {
+      const elapsed = (Date.now() - t0) / 1000;
+      process.stdout.write(`  ${done}/${rows.length} (${elapsed.toFixed(0)}s, avg ${(elapsed / done * 1000).toFixed(0)}ms/prompt)\n`);
+    }
+  }
+  fs.closeSync(resultsFd);
+  console.log(`\nDone in ${((Date.now() - t0) / 1000).toFixed(0)}s (results saved to ${resultsFile})`);
+
+  // Overall + per-tier accuracy
+  const perTier = {};
+  const confusion = {};
+  for (const t of TIERS) {
+    perTier[t] = { total: 0, correct: 0 };
+    confusion[t] = { SIMPLE: 0, MEDIUM: 0, COMPLEX: 0, REASONING: 0, null: 0 };
+  }
+  let overall = 0;
+  let classified = 0;
+  let skipped = 0;
+  for (const r of results) {
+    if (r.predicted === null) { skipped++; continue; }
+    classified++;
+    perTier[r.tier].total++;
+    confusion[r.tier][r.predicted] = (confusion[r.tier][r.predicted] || 0) + 1;
+    if (r.predicted === r.tier) { overall++; perTier[r.tier].correct++; }
+  }
+
+  console.log(`\n=== Accuracy ===`);
+  console.log(`Overall: ${overall}/${classified} (${(overall / classified * 100).toFixed(1)}%)`);
+  console.log(`Skipped (short text / classifier disabled): ${skipped}`);
+  console.log(`\nPer-tier:`);
+  for (const t of TIERS) {
+    const p = perTier[t];
+    if (p.total === 0) continue;
+    console.log(`  ${t}: ${p.correct}/${p.total} (${(p.correct / p.total * 100).toFixed(1)}%)`);
+  }
+
+  // Per-source accuracy: hand labels are trusted; benchmark labels have known
+  // difficulty-vs-tier bias (RouterArena's "easy" band ≠ MEDIUM tier; gpt4's
+  // mixtral_score ≠ tier). Report separately for an honest signal.
+  console.log(`\n=== Per-source ===`);
+  const sources = {};
+  for (const r of results) {
+    if (r.predicted === null) continue;
+    const s = r.source || 'unknown';
+    if (!sources[s]) sources[s] = { total: 0, correct: 0 };
+    sources[s].total++;
+    if (r.predicted === r.tier) sources[s].correct++;
+  }
+  for (const [s, v] of Object.entries(sources)) {
+    console.log(`  ${s}: ${v.correct}/${v.total} (${(v.correct / v.total * 100).toFixed(1)}%)`);
+  }
+
+  console.log(`\n=== Confusion matrix (rows = true, cols = predicted) ===`);
+  console.log(`         ${TIERS.map(t => t.padStart(9)).join("")}`);
+  for (const t of TIERS) {
+    const row = TIERS.map(pred => String(confusion[t][pred] || 0).padStart(9)).join("");
+    console.log(`${t.padStart(9)}${row}`);
+  }
+
+  // Critical failure mode: MEDIUM→REASONING (over-routing to expensive tier)
+  const overRouted = confusion.MEDIUM?.REASONING || 0;
+  const simpleToReasoning = confusion.SIMPLE?.REASONING || 0;
+  console.log(`\n=== Critical false positives ===`);
+  console.log(`MEDIUM→REASONING: ${overRouted} (must be 0 to ship)`);
+  console.log(`SIMPLE→REASONING: ${simpleToReasoning}`);
+  console.log(`MEDIUM→COMPLEX: ${confusion.MEDIUM?.COMPLEX || 0}`);
+
+  // Confidence histogram — validates whether the confidence gate in
+  // _reconcile carries signal. A degenerate distribution (everything ≥0.9)
+  // means confidence is decorative and the band cap is the only real guard.
+  const buckets = { "<0.6": 0, "0.6-0.8": 0, "0.8-0.9": 0, "0.9-1.0": 0 };
+  for (const r of results) {
+    if (r.confidence == null) continue;
+    if (r.confidence < 0.6) buckets["<0.6"]++;
+    else if (r.confidence < 0.8) buckets["0.6-0.8"]++;
+    else if (r.confidence < 0.9) buckets["0.8-0.9"]++;
+    else buckets["0.9-1.0"]++;
+  }
+  console.log(`\n=== Confidence histogram ===`);
+  for (const [b, n] of Object.entries(buckets)) console.log(`  ${b}: ${n}`);
+
+  // List misclassifications (cap at 20 per tier)
+  console.log(`\n=== Misclassifications (up to 5 per tier) ===`);
+  for (const t of TIERS) {
+    const errors = results.filter(r => r.tier === t && r.predicted && r.predicted !== t).slice(0, 5);
+    if (errors.length === 0) continue;
+    console.log(`\n${t}:`);
+    for (const e of errors) {
+      console.log(`  → ${e.predicted} (conf ${e.confidence?.toFixed(2)}): ${e.text.slice(0, 90)}`);
+    }
+  }
+}
+
+main().catch(err => { console.error(err); process.exit(1); });

package/scripts/validate-intent-anchors.js

@@ -0,0 +1,186 @@
+#!/usr/bin/env node
+/**
+ * Validate new anchors against old (config/ vs data/) on a held-out eval set.
+ *
+ * Mines ~40 real prompts from logs + hand-picked coding-agent asks, assigns
+ * target tiers, scores with both anchor sets, reports accuracy and tier-shift
+ * delta. Use to tune FRONTIER_MIN_SIM and confirm the mined anchors beat the
+ * 13-anchor baseline without false-positive REASONING routing.
+ *
+ * Usage: node scripts/validate-intent-anchors.js [--baseline|--new]
+ *   --baseline: score eval set with config/difficulty-anchors.json only
+ *   --new:      score eval set with data/difficulty-anchors.json only
+ *   (no flag):  compare both and report delta
+ */
+
+const fs = require("fs");
+const path = require("path");
+
+const CONFIG_ANCHORS = path.join(__dirname, "../config/difficulty-anchors.json");
+const DATA_ANCHORS = path.join(__dirname, "../data/difficulty-anchors.json");
+
+// Tier bands (from model-tiers.js defaults — verify if calibrated)
+const TIER_BANDS = {
+  SIMPLE: [0, 19],
+  MEDIUM: [20, 50],
+  COMPLEX: [51, 75],
+  REASONING: [76, 100],
+};
+
+function tierFromScore(score) {
+  for (const [tier, [lo, hi]] of Object.entries(TIER_BANDS)) {
+    if (score >= lo && score <= hi) return tier;
+  }
+  return "UNKNOWN";
+}
+
+// Eval set: real prompts from logs + coding-agent asks, with ground-truth tier.
+// Target tier is what config B SHOULD route (local/GLM/Claude) given intent.
+const EVAL_SET = [
+  // SIMPLE (local/ollama)
+  { text: "hi", target: "SIMPLE" },
+  { text: "ok thanks", target: "SIMPLE" },
+  { text: "yes continue", target: "SIMPLE" },
+  { text: "what time is it", target: "SIMPLE" },
+  { text: "got it", target: "SIMPLE" },
+  // MEDIUM (ollama)
+  { text: "Run the unit tests and summarize failures", target: "MEDIUM" },
+  { text: "What does this function do", target: "MEDIUM" },
+  { text: "Show me the git diff", target: "MEDIUM" },
+  { text: "List the exports from this file", target: "MEDIUM" },
+  { text: "Fix the linter warnings", target: "MEDIUM" },
+  { text: "Add error handling to this try block", target: "MEDIUM" },
+  { text: "Write a test for the login function", target: "MEDIUM" },
+  { text: "Explain this regex pattern", target: "MEDIUM" },
+  { text: "Debug this null reference error", target: "MEDIUM" },
+  { text: "Refactor this into smaller functions", target: "MEDIUM" },
+  // COMPLEX (GLM)
+  { text: "Do an architecture review of the orchestrator", target: "COMPLEX" },
+  { text: "Review this retry helper for bugs", target: "COMPLEX" },
+  { text: "Refactor the entire ingestion pipeline and give me the plan", target: "COMPLEX" },
+  { text: "Design a horizontally scalable architecture for the router", target: "COMPLEX" },
+  { text: "Code review the PR #84 routing hardening changes", target: "COMPLEX" },
+  { text: "Analyze every module in src/ for circular dependencies", target: "COMPLEX" },
+  { text: "Debug this complex race condition in the connection pool", target: "COMPLEX" },
+  { text: "Plan a zero-downtime migration to the new schema", target: "COMPLEX" },
+  { text: "Implement a distributed rate limiter with Redis", target: "COMPLEX" },
+  { text: "Design the caching strategy for this API gateway", target: "COMPLEX" },
+  // REASONING (Claude)
+  { text: "Prove the correctness of this lock-free queue implementation", target: "REASONING" },
+  { text: "Security audit the authentication middleware", target: "REASONING" },
+  { text: "Derive the optimal cache eviction policy and prove its competitive ratio", target: "REASONING" },
+  { text: "Formal verification of the state machine — prove it never deadlocks", target: "REASONING" },
+  { text: "From first principles, design a Byzantine-fault-tolerant consensus protocol variant", target: "REASONING" },
+  { text: "Think deeply about why this floating-point summation loses precision at scale", target: "REASONING" },
+  { text: "Prove this rate limiter is fair under concurrent refill or construct the starvation schedule", target: "REASONING" },
+  { text: "Reason through the exact memory ordering constraints this hashmap needs on ARM", target: "REASONING" },
+  { text: "Given these heap dumps, reason to the retention path causing the leak", target: "REASONING" },
+  { text: "Ultrathink: analyze whether this migration can run online without violating invariants", target: "REASONING" },
+];
+
+async function scoreWithAnchors(anchorsPath) {
+  const { scoreIntent, buildCentroids, CLASS_VALUES } = require("../src/routing/intent-score");
+  const { getKnnRouter } = require("../src/routing/knn-router");
+  const router = getKnnRouter();
+  const embedFn = (t) => router.embed(t);
+
+  let anchors = JSON.parse(fs.readFileSync(anchorsPath, "utf8"));
+  // Old 3-class anchors lack frontier — stub it with a heavyweight anchor so
+  // buildCentroids gets 4 centroids (it requires centroid count === CLASS_VALUES
+  // count). The stub's embedding pulls frontier sim down, keeping the 3-class
+  // baseline behavior (frontier excluded from blend via FRONTIER_MIN_SIM floor).
+  if (!anchors.frontier && CLASS_VALUES.frontier) {
+    anchors = { ...anchors, frontier: [anchors.heavyweight[0]] };
+  }
+  const centroids = await buildCentroids(anchors, embedFn);
+  if (!centroids) throw new Error(`Failed to build centroids from ${anchorsPath}`);
+
+  const results = [];
+  for (const item of EVAL_SET) {
+    const payload = { messages: [{ role: "user", content: item.text }] };
+    const r = await scoreIntent(payload, { embedFn, centroids });
+    const predicted = tierFromScore(r.score);
+    const correct = predicted === item.target;
+    results.push({ ...item, score: r.score, class: r.class, predicted, correct });
+  }
+  return results;
+}
+
+function reportResults(label, results) {
+  const correct = results.filter(r => r.correct).length;
+  const acc = (correct / results.length * 100).toFixed(1);
+  console.log(`\n=== ${label} ===`);
+  console.log(`Accuracy: ${correct}/${results.length} (${acc}%)`);
+
+  const byTarget = {};
+  for (const r of results) {
+    if (!byTarget[r.target]) byTarget[r.target] = { total: 0, correct: 0 };
+    byTarget[r.target].total++;
+    if (r.correct) byTarget[r.target].correct++;
+  }
+  console.log("\nPer-tier accuracy:");
+  for (const [tier, stats] of Object.entries(byTarget)) {
+    const pct = (stats.correct / stats.total * 100).toFixed(1);
+    console.log(`  ${tier}: ${stats.correct}/${stats.total} (${pct}%)`);
+  }
+
+  const errors = results.filter(r => !r.correct);
+  if (errors.length > 0) {
+    console.log(`\nMisclassified (${errors.length}):`);
+    for (const e of errors) {
+      console.log(`  [${e.target}→${e.predicted}, score=${e.score}] ${e.text.slice(0, 60)}`);
+    }
+  }
+}
+
+function compareResults(baseline, neu) {
+  console.log("\n=== Δ (new vs baseline) ===");
+  const baseCorrect = baseline.filter(r => r.correct).length;
+  const neuCorrect = neu.filter(r => r.correct).length;
+  const delta = neuCorrect - baseCorrect;
+  const sign = delta > 0 ? "+" : "";
+  console.log(`Accuracy: ${sign}${delta} (baseline ${baseCorrect}/${baseline.length}, new ${neuCorrect}/${neu.length})`);
+
+  const improved = [];
+  const regressed = [];
+  for (let i = 0; i < baseline.length; i++) {
+    if (!baseline[i].correct && neu[i].correct) improved.push(neu[i]);
+    if (baseline[i].correct && !neu[i].correct) regressed.push(neu[i]);
+  }
+  if (improved.length > 0) {
+    console.log(`\nFixed by new anchors (${improved.length}):`);
+    for (const r of improved) console.log(`  [${r.target}, score=${r.score}] ${r.text.slice(0, 60)}`);
+  }
+  if (regressed.length > 0) {
+    console.log(`\nBroken by new anchors (${regressed.length}):`);
+    for (const r of regressed) console.log(`  [${r.target}→${r.predicted}, score=${r.score}] ${r.text.slice(0, 60)}`);
+  }
+}
+
+async function main() {
+  const mode = process.argv[2];
+  if (mode === "--baseline") {
+    const results = await scoreWithAnchors(CONFIG_ANCHORS);
+    reportResults("Baseline (config/difficulty-anchors.json)", results);
+  } else if (mode === "--new") {
+    if (!fs.existsSync(DATA_ANCHORS)) {
+      console.error(`${DATA_ANCHORS} does not exist — run scripts/mine-difficulty-anchors.js first`);
+      process.exit(1);
+    }
+    const results = await scoreWithAnchors(DATA_ANCHORS);
+    reportResults("New (data/difficulty-anchors.json)", results);
+  } else {
+    if (!fs.existsSync(DATA_ANCHORS)) {
+      console.error(`${DATA_ANCHORS} does not exist — run scripts/mine-difficulty-anchors.js first`);
+      process.exit(1);
+    }
+    console.log("Scoring eval set with both anchor sets...");
+    const baseline = await scoreWithAnchors(CONFIG_ANCHORS);
+    const neu = await scoreWithAnchors(DATA_ANCHORS);
+    reportResults("Baseline (config/)", baseline);
+    reportResults("New (data/)", neu);
+    compareResults(baseline, neu);
+  }
+}
+
+main().catch(err => { console.error(err); process.exit(1); });

package/src/api/providers-handler.js

@@ -401,7 +401,6 @@ router.get("/config", (req, res) => {
       fallback_provider: config.modelProvider?.fallbackProvider || null,
       fallback_enabled: config.modelProvider?.fallbackEnabled || false,
       tier_routing_enabled: config.modelTiers?.enabled || false,
-      tool_execution_mode: config.toolExecutionMode || "server",
       configured_providers: providers.map(p => p.name),
       memory_enabled: config.memory?.enabled || false,
       smart_tool_selection: config.smartToolSelection?.enabled || false