lynkr 9.14.17 → 9.14.19

package/README.md

@@ -22,6 +22,8 @@
 
 > Numbers from the bundled benchmark against LiteLLM on identical free local backends — run it yourself: `node benchmark-tier-routing.js`. It doubles as a 19-scenario routing regression harness (currently 12/12 correctness checks), and `MODE=routing` runs a routing-only head-to-head that judges **both** proxies on the same acceptable-tier sets — including LiteLLM's Auto Router v2. [How it works →](docs/benchmarking.md)
 
+> **Agentic benchmark (Terminal-Bench core, 80 tasks, Oct 2026):** Lynkr **42/80 for $1.54** vs openrouter/auto **44/80 for $4.72** on the same VM and evening — tied on accuracy within single-run noise (Lynkr's three runs: 39, 39, 42) at ~30% of the cost; vLLM Semantic Router on the same models/hosts: 25/80 (34 agent timeouts from unbounded upstream stalls and per-turn CPU classification). [Full comparison, fairness notes and what it changed in Lynkr →](docs/terminal-bench-router-comparison.md)
+
 > **Third-party benchmark:** on [RouterArena](https://github.com/RouteWorks/RouterArena) (ICLR 2026, 8,400 queries) Lynkr's routing scores **67.65 arena / 68.41% accuracy at $0.29 per 1K queries with 92.38 robustness** — above GPT-5's built-in router and NotDiamond at a fraction of their cost. [Methodology & caveats →](docs/routerarena-benchmark.md)
 
 ---

package/bin/cli.js

@@ -16,6 +16,8 @@ const SUBCOMMANDS = {
   restart:       path.join(__dirname, "lynkr-restart.js"),
   run:           path.join(__dirname, "run.js"),
   connect:       path.join(__dirname, "lynkr-connect-orcarouter.js"),
+  route:         path.join(__dirname, "lynkr-route.js"),
+  audit:         path.join(__dirname, "lynkr-audit.js"),
 };
 
 const sub = process.argv[2];

package/bin/lynkr-audit.js

@@ -0,0 +1,58 @@
+#!/usr/bin/env node
+/**
+ * lynkr audit <session_id|--last N>   Per-session routing ledger from telemetry.
+ *
+ * Prints, per turn: time, tier, model, decision, effort, latency, status,
+ * cost, judge verdict, previous-turn outcome. Totals at the bottom.
+ * Reads <cwd>/.lynkr/telemetry.db (same path the server writes).
+ */
+'use strict';
+
+const path = require('path');
+const fs = require('fs');
+
+function main() {
+  const argv = process.argv.slice(2);
+  const json = argv.includes('--json');
+  const lastIdx = argv.indexOf('--last');
+  const lastVal = lastIdx >= 0 ? argv[lastIdx + 1] : null;
+  const sessionId = argv.find((a) => !a.startsWith('--') && a !== lastVal);
+  const dbPath = path.resolve(process.cwd(), '.lynkr', 'telemetry.db');
+  if (!fs.existsSync(dbPath)) { console.error(`no telemetry db at ${dbPath}`); process.exit(2); }
+  const Database = require('better-sqlite3');
+  const db = new Database(dbPath, { readonly: true });
+
+  if (!sessionId) {
+    const n = lastIdx >= 0 ? Number(argv[lastIdx + 1]) || 10 : 10;
+    const rows = db.prepare(`SELECT session_id, COUNT(*) turns, MIN(timestamp) t0, MAX(timestamp) t1, SUM(cost_usd) cost,
+        SUM(status_code != 200) errors, GROUP_CONCAT(DISTINCT tier) tiers
+      FROM routing_telemetry WHERE session_id IS NOT NULL GROUP BY session_id ORDER BY t1 DESC LIMIT ?`).all(n);
+    if (json) { console.log(JSON.stringify(rows, null, 2)); return; }
+    console.log(`\nlast ${rows.length} sessions`);
+    for (const r of rows) console.log(`  ${r.session_id}  turns=${String(r.turns).padStart(3)}  tiers=${r.tiers}  cost=$${(r.cost || 0).toFixed(4)}  errors=${r.errors}  ${new Date(r.t1).toISOString()}`);
+    console.log(`\nlynkr audit <session_id> for the per-turn ledger\n`);
+    return;
+  }
+  const rows = db.prepare(`SELECT id, timestamp, tier, model, provider, routing_method, decision_name, engine_tier, engine_mode, effort,
+      latency_ms, status_code, error_type, cost_usd, input_tokens, output_tokens, cache_read_tokens,
+      jev_verdict, jev_confidence, prev_turn_outcome, prev_turn_attributable, escalation_source
+    FROM routing_telemetry WHERE session_id = ? ORDER BY id`).all(sessionId);
+  if (!rows.length) { console.error(`no rows for session ${sessionId}`); process.exit(1); }
+  if (json) { console.log(JSON.stringify(rows, null, 2)); return; }
+  const pad = (s, n) => String(s ?? '-').padEnd(n);
+  console.log(`\nsession ${sessionId}  (${rows.length} turns)\n`);
+  console.log(`  ${pad('#', 3)} ${pad('time', 8)} ${pad('tier', 9)} ${pad('model', 30)} ${pad('decision', 22)} ${pad('eff', 6)} ${pad('ms', 7)} ${pad('st', 4)} ${pad('$', 9)} ${pad('judge', 14)} ${pad('prev-outcome', 16)}`);
+  let cost = 0, inTok = 0, outTok = 0, cached = 0, errs = 0;
+  rows.forEach((r, i) => {
+    cost += r.cost_usd || 0; inTok += r.input_tokens || 0; outTok += r.output_tokens || 0; cached += r.cache_read_tokens || 0; if (r.status_code !== 200) errs++;
+    const t = new Date(r.timestamp).toISOString().slice(11, 19);
+    const dec = r.decision_name ? `${r.decision_name}${r.engine_tier && r.engine_tier !== r.tier ? '→' + r.engine_tier : ''}` : '-';
+    const judge = r.jev_verdict ? `${r.jev_verdict}@${Number(r.jev_confidence || 0).toFixed(2)}` : '-';
+    const po = r.prev_turn_outcome ? `${r.prev_turn_outcome}${r.prev_turn_attributable ? '' : '(env)'}` : '-';
+    console.log(`  ${pad(i + 1, 3)} ${pad(t, 8)} ${pad(r.tier, 9)} ${pad(String(r.model || '').split('/').pop().slice(0, 30), 30)} ${pad(dec.slice(0, 22), 22)} ${pad(r.effort, 6)} ${pad(r.latency_ms, 7)} ${pad(r.status_code, 4)} ${pad((r.cost_usd || 0).toFixed(5), 9)} ${pad(judge, 14)} ${pad(po, 16)}${r.escalation_source ? '  esc:' + r.escalation_source : ''}`);
+  });
+  console.log(`\n  cost $${cost.toFixed(4)}   in ${inTok} (cached ${cached}, ${inTok ? Math.round(100 * cached / (inTok + cached)) : 0}%)   out ${outTok}   errors ${errs}\n`);
+}
+
+if (require.main === module || process.env._LYNKR_SUBCMD === 'audit') main();
+module.exports = { main };

package/bin/lynkr-route.js

@@ -0,0 +1,76 @@
+#!/usr/bin/env node
+/**
+ * lynkr route --preview <request.json|->   Explain how a request would route.
+ *
+ * Runs the full routing decision (signals, legacy chain, decision engine)
+ * without calling any provider or writing telemetry. Prints every signal's
+ * value, the legacy tier, the matched decision and its tier/effort, and
+ * whether observe/enforce mode would change the served tier.
+ *
+ * Options: --json   machine-readable output
+ *          --config <path>  use an alternative config/routing.json
+ */
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+function main() {
+  const argv = process.argv.slice(2);
+  const json = argv.includes('--json');
+  const ci = argv.indexOf('--config');
+  if (ci >= 0) process.env.LYNKR_ROUTING_CONFIG = path.resolve(argv[ci + 1]);
+  const pi = argv.indexOf('--preview');
+  const file = pi >= 0 ? argv[pi + 1] : argv.find((a) => !a.startsWith('--'));
+  if (!file) {
+    console.error('usage: lynkr route --preview <request.json|-> [--json] [--config routing.json]');
+    process.exit(2);
+  }
+  const raw = file === '-' ? fs.readFileSync(0, 'utf8') : fs.readFileSync(file, 'utf8');
+  let payload;
+  try { payload = JSON.parse(raw); } catch (e) { console.error(`not JSON: ${e.message}`); process.exit(2); }
+  if (typeof payload === 'string' || (payload && !payload.messages)) {
+    // Bare text → a single user message.
+    const text = typeof payload === 'string' ? payload : raw;
+    payload = { messages: [{ role: 'user', content: text }], tools: [] };
+  }
+
+  // Load the operator .env like the server does; silence logs.
+  const envPath = fs.existsSync(path.join(process.cwd(), '.env')) ? path.join(process.cwd(), '.env') : path.join(require('os').homedir(), '.env');
+  require('dotenv').config({ path: envPath });
+  process.env.LOG_LEVEL = 'silent'; process.env.LOG_FILE_ENABLED = 'false';
+
+  const { determineProviderSmart } = require('../src/routing');
+  const rc = require('../src/routing/routing-config');
+
+  (async () => {
+    const d = await determineProviderSmart(payload, {});
+    const e = d.engine || null;
+    const out = {
+      config: { path: rc.configPath(), mode: rc.mode() },
+      legacy: { tier: d.tier, provider: d.provider, model: d.model, method: d.method, score: d.score ?? null, anchorScore: d.analysis?.anchorScore ?? null, escalations: d.escalations || [] },
+      judge: d.analysis?.jev ? { tier: d.analysis.jev.tier, confidence: d.analysis.jev.confidence, probabilities: d.analysis.jev.probabilities } : null,
+      shortfall: d.shortfall ? { selected: d.shortfall.selected, lift: d.shortfall.lift, req: d.shortfall.req } : null,
+      engine: e ? { decision: e.decision, tier: e.tier, effort: e.effort, hosts: e.hosts, mode: e.mode, agreesWithLegacy: e.agreesWithLegacy, considered: e.trace?.considered, signals: e.signals } : null,
+      served: e && e.mode === 'enforce' && e.tier ? e.tier : d.tier,
+    };
+    if (json) { console.log(JSON.stringify(out, null, 2)); return; }
+    const pad = (s, n) => String(s ?? '').padEnd(n);
+    console.log(`\nconfig   ${out.config.path}  (mode: ${out.config.mode})`);
+    console.log(`\nSIGNALS`);
+    for (const [k, v] of Object.entries(out.engine?.signals || {})) {
+      const val = v.band || v.tier || (v.value && typeof v.value === 'object' ? JSON.stringify(v.value) : v.value);
+      console.log(`  ${pad(k, 12)} ${v.matched ? 'matched  ' : '-        '} ${pad(val, 22)} ${v.confidence != null ? 'conf ' + Number(v.confidence).toFixed(2) : ''}`);
+    }
+    console.log(`\nLEGACY   tier=${out.legacy.tier}  model=${out.legacy.model}  method=${out.legacy.method}  anchor=${out.legacy.anchorScore}` + (out.judge ? `  judge=${out.judge.tier}@${out.judge.confidence}` : ''));
+    if (out.legacy.escalations.length) console.log(`         escalations: ${out.legacy.escalations.map((x) => `${x.source}:${x.fromTier}→${x.toTier}`).join(', ')}`);
+    if (out.shortfall) console.log(`SHORTFALL wants ${out.shortfall.selected?.tier}:${out.shortfall.selected?.model}  lift=${(out.shortfall.lift || []).join('+') || '-'}`);
+    console.log(`\nDECISIONS`);
+    for (const c of (out.engine?.considered || [])) console.log(`  ${c.matched ? '✔' : '·'} ${pad(c.name, 32)} priority ${c.priority}`);
+    console.log(`\nENGINE   decision=${out.engine?.decision}  tier=${out.engine?.tier}  effort=${out.engine?.effort ?? '-'}  hosts=${out.engine?.hosts ? out.engine.hosts.join(',') : '-'}`);
+    console.log(`SERVED   ${out.served}${out.engine && !out.engine.agreesWithLegacy ? (out.engine.mode === 'enforce' ? '  (engine overrode legacy)' : '  (observe: engine would pick ' + out.engine.tier + ')') : ''}\n`);
+  })().catch((err) => { console.error(err); process.exit(1); });
+}
+
+if (require.main === module || process.env._LYNKR_SUBCMD === 'route') main();
+module.exports = { main };