lynkr 9.14.2 → 9.14.7

package/src/routing/capability-seeds/sources.js

@@ -0,0 +1,69 @@
+/**
+ * Shared plumbing for capability-seed sources (online benchmark data).
+ *
+ * Every adapter is optional and fail-soft: no network, bad payload, or
+ * unknown shape → { status:'skipped', reason } — never throws, never blocks
+ * the script, and never touches request-path routing (this only runs in
+ * scripts/seed-capabilities.js). Fetched payloads are cached under
+ * data/capability-sources-cache.json with a 7-day TTL so refreshes are
+ * cheap and CI stays hermetic (tests use fixtures, never the network).
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const CACHE_PATH = path.join(__dirname, '../../../data/capability-sources-cache.json');
+const CACHE_TTL_MS = 7 * 24 * 60 * 60 * 1000;
+const FETCH_TIMEOUT_MS = 20000;
+
+function _readCache() {
+  try {
+    if (!fs.existsSync(CACHE_PATH)) return {};
+    return JSON.parse(fs.readFileSync(CACHE_PATH, 'utf8'));
+  } catch {
+    return {};
+  }
+}
+
+function _writeCache(cache) {
+  try {
+    fs.mkdirSync(path.dirname(CACHE_PATH), { recursive: true });
+    fs.writeFileSync(CACHE_PATH, JSON.stringify(cache, null, 2));
+  } catch {
+    // cache is best-effort; a failed write must not fail seeding
+  }
+}
+
+function getCached(source) {
+  const cache = _readCache();
+  const entry = cache[source];
+  if (!entry || typeof entry !== 'object') return null;
+  if (Date.now() - (entry.fetchedAt || 0) > CACHE_TTL_MS) return null;
+  return entry.payload ?? null;
+}
+
+function setCached(source, payload) {
+  const cache = _readCache();
+  cache[source] = { fetchedAt: Date.now(), payload };
+  _writeCache(cache);
+}
+
+async function fetchJson(url, { timeoutMs = FETCH_TIMEOUT_MS } = {}) {
+  const response = await fetch(url, {
+    signal: AbortSignal.timeout(timeoutMs),
+    headers: { Accept: 'application/json' },
+  });
+  if (!response.ok) throw new Error(`HTTP ${response.status}`);
+  return response.json();
+}
+
+const skipped = (reason) => ({ status: 'skipped', reason });
+
+module.exports = {
+  CACHE_PATH,
+  CACHE_TTL_MS,
+  getCached,
+  setCached,
+  fetchJson,
+  skipped,
+};

package/src/routing/capability-seeds/swebench.js

@@ -0,0 +1,66 @@
+/**
+ * SWE-Bench Verified adapter: per-model % resolved from the public
+ * leaderboard raw data.
+ *
+ * Source: https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/master/data/leaderboards.json
+ * Shape: { leaderboards: [{ name: "Verified"|"Lite"|..., results: [{ name, resolved, date, ... }] }] }
+ * Entry names look like "Claude 4.5 Opus (high) · mini-SWE-agent · 2026-02-17"
+ * (model + effort qualifier + scaffold + date). We keep the best resolved%
+ * per cleaned model name; family mapping happens in normalize.js
+ * (token-overlap against known families, unmapped surfaced for review).
+ *
+ * Fail-soft: any fetch/parse problem → { status:'skipped', reason }.
+ */
+
+const { fetchJson, getCached, setCached, skipped } = require('./sources');
+
+const LEADERBOARDS_URL = 'https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/master/data/leaderboards.json';
+const BOARD = 'Verified';
+
+function cleanName(name) {
+  return String(name || '')
+    .split('·')[0] // drop scaffold + date segments
+    .replace(/\((high|medium|low)\)/gi, '') // effort qualifier (we keep max across them)
+    .trim()
+    .replace(/\s+/g, ' ')
+    .toLowerCase();
+}
+
+/**
+ * @param {object} [opts] — { refresh:boolean } (false = cache-only)
+ * @returns {Promise<{ status, board?, entries?:Array<{name,resolved,date}>, reason? }>}
+ */
+async function fetchSweBench({ refresh = true } = {}) {
+  if (!refresh) {
+    const cached = getCached('swebench');
+    if (cached) return { status: 'ok', board: BOARD, entries: cached, cached: true };
+    return skipped('no cache (run with --refresh)');
+  }
+  try {
+    const data = await fetchJson(LEADERBOARDS_URL);
+    const boards = Array.isArray(data?.leaderboards) ? data.leaderboards : [];
+    const verified = boards.find((b) => String(b?.name || '').toLowerCase() === BOARD.toLowerCase());
+    if (!verified || !Array.isArray(verified.results)) {
+      return skipped(`board "${BOARD}" not found in payload`);
+    }
+    const best = new Map();
+    for (const r of verified.results) {
+      const name = cleanName(r?.name);
+      const resolved = Number(r?.resolved);
+      if (!name || !Number.isFinite(resolved)) continue;
+      const prev = best.get(name);
+      if (!prev || resolved > prev.resolved) {
+        best.set(name, { name, resolved, date: r?.date ?? null });
+      }
+    }
+    const entries = [...best.values()];
+    setCached('swebench', entries);
+    return { status: 'ok', board: BOARD, entries };
+  } catch (err) {
+    const cached = getCached('swebench');
+    if (cached) return { status: 'ok', board: BOARD, entries: cached, cached: true, stale: err.message };
+    return skipped(`fetch failed and no cache: ${err.message}`);
+  }
+}
+
+module.exports = { fetchSweBench, cleanName, LEADERBOARDS_URL, BOARD };

package/src/routing/decide.js

@@ -0,0 +1,149 @@
+/**
+ * Routing decision core — the harness-shaped, standalone pieces of the
+ * per-request model decision, extracted from routing/index.js so they can be
+ * consumed by more than one caller:
+ *
+ *   - routing/index.js (the live request path, unchanged behavior)
+ *   - the off-policy evaluator (WS4 follow-up), which must re-derive a
+ *     candidate policy's π(a|x) on logged rows using the exact same context
+ *     vector and candidate-eligibility rules the live policy used
+ *   - a future remote decision endpoint, which needs decide-shaped logic
+ *     that doesn't reach into request/response plumbing
+ *
+ * Deliberately excluded: session pins, the escalation ladder, kNN confidence
+ * branching, deadline/tenant overrides. Those are decision *inputs and
+ * post-passes* owned by the caller; this module owns only "given a context
+ * and a candidate set, pick one and report the probability of that pick."
+ *
+ * Nothing here catches errors — callers keep their existing
+ * degradation.record() wrapping so failure accounting stays where it was.
+ */
+
+const { getBandit } = require('./bandit');
+
+/**
+ * Task-type one-hot vocabulary. Order is load-bearing: it defines feature
+ * indices 6..11 of the context vector, and every persisted bandit arm was
+ * trained against this order. Do not reorder or insert — append only, and
+ * only together with a bandit-state migration.
+ */
+const TASK_TYPES = ['code_gen', 'summarization', 'reasoning', 'factoid', 'chat', 'other'];
+
+/**
+ * Build the 12-dim context vector the bandit (and any off-policy estimator
+ * replaying its decisions) scores against.
+ *
+ * Layout: [score, log-tokens, has-tools, streaming, risk, agentic,
+ *          ...one-hot task type (6)].
+ *
+ * @param {object} args
+ * @param {object} args.analysis — complexity analysis ({ score, breakdown })
+ * @param {object} [args.payload] — request payload (for tools presence)
+ * @param {object} [args.options] — routing options (for streaming flag)
+ * @param {object} [args.risk] — risk analysis ({ level })
+ * @param {object} [args.agenticResult] — agentic detection ({ isAgentic })
+ * @returns {number[]} 12-dim feature vector
+ */
+function buildContextVector({ analysis, payload, options, risk, agenticResult }) {
+  const inferredTask = (analysis?.breakdown?.taskType?.reason || 'other').toLowerCase();
+  const taskIdx = Math.max(0, TASK_TYPES.findIndex(t => inferredTask.includes(t)));
+  return [
+    (analysis?.score || 0) / 100,
+    Math.log(Math.max(1, analysis?.breakdown?.tokenCount || 0) + 1) / 15,
+    ((payload?.tools?.length ?? 0) > 0) ? 1 : 0,
+    options?.streaming ? 1 : 0,
+    risk?.level === 'high' ? 1 : risk?.level === 'medium' ? 0.5 : 0,
+    agenticResult?.isAgentic ? 1 : 0,
+    ...TASK_TYPES.map((_, i) => i === taskIdx ? 1 : 0),
+  ];
+}
+
+/**
+ * Build the bandit's candidate set: the current selection plus the kNN
+ * alternative, if it differs AND is configured in some TIER_* entry.
+ *
+ * Tier-aware filter: the bandit may explore freely across the user's
+ * configured tiers (e.g. swap a SIMPLE request to the COMPLEX-tier model),
+ * but never pick a credentialed-but-untiered model (e.g. an Azure deployment
+ * present in .env for another purpose but referenced by no TIER_*). Tier
+ * routing stays the source of truth for eligibility.
+ *
+ * @param {{ provider: string, model: string }} current
+ * @param {{ provider: string, model: string }|null} alternative
+ * @returns {Array<{ provider: string, model: string }>}
+ */
+function buildCandidates(current, alternative) {
+  const candidates = [{ provider: current.provider, model: current.model }];
+  if (alternative && alternative.model && alternative.model !== current.model) {
+    const configured = require('./model-tiers').getModelTierSelector().getAllConfiguredModels();
+    const inConfig = configured.some(
+      m => m.provider === alternative.provider && m.model === alternative.model
+    );
+    if (inConfig) {
+      candidates.push({ provider: alternative.provider, model: alternative.model });
+    }
+  }
+  return candidates;
+}
+
+/**
+ * The decision core: given a tier, a candidate set, and a context vector,
+ * have the bandit pick one and report the pick's propensity.
+ *
+ * Returns null when there's nothing to adjudicate (fewer than 2 candidates)
+ * or the bandit declined — callers keep their pre-existing selection.
+ *
+ * @param {string} tier
+ * @param {Array<{ provider, model }>} candidates
+ * @param {number[]} ctx — from buildContextVector
+ * @returns {null | { provider, model, ucb, explored, propensity, candidates, context }}
+ */
+function decide(tier, candidates, ctx) {
+  if (!candidates || candidates.length < 2) return null;
+  const picked = getBandit().pick(tier, candidates, ctx);
+  if (!picked) return null;
+  return { ...picked, candidates, context: ctx };
+}
+
+/**
+ * Stamp propensity/candidates/_banditContext onto a built decision (WS4.2).
+ *
+ * Collapse rule: the bandit's propensity only describes the served choice if
+ * the served (provider, model) is still one of the bandit's candidates. If a
+ * deterministic downstream override (deadline / tenant) swapped the served
+ * model out of that set — or the bandit never ran — collapse to
+ * propensity=1.0 with a single-entry candidate list, so off-policy
+ * estimators treat the row as a deterministic decision.
+ *
+ * _banditContext is underscored so it never leaks to response headers; the
+ * feedback path consumes it to call bandit.update().
+ *
+ * @param {object} decision — mutated in place (matches prior inline behavior)
+ * @param {{ provider: string, model: string }} served
+ * @param {null | { propensity, candidates, context }} banditResult — from decide()
+ * @returns {object} the same decision, for chaining
+ */
+function stampPropensity(decision, served, banditResult) {
+  const banditPickedServed = banditResult?.candidates
+    && banditResult.candidates.some(
+      c => c.provider === served.provider && c.model === served.model
+    );
+  if (banditPickedServed) {
+    decision.propensity = banditResult.propensity ?? 1.0;
+    decision.candidates = banditResult.candidates;
+    decision._banditContext = banditResult.context;
+  } else {
+    decision.propensity = 1.0;
+    decision.candidates = [{ provider: served.provider, model: served.model }];
+    decision._banditContext = null;
+  }
+  return decision;
+}
+
+module.exports = {
+  TASK_TYPES,
+  buildContextVector,
+  buildCandidates,
+  decide,
+  stampPropensity,
+};

package/src/routing/deescalator.js

@@ -103,6 +103,42 @@ function _clearCache() {
   _cache.clear();
 }
 
+/**
+ * Percentage-based demotion holdout (ROUTING-NOTES §4.10.4).
+ *
+ * Even when the evidence supports demotion, a configurable slice of sessions
+ * is deliberately excluded and served at the original tier. Those held-out
+ * rows are a continuously-running baseline: comparing their outcomes against
+ * demoted rows (method suffix '+deescalated' vs '+deescalation_holdout' in
+ * routing telemetry) proves the demotion rule is still net-positive over
+ * time, instead of trusting a one-time calibration.
+ *
+ * The bucket is a deterministic hash of the session key, so a session lands
+ * on the same side of the holdout on every turn — cohorts stay clean and a
+ * conversation never flip-flops tiers because of the holdout itself.
+ *
+ * LYNKR_DEESCALATION_HOLDOUT_PCT: 0–100, default 10. 0 disables the holdout.
+ * Read at call time (not module load) so tests and live re-config work.
+ *
+ * @param {string|null} sessionKey — session id/fingerprint; null → never held out
+ * @returns {boolean} true when this session must NOT be demoted (baseline cohort)
+ */
+function isHeldOut(sessionKey) {
+  if (!sessionKey) return false;
+  const raw = Number.parseInt(process.env.LYNKR_DEESCALATION_HOLDOUT_PCT, 10);
+  const pct = Number.isNaN(raw) ? 10 : Math.min(100, Math.max(0, raw));
+  if (pct === 0) return false;
+  // FNV-1a over the session key → stable bucket in 0..99.
+  let hash = 0x811c9dc5;
+  const s = String(sessionKey);
+  for (let i = 0; i < s.length; i++) {
+    hash ^= s.charCodeAt(i);
+    hash = Math.imul(hash, 0x01000193);
+  }
+  const bucket = (hash >>> 0) % 100;
+  return bucket < pct;
+}
+
 /**
  * Shadow-mode policy. Wraps the live routing decision by delegating to
  * `determineProviderSmart` and then applying `suggestDemotion` on the result.
@@ -143,6 +179,7 @@ async function shadowDeescalate(payload) {
 module.exports = {
   suggestDemotion,
   shadowDeescalate,
+  isHeldOut,
   TIER_ORDER,
   _clearCache,
 };