@ludi-uni/ludi-agent-kit 0.1.0

package/tests/qoder-observer.test.mjs

@@ -0,0 +1,266 @@
+// Qoder provider-metadata observer (qoder-models-cache.json priceFactor).
+// Cases A–J. The cache is read-only; tests use temp fixture files. Nothing real is
+// written — catalog/routing/settings untouched, only out/ artifacts + temp state.
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtempSync, rmSync, writeFileSync, existsSync, readFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { resolve, dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { fromQoderCache, fromFixture } from '../lib/observe/sources.mjs';
+import { validateObservation, normalizeObservation, ingestObservations, loadObservationStore, productionObservations } from '../lib/observe/observation.mjs';
+import { diffCatalog, buildCatalogProposal, applyProposalToCatalog, proposalToEvents } from '../lib/observe/differ.mjs';
+import { runMaintenanceJob } from '../lib/job.mjs';
+import { runMaintenancePlan, selectTierModel, DEFAULT_POLICY } from '../lib/maintenance-exec.mjs';
+import { evaluateMaintenance, effectiveCatalog } from '../lib/maintenance.mjs';
+import { loadRouting } from '../lib/routing.mjs';
+import { loadAgents } from '../lib/agents.mjs';
+import { mergeRegistries } from '../lib/registry.mjs';
+
+const kit = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const routing = loadRouting(join(kit, 'routing/routing.json'));
+const { agents } = loadAgents(join(kit, 'agents'), routing);
+const tmp = () => mkdtempSync(join(tmpdir(), 'qoder-'));
+
+const cacheDoc = (priceFactor, extra = {}) => ({
+  version: 2, updatedAt: 1790224130520,
+  models: [
+    { id: 'Qwen3.8-Flash', name: 'Qwen3.8-Flash (0x)', priceFactor, api: 'qoder-api', provider: 'qoder', baseUrl: 'https://api3.qoder.sh/', contextWindow: 1000000, ...extra },
+    { id: 'Qwen3.8-Max', priceFactor: 0.5, api: 'qoder-api', provider: 'qoder' },
+  ],
+});
+const MODELS = [{ provider: 'qoder', model: 'Qwen3.8-Flash' }];
+const CAT = {
+  version: 1, updatedAt: '2026-03-01',
+  models: [
+    { provider: 'qoder', model: 'Qwen3.8-Flash', status: 'free-campaign', cost: { free: true }, postCampaignCost: null, contextK: 1024, vision: false, toolUse: 'good', location: 'cloud', scores: { coding: 62, reasoning: 55, speed: 85 } },
+    { provider: 'cloudp', model: 'cheap-ok', status: 'active', cost: { usdPerMInput: 0.2, usdPerMOutput: 0.8 }, contextK: 256, vision: false, toolUse: 'good', location: 'cloud', scores: { coding: 68, reasoning: 62, speed: 75 } },
+    { provider: 'cloudp', model: 'pro-strong', status: 'active', cost: { usdPerMInput: 5, usdPerMOutput: 20 }, contextK: 400, vision: false, toolUse: 'good', location: 'cloud', scores: { coding: 90, reasoning: 88, speed: 60 } },
+    { provider: 'freetoken', model: 'Local-1', status: 'active', cost: { free: true }, location: 'local', local: { powerWatts: 450, taskMinutes: 4 }, contextK: 64, vision: false, toolUse: 'basic', scores: { coding: 40, reasoning: 35, speed: 70 } },
+  ],
+};
+const REG = mergeRegistries(null, { version: 1, backends: {
+  qoder: { provider: 'qoder', model: 'Qwen3.8-Flash' },
+  cheap: { provider: 'cloudp', model: 'cheap-ok' },
+  sol: { provider: 'cloudp', model: 'pro-strong' },
+  local: { provider: 'freetoken', model: 'Local-1' },
+} });
+
+const writeCache = (dir, doc) => { const p = join(dir, 'qoder-models-cache.json'); writeFileSync(p, JSON.stringify(doc)); return p; };
+
+// --- A: priceFactor 0 -> free=true -------------------------------------------
+test('A: priceFactor 0 -> free=true observation', () => {
+  const r = fromQoderCache(cacheDoc(0), { models: MODELS });
+  assert.equal(r.state, 'ok');
+  assert.equal(r.observations.length, 1);
+  assert.equal(r.observations[0].changes.free, true);
+  assert.equal(r.snapshot['Qwen3.8-Flash'], 0);
+});
+
+// --- B: priceFactor > 0 -> free=false ----------------------------------------
+test('B: priceFactor 1 -> free=false observation', () => {
+  const r = fromQoderCache(cacheDoc(1), { models: MODELS });
+  assert.equal(r.observations[0].changes.free, false);
+  assert.equal(r.snapshot['Qwen3.8-Flash'], 1);
+});
+
+// --- C: 0 -> 1 = free-campaign-ended ------------------------------------------
+test('C: transition 0 -> 1 emits free=false (campaign ended)', () => {
+  const r = fromQoderCache(cacheDoc(1), { models: MODELS, previous: { 'Qwen3.8-Flash': 0 } });
+  assert.equal(r.observations.length, 1);
+  assert.equal(r.observations[0].changes.free, false);
+  assert.match(r.observations[0].evidence[0], /was 0/);
+});
+
+// --- D: 1 -> 0 = free-campaign-started ----------------------------------------
+test('D: transition 1 -> 0 emits free=true (campaign started)', () => {
+  const r = fromQoderCache(cacheDoc(0), { models: MODELS, previous: { 'Qwen3.8-Flash': 1 } });
+  assert.equal(r.observations.length, 1);
+  assert.equal(r.observations[0].changes.free, true);
+});
+
+// --- E: 0 -> 0 = no event ------------------------------------------------------
+test('E: same priceFactor -> no observation', () => {
+  const r = fromQoderCache(cacheDoc(0), { models: MODELS, previous: { 'Qwen3.8-Flash': 0 } });
+  assert.equal(r.observations.length, 0);
+  const r2 = fromQoderCache(cacheDoc(1), { models: MODELS, previous: { 'Qwen3.8-Flash': 1 } });
+  assert.equal(r2.observations.length, 0);
+});
+
+// --- F: cache missing -> unknown, catalog unchanged ----------------------------
+test('F: missing cache file -> state=missing, no observation, catalog unchanged', () => {
+  const dir = tmp();
+  const r = fromQoderCache(join(dir, 'nope.json'), { models: MODELS });
+  assert.equal(r.state, 'missing');
+  assert.equal(r.observations.length, 0);
+  assert.equal(CAT.models[0].cost.free, true); // untouched
+  rmSync(dir, { recursive: true, force: true });
+});
+
+// --- G: malformed JSON / schema -> unknown, job continues ----------------------
+test('G: malformed JSON and unknown schema -> unknown, no crash', async () => {
+  const dir = tmp();
+  const bad = join(dir, 'bad.json'); writeFileSync(bad, '{ not json');
+  assert.equal(fromQoderCache(bad, { models: MODELS }).state, 'malformed');
+  assert.equal(fromQoderCache({ version: 2, models: 'nope' }, { models: MODELS }).state, 'no-schema');
+  assert.equal(fromQoderCache(cacheDoc(0), { models: [{ provider: 'qoder', model: 'Ghost-9' }] }).state, 'entry-missing');
+  assert.equal(fromQoderCache(cacheDoc(undefined, { priceFactor: undefined }), { models: MODELS }).state, 'priceFactor-missing');
+  // job continues on probe failure
+  const run = await runMaintenanceJob({ outDir: dir, adapterDir: join(kit, 'adapters/pi'), kit, catalog: CAT, routing, registry: REG, agents, policy: DEFAULT_POLICY, checkQoder: true, qoderListing: false });
+  assert.equal(run.status, 'ok');
+  assert.equal(run.ingest.probeFailed, true);
+  assert.equal(run.quiet, true);
+  rmSync(dir, { recursive: true, force: true });
+});
+
+// --- H: campaign end -> maintenance preview re-evaluates -----------------------
+test('H: 0->non-zero -> diff proposes cost.free=false; preview re-evaluates cheapest-sufficient', () => {
+  const dir = tmp();
+  const store = join(dir, 'obs.jsonl');
+  const r = fromQoderCache(cacheDoc(1), { models: MODELS, previous: { 'Qwen3.8-Flash': 0 } });
+  ingestObservations(store, r.observations);
+  const observations = loadObservationStore(store).observations;
+  const proposal = buildCatalogProposal(CAT, diffCatalog(CAT, observations), observations);
+  assert.ok(proposal.updates.some(u => u.field === 'cost.free' && u.observedValue === false));
+  // proposal -> event -> maintenance preview on hypothetical catalog
+  const events = proposalToEvents(proposal);
+  assert.ok(events.some(e => e.type === 'free-campaign-ended' && e.model === 'Qwen3.8-Flash'));
+  const hypothetical = applyProposalToCatalog(CAT, proposal);
+  const run = runMaintenancePlan({ routing, registry: REG, agents, catalog: hypothetical, events, policy: DEFAULT_POLICY });
+  assert.equal(run.monitor.changed, true);
+  const qd = run.proposal.decisions.find(d => d.backend === 'qoder');
+  assert.ok(qd, 'qoder backend re-evaluated');
+  // postCampaignCost is null -> costUnknown; never treated as $0
+  const hq = effectiveCatalog(hypothetical, events).find(m => m.model === 'Qwen3.8-Flash');
+  assert.equal(hq.costUnknown, true);
+  assert.notEqual(hq.cost?.free, true);
+  // monitor tier can no longer pick it as a free cloud model
+  const after = selectTierModel(effectiveCatalog(hypothetical, events), 'monitor', DEFAULT_POLICY);
+  assert.notEqual(after.selectionPath, 'free-cloud');
+  rmSync(dir, { recursive: true, force: true });
+});
+
+// --- I: campaign start -> Qwen becomes free-eligible again ---------------------
+test('I: non-zero -> 0 -> free=true; Qwen re-enters free-eligible pool in preview', () => {
+  const dir = tmp();
+  const store = join(dir, 'obs.jsonl');
+  // catalog currently shows paid (campaign previously ended)
+  const paidCat = structuredClone(CAT);
+  paidCat.models[0].status = 'active';
+  paidCat.models[0].cost = { free: false, usdPerMInput: null, usdPerMOutput: null };
+  paidCat.models[0].costUnknown = true;
+  const r = fromQoderCache(cacheDoc(0), { models: MODELS, previous: { 'Qwen3.8-Flash': 1 } });
+  ingestObservations(store, r.observations);
+  const observations = loadObservationStore(store).observations;
+  const proposal = buildCatalogProposal(paidCat, diffCatalog(paidCat, observations), observations);
+  assert.ok(proposal.updates.some(u => u.field === 'cost.free' && u.observedValue === true));
+  const events = proposalToEvents(proposal);
+  assert.ok(events.some(e => e.type === 'free-campaign-started' && e.model === 'Qwen3.8-Flash'));
+  const hypothetical = applyProposalToCatalog(paidCat, proposal);
+  const hq = effectiveCatalog(hypothetical, events).find(m => m.model === 'Qwen3.8-Flash');
+  assert.equal(hq.status, 'free-campaign');
+  assert.equal(hq.cost.free, true);
+  const sel = selectTierModel(effectiveCatalog(hypothetical, events), 'monitor', DEFAULT_POLICY);
+  assert.equal(sel.selectionPath, 'free-cloud');
+  assert.equal(sel.selected.model, 'qoder/Qwen3.8-Flash');
+  rmSync(dir, { recursive: true, force: true });
+});
+
+// --- J: costUnknown is never treated as $0 -------------------------------------
+test('J: post-campaign costUnknown -> not free, not $0, flagged in proposal', () => {
+  const events = [{ type: 'free-campaign-ended', provider: 'qoder', model: 'Qwen3.8-Flash', asOf: '2026-03-01' }];
+  const eff = effectiveCatalog(CAT, events).find(m => m.model === 'Qwen3.8-Flash');
+  assert.equal(eff.costUnknown, true);
+  assert.equal(eff.cost, null);
+  // costPoints(scoreModel) treats null cost as unknown, not 0
+  const report = evaluateMaintenance({ routing, registry: REG, agents, catalog: CAT, events });
+  const qd = report.decisions.find(d => d.backend === 'qoder');
+  assert.ok(qd.currentScore.unknown.includes('cost'));
+  // decision reason / proposal carries the uncertainty (never a silent $0 switch)
+  if (qd.decision === 'propose') {
+    const change = report.changes.find(c => c.affected.backend === 'qoder');
+    assert.match(change.expectedCostImpact, /unknown/);
+  }
+});
+
+// --- combined probes: --check-pi + --check-qoder run together ----------------
+test('combined: checkPi + checkQoder both contribute; a stale qoder cache does not block pi availability', async () => {
+  const dir = tmp();
+  const listing = { models: new Set(['qoder/Qwen3.8-Flash', 'cloudp/cheap-ok', 'cloudp/pro-strong', 'freetoken/Local-1']), providers: new Set(['qoder', 'cloudp', 'freetoken']), source: 'test' };
+  // observedAt must be ~now: a stale timestamp is correctly deduped as stale.
+  const qoderOk = fromQoderCache(cacheDoc(0), { models: MODELS, observedAt: new Date().toISOString() }); // free:true, first sighting
+  const run = await runMaintenanceJob({ outDir: dir, adapterDir: join(kit, 'adapters/pi'), kit, catalog: CAT, routing, registry: REG, agents, policy: DEFAULT_POLICY,
+    checkPi: true, listing, checkQoder: true, qoderListing: qoderOk });
+  assert.equal(run.status, 'ok');
+  assert.equal(run.ingest.probeFailed, false);
+  // both sources contributed: availability (pi) + free (qoder cache)
+  const store = loadObservationStore(join(dir, 'model-observations.jsonl')).observations;
+  assert.ok(store.some(o => o.changes?.availability === 'available'), 'pi availability observation stored');
+  assert.ok(store.some(o => o.changes?.free === true && o.source?.trust === 'provider_local_cache'), 'qoder free observation stored');
+  rmSync(dir, { recursive: true, force: true });
+});
+
+test('combined: qoder probe failure is quiet and does not suppress pi observations', async () => {
+  const dir = tmp();
+  const listing = { models: new Set(['qoder/Qwen3.8-Flash']), providers: new Set(['qoder']), source: 'test' };
+  const run = await runMaintenanceJob({ outDir: dir, adapterDir: join(kit, 'adapters/pi'), kit, catalog: CAT, routing, registry: REG, agents, policy: DEFAULT_POLICY,
+    checkPi: true, listing, checkQoder: true, qoderListing: false }); // qoder probe failed
+  assert.equal(run.status, 'ok');
+  assert.equal(run.ingest.probeFailed, true);
+  const store = loadObservationStore(join(dir, 'model-observations.jsonl')).observations;
+  assert.ok(store.some(o => o.changes?.availability === 'available'), 'pi observation still stored despite qoder failure');
+  rmSync(dir, { recursive: true, force: true });
+});
+
+// --- observation schema + trust ------------------------------------------------
+test('qoder observation validates and carries provider_local_cache trust', () => {
+  const r = fromQoderCache(cacheDoc(0), { models: MODELS });
+  const o = normalizeObservation(r.observations[0]);
+  assert.deepEqual(validateObservation(o), []);
+  assert.equal(o.source.trust, 'provider_local_cache');
+  assert.equal(o.source.type, 'cli');
+});
+
+// --- fixture/production boundary ------------------------------------------------
+test('fixture observations are tagged test and excluded from production proposals', async () => {
+  const dir = tmp();
+  // a fixture observation claiming qoder free:false (the contamination we just cleaned)
+  const fixtureObs = fromFixture({ announcements: [{ provider: 'qoder', model: 'Qwen3.8-Flash', type: 'free-campaign-ended', asOf: '2026-03-01', label: 'Qoder pricing page', url: 'https://example.invalid/x' }] });
+  assert.equal(fixtureObs[0].environment, 'test');
+  assert.equal(fixtureObs[0].sourceFixture, true);
+  // real production observation: qoder free:true from the cache
+  const prodObs = fromQoderCache(cacheDoc(0), { models: MODELS, observedAt: new Date().toISOString() }).observations;
+  const store = join(dir, 'obs.jsonl');
+  ingestObservations(store, [...fixtureObs, ...prodObs]);
+  const all = loadObservationStore(store).observations;
+  const prod = productionObservations(all);
+  assert.equal(all.length, 2);
+  assert.equal(prod.length, 1);
+  assert.equal(prod[0].changes.free, true); // fixture free:false excluded
+  // production proposal sees only the real free:true
+  const proposal = buildCatalogProposal(CAT, diffCatalog(CAT, prod), prod);
+  assert.equal(proposal.updates.filter(u => u.field === 'cost.free' && u.observedValue === false).length, 0);
+  rmSync(dir, { recursive: true, force: true });
+});
+
+test('job excludes fixture observations from the production diff even when stored', async () => {
+  const dir = tmp();
+  // seed the store with a fixture observation directly (simulating prior contamination)
+  const fixtureObs = fromFixture({ announcements: [{ provider: 'qoder', model: 'Qwen3.8-Flash', type: 'free-campaign-ended', asOf: '2026-03-01', label: 'fixture' }] });
+  ingestObservations(join(dir, 'model-observations.jsonl'), fixtureObs);
+  const run = await runMaintenanceJob({ ...{ outDir: dir, adapterDir: join(kit, 'adapters/pi'), kit, catalog: CAT, routing, registry: REG, agents, policy: DEFAULT_POLICY }, checkQoder: true, qoderListing: fromQoderCache(cacheDoc(0), { models: MODELS, observedAt: new Date().toISOString() }) });
+  assert.equal(run.status, 'ok');
+  assert.equal(run.excludedTestObservations, 1); // fixture record excluded from the diff
+  rmSync(dir, { recursive: true, force: true });
+});
+
+// --- real cache smoke (read-only) ----------------------------------------------
+test('smoke: real ~/.pi/agent/qoder-models-cache.json parses and reports Qwen3.8-Flash state', () => {
+  const real = join(process.env.USERPROFILE ?? process.env.HOME, '.pi', 'agent', 'qoder-models-cache.json');
+  if (!existsSync(real)) { console.log('  (skipped: real cache not present)'); return; }
+  const r = fromQoderCache(real, { models: MODELS });
+  assert.equal(r.state, 'ok');
+  assert.equal(typeof r.snapshot['Qwen3.8-Flash'], 'number');
+  // first sighting emits the current free state (0 -> free:true on this machine)
+  assert.equal(r.observations[0].changes.free, r.snapshot['Qwen3.8-Flash'] === 0);
+});

package/tests/reassign-progression.test.mjs

@@ -0,0 +1,104 @@
+// REASSIGN failure (MODEL_FAILURE / BACKEND_LIMIT) candidate semantics: a dead
+// candidate is never re-invoked on the same-capability retry; untried candidates
+// (e.g. local) are reached; health skips never consume the attempt budget.
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { resolve, dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { loadRouting } from '../lib/routing.mjs';
+import { loadAgents } from '../lib/agents.mjs';
+import { DEFAULT_POLICY, mergePolicy } from '../lib/orchestrator/policy.mjs';
+import { createAgentRunner } from '../lib/orchestrator/runner.mjs';
+import { orchestrate } from '../lib/orchestrator/orchestrator.mjs';
+import { createHealthMonitor } from '../lib/orchestrator/health.mjs';
+import { createScriptedInvoker } from '../adapters/pi/lib/invoke.mjs';
+
+const kit = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const routing = loadRouting(join(kit, 'routing/routing.json'));
+const { agents } = loadAgents(join(kit, 'agents'), routing);
+// strong-code chain: devin -> qoder -> sol -> codex -> local
+const REG = { version: 1, backends: {
+  local: { provider: 'pl', model: 'ml', thinking: 'off' },
+  cheap: { provider: 'pc', model: 'mc', thinking: 'low' },
+  sol:   { provider: 'ps', model: 'ms', thinking: 'high' },
+  codex: { provider: 'px', model: 'mx', thinking: 'high' },
+  qoder: { provider: 'pq', model: 'mq', thinking: 'low' },
+  devin: { provider: 'pd', model: 'md', thinking: 'high' },
+} };
+const policyWith = over => mergePolicy(DEFAULT_POLICY, over ?? {});
+const jsonReply = obj => `notes\n\n\`\`\`json\n${JSON.stringify(obj)}\n\`\`\``;
+const okReply = jsonReply({ status: 'completed', summary: 'done', acceptance: [{ id: 'A1', met: true, evidence: 'e' }] });
+const spec = (id, agent, extra = {}) => ({ id, title: `task ${id}`, goal: `goal ${id}`, agent, dependencies: [], acceptance: ['done'], ...extra });
+const run = ({ policy, ...o }) => orchestrate({ request: 'r', agents, routing, registry: REG, policy: policyWith(policy), ...o });
+// Unclassified backend error: NOT matched by classifyBackendFailure, so health never skips it.
+const DEAD = new Error('pi exited 1: upstream returned status 402');
+const count = (calls, id) => calls.filter(c => c.modelId === id).length;
+
+function scripted(mapper, { health = null } = {}) {
+  const calls = [];
+  const invoke = createScriptedInvoker(mapper, calls);
+  return { calls, runner: createAgentRunner({ invoke, agents, routing, registry: REG, health, maxModelAttempts: 3 }) };
+}
+
+// A: strong-code (ladder end): devin/qoder/sol die with MODEL_FAILURE -> retry does
+// not re-invoke any of them; codex (untried) is reached.
+test('A: strong-code end of ladder: dead candidates are excluded on retry, untried candidate reached', async () => {
+  const { calls, runner } = scripted({ 'pd/md:high': DEAD, 'pq/mq:low': DEAD, 'ps/ms:high': DEAD, '*': okReply });
+  const r = await run({ plan: [spec('a', 'coder')], runner, policy: { limits: { max_retries: 2, model_attempts_per_task: 3, max_total_attempts_per_task: 8 } } });
+  assert.equal(r.status, 'completed', JSON.stringify(r.tasks[0]));
+  assert.deepEqual(calls.map(c => c.modelId), ['pd/md:high', 'pq/mq:low', 'ps/ms:high', 'px/mx:high']);
+  assert.equal(count(calls, 'ps/ms:high'), 1, 'the last dead candidate was not re-invoked');
+  assert.equal(r.tasks[0].attempts, 2);
+  const retry = r.trace.find(e => e.type === 'retry');
+  assert.equal(retry.failureClass, 'MODEL_FAILURE');
+  assert.ok(r.autoDecisions.some(d => /next untried candidate on strong-code/.test(d.choice)));
+});
+
+// B: unclassified quota error on every hosted candidate: the same model is never
+// re-run "just because" it was last; the task fails cleanly once nothing is left.
+test('B: unclassified quota error is not health-skipped, yet no model runs twice', async () => {
+  const { calls, runner } = scripted({ '*': DEAD });
+  const r = await run({ plan: [spec('a', 'coder')], runner, policy: { limits: { max_retries: 5, model_attempts_per_task: 3, max_total_attempts_per_task: 20 } } });
+  assert.equal(r.tasks[0].status, 'failed');
+  const ids = calls.map(c => c.modelId);
+  assert.equal(new Set(ids).size, ids.length, `a model was invoked twice: ${ids.join(',')}`);
+  assert.deepEqual(ids, ['pd/md:high', 'pq/mq:low', 'ps/ms:high', 'px/mx:high', 'pl/ml:off']);
+  assert.match(r.tasks[0].blockedReason, /no untried model candidate left on strong-code/);
+  assert.equal(r.tasks[0].attempts, 2, 'no extra rounds spent invoking nothing');
+});
+
+// C: local is reached even with the default budget (4) when the first pass consumes 3.
+test('C: untried local candidate is reached with the default attempt budget', async () => {
+  const { calls, runner } = scripted({ 'pd/md:high': DEAD, 'pq/mq:low': DEAD, 'ps/ms:high': DEAD, 'px/mx:high': DEAD, '*': okReply });
+  const r = await run({ plan: [spec('a', 'coder')], runner, policy: { limits: { max_retries: 2, model_attempts_per_task: 4, max_total_attempts_per_task: 5 } } });
+  assert.equal(r.status, 'completed', JSON.stringify(r.tasks[0]));
+  assert.equal(calls.at(-1).modelId, 'pl/ml:off');
+  assert.equal(count(calls, 'px/mx:high'), 1);
+});
+
+// D: attempt budget counts real invocations only; health skips are free.
+test('D: health-skipped candidates cost no budget; budget counts invocations only', async () => {
+  const policy = policyWith({ limits: { max_retries: 2, model_attempts_per_task: 3, max_total_attempts_per_task: 3 } });
+  const health = createHealthMonitor({ policy });
+  health.bindRun('run-x');
+  // devin and qoder are known-exhausted before the run.
+  health.report({ provider: 'pd', model: 'md' }, 'usage limit has been reached');
+  health.report({ provider: 'pq', model: 'mq' }, 'usage limit has been reached');
+  const { calls, runner } = scripted({ 'ps/ms:high': DEAD, 'px/mx:high': DEAD, '*': okReply }, { health });
+  const r = await run({ plan: [spec('a', 'coder')], runner, policy: policy });
+  assert.equal(r.status, 'completed', JSON.stringify(r.tasks[0]));
+  assert.deepEqual(calls.map(c => c.modelId), ['ps/ms:high', 'px/mx:high', 'pl/ml:off']);
+  assert.equal(r.tasks[0].totalModelAttempts, 3, 'exactly three real invocations counted');
+  const first = r.trace.find(e => e.type === 'result');
+  assert.equal(first.counters.candidatesSkipped, 2);
+  assert.equal(first.counters.invocationsStarted, 3);
+});
+
+// E: recoverable failure (TEST_FAILURE) still keeps the same model retryable (unchanged semantics).
+test('E: recoverable failure keeps the last model retryable (feedback retry)', async () => {
+  let n = 0;
+  const { calls, runner } = scripted({ '*': () => (++n === 1 ? jsonReply({ status: 'failed', summary: 'tests failed', acceptance: [{ id: 'A1', met: false, evidence: '1 failing' }] }) : okReply) });
+  const r = await run({ plan: [spec('a', 'coder')], runner });
+  assert.equal(r.status, 'completed');
+  assert.deepEqual(calls.map(c => c.modelId), ['pd/md:high', 'pd/md:high']);
+});

package/tests/retry-escalation.test.mjs

@@ -0,0 +1,120 @@
+// Failure-aware retry/escalation: protocol-quality failures advance to the next
+// candidate (or escalate capability) instead of burning same-model retries.
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { resolve, dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { loadRouting } from '../lib/routing.mjs';
+import { loadAgents } from '../lib/agents.mjs';
+import { DEFAULT_POLICY, mergePolicy } from '../lib/orchestrator/policy.mjs';
+import { createAgentRunner } from '../lib/orchestrator/runner.mjs';
+import { orchestrate } from '../lib/orchestrator/orchestrator.mjs';
+import { isProtocolFailure, classifyRun } from '../lib/orchestrator/failures.mjs';
+import { createScriptedInvoker } from '../adapters/pi/lib/invoke.mjs';
+
+const kit = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const routing = loadRouting(join(kit, 'routing/routing.json'));
+const { agents } = loadAgents(join(kit, 'agents'), routing);
+// cheap-code chain resolves to cheap -> local -> sol (no qoder/devin in this registry)
+const REG = { version: 1, backends: {
+  local: { provider: 'pl', model: 'ml', thinking: 'off' },
+  cheap: { provider: 'pc', model: 'mc', thinking: 'low' },
+  sol:   { provider: 'ps', model: 'ms', thinking: 'high' },
+  astra: { provider: 'pa', model: 'ma', thinking: 'medium', vision: true },
+  codex: { provider: 'px', model: 'mx', thinking: 'high' },
+} };
+const policyWith = over => mergePolicy(DEFAULT_POLICY, over ?? {});
+const jsonReply = obj => `notes\n\n\`\`\`json\n${JSON.stringify(obj)}\n\`\`\``;
+const spec = (id, agent, dependencies = [], extra = {}) => ({ id, title: `task ${id}`, goal: `goal ${id}`, agent, dependencies, acceptance: ['done'], ...extra });
+const run = ({ policy, ...o }) => orchestrate({ request: 'r', agents, routing, registry: REG, policy: policyWith(policy), ...o });
+const okReply = jsonReply({ status: 'completed', summary: 'done', acceptance: [{ id: 'A1', met: true, evidence: 'e' }] });
+
+function scripted(mapper) {
+  const calls = [];
+  const invoke = createScriptedInvoker(mapper, calls);
+  return { calls, runner: createAgentRunner({ invoke, agents, routing, registry: REG }) };
+}
+
+// A: MALFORMED_RESULT advances to the next candidate, not an infinite same-model retry.
+test('A: malformed result -> next candidate within the same attempt', async () => {
+  const { calls, runner } = scripted({ '*': req => req.modelId.startsWith('pc/') ? 'Done' : okReply });
+  const r = await run({ plan: [spec('a', 'scout')], runner });
+  assert.equal(r.status, 'completed');
+  assert.equal(r.tasks[0].attempts, 1);             // no wasted orchestrator retry
+  assert.deepEqual(calls.map(c => c.modelId), ['pc/mc:low', 'pl/ml:off']);
+});
+
+// B: turn-limit TIMEOUT with toolCalls=0 -> next candidate.
+test('B: turn-limit timeout with no tool progress -> next candidate', async () => {
+  assert.equal(isProtocolFailure('TIMEOUT', { child: { toolCalls: 0 } }), true);
+  const { calls, runner } = scripted({ '*': req => req.modelId.startsWith('pc/') ? { ok: false, error: 'turn limit 12', failureClass: 'TIMEOUT', child: { toolCalls: 0 } } : okReply });
+  const r = await run({ plan: [spec('a', 'scout')], runner });
+  assert.equal(r.status, 'completed');
+  assert.deepEqual(calls.map(c => c.modelId), ['pc/mc:low', 'pl/ml:off']);
+});
+
+// C: empty response -> next candidate.
+test('C: empty response -> next candidate', async () => {
+  assert.equal(classifyRun({ error: 'empty model response' }), 'EMPTY_RESPONSE');
+  const { calls, runner } = scripted({ '*': req => req.modelId.startsWith('pc/') ? { ok: false, error: 'empty model response', failureClass: 'EMPTY_RESPONSE' } : okReply });
+  const r = await run({ plan: [spec('a', 'scout')], runner });
+  assert.equal(r.status, 'completed');
+  assert.deepEqual(calls.map(c => c.modelId), ['pc/mc:low', 'pl/ml:off']);
+});
+
+// D: protocol failure on every cheap candidate -> escalate to strong-code.
+test('D: cheap candidates exhausted by protocol failures -> escalate to strong-code', async () => {
+  // Every candidate fails protocol on the FIRST pass; on escalation to strong-code
+  // the codex candidate (px/mx, strong-only) succeeds.
+  const { calls, runner } = scripted({ '*': req => req.modelId.startsWith('px/') ? okReply : 'Done' });
+  const r = await run({ plan: [spec('a', 'scout')], runner, policy: { limits: { max_retries: 4, model_attempts_per_task: 3, max_total_attempts_per_task: 8 } } });
+  assert.equal(r.status, 'completed');
+  assert.ok(calls.some(c => c.modelId === 'px/mx:high'), 'reached a strong-code-only candidate');
+  assert.ok(r.tasks[0].capability === 'strong-code' || r.autoDecisions.some(d => /escalate to strong-code/.test(d.choice)));
+});
+
+// E: recoverable failure (test failure) -> same-model feedback retry allowed.
+test('E: recoverable failure keeps the same model retryable', async () => {
+  let n = 0;
+  const { calls, runner } = scripted({ '*': () => (++n === 1
+    ? jsonReply({ status: 'failed', summary: 'tests failed', acceptance: [{ id: 'A1', met: false, evidence: 'x' }] })
+    : okReply) });
+  const r = await run({ plan: [spec('a', 'scout')], runner });
+  assert.equal(r.status, 'completed');
+  assert.equal(r.tasks[0].attempts, 2);                       // a real retry happened
+  assert.deepEqual(calls.map(c => c.modelId), ['pc/mc:low', 'pc/mc:low']); // same model retried
+});
+
+// F: a different candidate is reached before the per-task attempt budget is spent.
+test('F: candidate progression happens before maxAttempts is exhausted', async () => {
+  const { calls, runner } = scripted({ '*': req => req.modelId.startsWith('pc/') ? 'Done' : okReply });
+  const r = await run({ plan: [spec('a', 'scout')], runner });
+  assert.equal(r.status, 'completed');
+  assert.ok(calls.length >= 2 && calls[0].modelId !== calls[1].modelId, 'tried a different candidate');
+});
+
+// G: every candidate fails -> task ultimately fails.
+test('G: all candidates fail -> task failed', async () => {
+  const { runner } = scripted({ '*': () => 'Done' });
+  const r = await run({ plan: [spec('a', 'scout')], runner, policy: { limits: { max_retries: 2, model_attempts_per_task: 3, max_total_attempts_per_task: 4 } } });
+  assert.equal(r.tasks[0].status, 'failed');
+  assert.equal(r.status, 'incomplete');
+});
+
+// H: a timeout WITH real tool progress stays retryable on the same model.
+test('H: turn-limit timeout WITH tool progress is recoverable (same-model retry)', async () => {
+  assert.equal(isProtocolFailure('TIMEOUT', { child: { toolCalls: 5 } }), false);
+});
+
+// Telemetry: protocol outcomes are recorded per provider/model (audit only).
+test('telemetry: malformed/empty/turn_limit/structured_ok are counted per model', async () => {
+  const recorded = [];
+  const session = { recordProtocol: r => recorded.push(`${r.provider}/${r.model}:${r.kind}`) };
+  const calls = [];
+  const invoke = createScriptedInvoker({ '*': req => req.modelId.startsWith('pc/') ? 'Done' : okReply }, calls);
+  const runner = createAgentRunner({ invoke, agents, routing, registry: REG, session });
+  const r = await run({ plan: [spec('a', 'scout')], runner });
+  assert.equal(r.status, 'completed');
+  assert.ok(recorded.includes('pc/mc:malformed'), `malformed recorded: ${recorded}`);
+  assert.ok(recorded.includes('pl/ml:structured_ok'), `structured_ok recorded: ${recorded}`);
+});

package/tests/routing.test.mjs

@@ -0,0 +1,110 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { resolve, dirname, join } from 'node:path';
+import { mkdtempSync, writeFileSync, mkdirSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { fileURLToPath } from 'node:url';
+import { loadRouting, mergeLocalRouting, validateRouting, resolveBackends, resolveModels } from '../lib/routing.mjs';
+import { loadAgents } from '../lib/agents.mjs';
+
+const kit = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+
+const base = () => ({
+  version: 1,
+  backends: { a: { tier: 'low' }, b: { tier: 'high', vision: true }, c: {} },
+  capabilities: { x: { primary: 'a', fallback: ['b'] }, see: { primary: 'b', fallback: ['a'], requires: { vision: true } } },
+});
+
+test('shipped routing.json validates', () => {
+  const r = loadRouting(resolve(kit, 'routing/routing.json'));
+  for (const c of ['cheap-code', 'strong-code', 'vision-reasoning', 'deep-review', 'browser']) assert.ok(r.capabilities[c], c);
+  for (const b of ['local', 'cheap', 'sol', 'astra', 'codex']) assert.ok(r.backends[b], b);
+});
+
+test('valid minimal config has no errors', () => assert.deepEqual(validateRouting(base()), []));
+
+test('local routes override, add and disable capabilities without changing shared routing', () => {
+  const config = base();
+  config.escalation = { ladders: { code: ['x', 'see'] } };
+  const effective = mergeLocalRouting(config, { version: 1, capabilities: {
+    x: null,
+    see: { primary: 'b', fallback: ['c'] },
+    new: { primary: 'a', fallback: ['c'] },
+  } });
+  assert.equal(config.capabilities.x.primary, 'a');
+  assert.equal(effective.capabilities.x, undefined);
+  assert.deepEqual(effective.escalation.ladders.code, ['see']);
+  assert.deepEqual(resolveBackends(effective, 'new'), ['a', 'c']);
+  assert.deepEqual(resolveBackends(effective, 'see'), ['b', 'c']);
+});
+
+test('local routes reject broken references and malformed overrides', () => {
+  assert.throws(() => mergeLocalRouting(base(), { version: 1, capabilities: { x: { primary: 'missing' } } }), /not a defined backend/);
+  assert.throws(() => mergeLocalRouting(base(), { version: 1, capabilities: { x: { primary: 'a', fallback: ['a'] } } }), /repeats primary/);
+  assert.throws(() => mergeLocalRouting(base(), { version: 1, capabilities: { ghost: null } }), /unknown capability/);
+});
+
+test('loadRouting applies local file and excludes agents for disabled capabilities', () => {
+  const dir = mkdtempSync(join(tmpdir(), 'kit-routing-'));
+  try {
+    mkdirSync(join(dir, 'routing'));
+    writeFileSync(join(dir, 'routing', 'routing.json'), JSON.stringify(base()));
+    writeFileSync(join(dir, 'routing', 'routing.local.json'), JSON.stringify({ version: 1, capabilities: { x: null } }));
+    const routing = loadRouting(join(dir, 'routing', 'routing.json'));
+    assert.equal(routing.capabilities.x, undefined);
+    const { agents, errors } = loadAgents(join(kit, 'agents'), mergeLocalRouting(loadRouting(join(kit, 'routing', 'routing.json')), { version: 1, capabilities: { 'cheap-code': null } }));
+    assert.deepEqual(errors, []);
+    assert.ok(!agents.some(agent => agent.meta.capability === 'cheap-code'));
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+test('unknown primary / fallback backends are rejected', () => {
+  const c = base(); c.capabilities.x.primary = 'nope'; c.capabilities.x.fallback = ['zzz'];
+  const errs = validateRouting(c);
+  assert.ok(errs.some(e => e.includes('primary "nope"')));
+  assert.ok(errs.some(e => e.includes('fallback "zzz"')));
+});
+
+test('fallback may not repeat primary or itself', () => {
+  const c = base(); c.capabilities.x.fallback = ['a', 'b', 'b'];
+  const errs = validateRouting(c);
+  assert.ok(errs.some(e => e.includes('repeats primary')));
+  assert.ok(errs.some(e => e.includes('repeats "b"')));
+});
+
+test('vision requirement is enforced on primary only', () => {
+  const c = base(); c.capabilities.see.primary = 'a';
+  assert.ok(validateRouting(c).some(e => e.includes('requires vision')));
+  assert.deepEqual(validateRouting(base()), []);
+});
+
+test('version, names and tiers are checked', () => {
+  const c = base(); c.version = 2; c.backends['Bad Name'] = {}; c.backends.a.tier = 'ultra';
+  const errs = validateRouting(c);
+  assert.ok(errs.some(e => e.includes('version')));
+  assert.ok(errs.some(e => e.includes('invalid backend name')));
+  assert.ok(errs.some(e => e.includes('invalid tier')));
+});
+
+test('escalation ladders must reference known capabilities', () => {
+  const c = base(); c.escalation = { ladders: { l: ['x', 'ghost'] } };
+  assert.ok(validateRouting(c).some(e => e.includes('unknown capability "ghost"')));
+});
+
+test('resolveBackends returns primary then fallbacks in order', () => {
+  assert.deepEqual(resolveBackends(base(), 'x'), ['a', 'b']);
+  assert.throws(() => resolveBackends(base(), 'nope'));
+});
+
+test('resolveModels binds through a model map and skips unbound backends; no provider names in code', () => {
+  const map = { version: 1, backends: { b: { provider: 'p1', model: 'm1' } } };
+  const r = resolveModels(base(), map, 'x');
+  assert.deepEqual(r.candidates.map(c => c.backend), ['b']);
+  assert.deepEqual(r.unbound, ['a']);
+  const swapped = { version: 1, backends: { a: { provider: 'other', model: 'other-m' }, b: { provider: 'p1', model: 'm1' } } };
+  assert.equal(resolveModels(base(), swapped, 'x').candidates[0].provider, 'other');
+  const vis = resolveModels(base(), swapped, 'see');
+  assert.equal(vis.candidates[1].degraded, true);
+});