@ludi-uni/ludi-agent-kit 0.1.0

package/tests/maintenance-exec.test.mjs

@@ -0,0 +1,218 @@
+// Phase 2: maintenance execution tiers + cheapest-sufficient selection.
+// Cases A-G from the Phase 2 spec. Selection is a dry-run decision layer: no model
+// is invoked and no config is written.
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { resolve, dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import {
+  selectTierModel, buildMonitorOutput, escalationDecision, runMaintenancePlan,
+  localElectricityCost, estimatedCostPerRun, validateExecPolicy, loadExecPolicy, DEFAULT_POLICY,
+} from '../lib/maintenance-exec.mjs';
+import { effectiveCatalog, evaluateMaintenance, loadCatalog, validateCatalog } from '../lib/maintenance.mjs';
+import { loadRouting } from '../lib/routing.mjs';
+import { loadAgents } from '../lib/agents.mjs';
+import { mergeRegistries } from '../lib/registry.mjs';
+
+const kit = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const routing = loadRouting(join(kit, 'routing/routing.json'));
+const { agents } = loadAgents(join(kit, 'agents'), routing);
+
+// --- fixture helpers -------------------------------------------------------
+const m = (provider, model, over = {}) => ({
+  provider, model, status: 'active', cost: { usdPerMInput: 0.5, usdPerMOutput: 2 },
+  contextK: 256, vision: false, toolUse: 'good', location: 'cloud',
+  scores: { coding: 60, reasoning: 55, speed: 70 }, ...over,
+});
+const cat = models => ({ version: 1, updatedAt: '2026-03-01', models });
+const registryWith = backends => mergeRegistries(null, { version: 1, backends });
+const sel = (models, tier, opts) => selectTierModel(models, tier, DEFAULT_POLICY, opts);
+
+test('SWE-2 campaign cutoff changes free-capacity advice without live rebinding', () => {
+  const catalog = loadCatalog(join(kit, 'adapters/pi/model-catalog.json'));
+  const policy = loadExecPolicy(join(kit, 'adapters/pi/maintenance-policy.json'));
+  const reg = registryWith({ devin: { provider: 'devin', model: 'swe-2-high' } });
+  const before = runMaintenancePlan({ routing, registry: reg, agents, catalog, policy, asOf: '2026-10-10T14:59:59Z' });
+  const beforeCode = before.freeCapacityPlan.find(p => p.capability === 'strong-code');
+  assert.equal(beforeCode.recommendedFree, 'devin/swe-2-high');
+  assert.equal(before.freeCapacityPlan.find(p => p.capability === 'deep-review').recommendedFree, 'devin/swe-2-high');
+  assert.equal(before.freeCapacityPlan.find(p => p.capability === 'cheap-code').recommendedFree, 'qoder/Qwen3.8-Flash');
+  assert.equal(before.freeCapacityPlan.find(p => p.capability === 'vision-reasoning').recommendedFree, null);
+  assert.equal(beforeCode.freeCandidates[0].freeUntil, '2026-10-10T15:00:00Z');
+  assert.equal(before.outcome, 'no-change');
+
+  const after = runMaintenancePlan({ routing, registry: reg, agents, catalog, policy, asOf: '2026-10-10T15:00:00Z' });
+  assert.equal(after.freeCapacityPlan.find(p => p.capability === 'strong-code').recommendedFree, null);
+  assert.equal(after.monitor.changed, true);
+  assert.match(after.monitor.reasons.join(' '), /free campaign cutoff reached/);
+  assert.equal(after.proposal.decisions.find(d => d.backend === 'devin').decision === 'keep' || after.proposal.decisions.find(d => d.backend === 'devin').decision === 'propose', true);
+  assert.equal(effectiveCatalog(catalog, [], '2026-10-10T15:00:00Z').find(m => m.provider === 'devin').cost.free, false);
+  assert.deepEqual(validateCatalog({ version: 1, models: [{ ...catalog.models.find(m => m.provider === 'devin'), freeUntil: 'tomorrow' }] }).some(e => e.includes('freeUntil')), true);
+});
+
+const FREE_STRONG = m('freep', 'free-strong', { cost: { free: true }, scores: { coding: 78, reasoning: 70, speed: 80 } });
+const FREE_WEAK = m('freep', 'free-weak', { cost: { free: true }, scores: { coding: 30, reasoning: 25, speed: 60 } });
+const CHEAP_OK = m('cloudp', 'cheap-ok', { cost: { usdPerMInput: 0.2, usdPerMOutput: 0.8 }, scores: { coding: 68, reasoning: 62, speed: 75 } });
+const EXPENSIVE_STRONG = m('cloudp', 'pro-strong', { cost: { usdPerMInput: 5, usdPerMOutput: 20 }, scores: { coding: 90, reasoning: 88, speed: 60 } });
+const LOCAL = m('localp', 'local-mid', { location: 'local', cost: { free: true }, local: { powerWatts: 400, taskMinutes: 10 }, scores: { coding: 45, reasoning: 40, speed: 50 }, toolUse: 'basic' });
+
+// --- A: strong free model -> monitor and evaluate both pick it --------------
+test('Case A: a strong free model is selected for monitor and evaluate', () => {
+  const models = [FREE_STRONG, CHEAP_OK, EXPENSIVE_STRONG];
+  for (const tier of ['monitor', 'evaluate']) {
+    const r = sel(models, tier);
+    assert.equal(r.selected.model, 'freep/free-strong', `${tier}: ${r.selectionReason}`);
+    assert.equal(r.selectionPath, 'free-cloud');
+    assert.equal(r.selected.effectiveCostUsd < 0.02, true); // only the speed penalty
+  }
+});
+
+// --- B: free ends, cheap sufficient cloud -> cheapest-sufficient, no premium
+test('Case B: free campaign end -> cheapest-sufficient cloud, no reconfigure escalation', () => {
+  const catalog = cat([
+    m('qoder', 'Qwen3.8-Flash', { status: 'free-campaign', cost: { free: true }, postCampaignCost: { usdPerMInput: 0.3, usdPerMOutput: 1.2 }, scores: { coding: 62, reasoning: 55, speed: 85 } }),
+    CHEAP_OK, EXPENSIVE_STRONG,
+  ]);
+  const events = [{ type: 'free-campaign-ended', provider: 'qoder', model: 'Qwen3.8-Flash', asOf: '2026-03-01' }];
+  const effective = effectiveCatalog(catalog, events);
+  const r = sel(effective, 'evaluate');
+  assert.equal(r.selected.model, 'cloudp/cheap-ok', r.selectionReason);
+  assert.equal(r.selectionPath, 'cheapest-sufficient-cloud');
+  // and the pipeline: monitor detects, evaluate decides, no premium escalation
+  const registry = registryWith({ cheap: { provider: 'qoder', model: 'Qwen3.8-Flash' } });
+  const run = runMaintenancePlan({ routing, registry, agents, catalog, events });
+  assert.equal(run.monitor.changed, true);
+  assert.equal(run.escalation, null);
+  assert.equal(run.outcome, 'evaluated-no-change'); // advantage below margin -> keep
+});
+
+// --- C: cheap models below requiredQuality -> escalate to a stronger model -
+test('Case C: models below requiredQuality are rejected; a stronger candidate is selected', () => {
+  const weak = m('cloudp', 'too-weak', { cost: { usdPerMInput: 0.05, usdPerMOutput: 0.2 }, scores: { coding: 30, reasoning: 25, speed: 80 } });
+  const r = sel([weak, EXPENSIVE_STRONG], 'evaluate'); // requiredQuality 65
+  assert.equal(r.selected.model, 'cloudp/pro-strong');
+  const rejected = r.candidates.find(c => c.model === 'cloudp/too-weak');
+  assert.equal(rejected.eligible, false);
+  assert.ok(rejected.rejectedReasons.some(x => x.includes('requiredQuality')));
+});
+
+// --- D: no eligible cloud -> local fallback --------------------------------
+test('Case D: cloud unusable -> local fallback with electricity cost', () => {
+  const gone = m('cloudp', 'gone', { status: 'removed' });
+  const r = sel([gone, LOCAL], 'monitor'); // requiredQuality 40; local quality ~44
+  assert.equal(r.selectionPath, 'local-fallback');
+  assert.equal(r.selected.model, 'localp/local-mid');
+  assert.equal(r.fallbackOccurred, true);
+  assert.equal(r.selected.location, 'local');
+  assert.ok(r.selected.costDetail.electricityUsd > 0);
+});
+
+// --- E: cloud API cheaper than local electricity -> cloud chosen ------------
+test('Case E: cheaper cloud API beats local electricity estimate', () => {
+  const powerHungry = structuredClone(LOCAL);
+  powerHungry.local = { powerWatts: 900, taskMinutes: 30 }; // 0.45kWh * $0.30 = $0.135/run
+  const cheapCloud = m('cloudp', 'tiny', { cost: { usdPerMInput: 0.05, usdPerMOutput: 0.1 }, scores: { coding: 55, reasoning: 50, speed: 90 } });
+  const r = sel([powerHungry, cheapCloud], 'monitor');
+  assert.equal(r.selected.location, 'cloud');
+  assert.equal(r.selected.model, 'cloudp/tiny');
+  // sanity: the local run really would cost more
+  assert.ok(localElectricityCost(powerHungry.local, 0.30) > estimatedCostPerRun(cheapCloud, DEFAULT_POLICY, [powerHungry, cheapCloud], 'monitor').total);
+});
+
+// --- F: multi-capability change -> reconfigure escalation -------------------
+test('Case F: changes spanning multiple capabilities escalate to reconfigure', () => {
+  const catalog = cat([
+    m('qoder', 'Qwen3.8-Flash', { status: 'free-campaign', cost: { free: true }, postCampaignCost: { usdPerMInput: 0.3, usdPerMOutput: 1.2 }, scores: { coding: 30, reasoning: 25, speed: 85 } }),
+    CHEAP_OK, EXPENSIVE_STRONG,
+    m('cloudp', 'vis-strong', { vision: true, cost: { usdPerMInput: 3, usdPerMOutput: 12 }, scores: { coding: 85, reasoning: 88, speed: 55 } }),
+  ]);
+  const registry = registryWith({
+    cheap: { provider: 'qoder', model: 'Qwen3.8-Flash' },
+    sol: { provider: 'qoder', model: 'Qwen3.8-Flash' },   // two primaries on the same dying model
+    astra: { provider: 'qoder', model: 'Qwen3.8-Flash' }, // vision caps too
+  });
+  const events = [{ type: 'removed', provider: 'qoder', model: 'Qwen3.8-Flash', asOf: '2026-03-01' }];
+  const run = runMaintenancePlan({ routing, registry, agents, catalog, events });
+  assert.ok(run.escalation, 'expected escalation');
+  assert.equal(run.escalation.targetTier, 'reconfigure');
+  assert.equal(run.escalation.sourceTier, 'evaluate');
+  assert.ok(run.escalation.affectedCapabilities.length >= 2);
+  assert.ok(run.tiers.some(t => t.role === 'reconfigure'));
+  assert.ok(run.escalation.estimatedDecisionCostUsd >= 0);
+  // proposal changes carry the escalation record
+  assert.ok(run.proposal.changes.every(c => c.escalation?.targetTier === 'reconfigure'));
+});
+
+// --- G: availability probe failure -> unknown, never "gone" -----------------
+test('Case G: failed availability probe is unknown, not unavailable', () => {
+  const monitor = buildMonitorOutput({ events: [], catalog: cat([CHEAP_OK]), availability: null, availabilitySource: 'pi --list-models (failed)' });
+  assert.equal(monitor.infoStatus.probeFailed, true);
+  assert.equal(monitor.changed, false); // nothing declared missing
+  const r = sel([CHEAP_OK], 'monitor', { availability: null });
+  assert.equal(r.selected.model, 'cloudp/cheap-ok'); // still eligible
+  assert.equal(r.candidates[0].availability, 'unknown');
+});
+
+// --- supporting behaviour ---------------------------------------------------
+test('free below requiredQuality is not used (free never overrides the quality bar)', () => {
+  const r = sel([FREE_WEAK, CHEAP_OK], 'evaluate');
+  assert.equal(r.selected.model, 'cloudp/cheap-ok');
+  assert.equal(r.selectionPath, 'cheapest-sufficient-cloud');
+  const rejected = r.candidates.find(c => c.model === 'freep/free-weak');
+  assert.equal(rejected.eligible, false);
+});
+
+test('localElectricityCost: watts x minutes x price', () => {
+  assert.ok(Math.abs(localElectricityCost({ powerWatts: 450, taskMinutes: 4 }, 0.30) - 0.009) < 1e-9);
+  assert.equal(localElectricityCost({ powerWatts: 450 }, 0.30), null);
+  assert.equal(localElectricityCost(null, 0.30), null);
+});
+
+test('monitor output shape: changed/reasons/affectedModels/severity/escalationRequired', () => {
+  const out = buildMonitorOutput({ events: [{ type: 'price-changed', provider: 'p', model: 'x' }], catalog: cat([m('p', 'x')]) });
+  assert.equal(out.changed, true);
+  assert.deepEqual(out.affectedModels, ['p/x']);
+  assert.equal(out.severity, 'low');
+  assert.equal(out.escalationRequired, false);
+  const quiet = buildMonitorOutput({ events: [], catalog: cat([m('p', 'x')]) });
+  assert.equal(quiet.changed, false);
+  assert.equal(quiet.severity, 'none');
+});
+
+test('runMaintenancePlan short-circuits on no change (only monitor tier runs)', () => {
+  const registry = registryWith({ cheap: { provider: 'cloudp', model: 'cheap-ok' } });
+  const run = runMaintenancePlan({ routing, registry, agents, catalog: cat([CHEAP_OK]), events: [] });
+  assert.equal(run.outcome, 'no-change');
+  assert.deepEqual(run.tiers.map(t => t.role), ['monitor']);
+  assert.equal(run.proposal, null);
+});
+
+test('escalationDecision returns null for small single-capability changes', () => {
+  const result = { changes: [{ affected: { capabilities: ['cheap-code'], agents: ['scout'] }, scores: { current: 50, proposed: 62, delta: 12 }, confidence: 'high' }], decisions: [], infoStatus: { eventsApplied: [] } };
+  assert.equal(escalationDecision(result, { severity: 'low' }, DEFAULT_POLICY), null);
+});
+
+test('exec policy validates and merges over defaults', () => {
+  assert.ok(validateExecPolicy({ version: 1, requiredQuality: { monitor: 200 } }).some(e => e.includes('requiredQuality')));
+  const p = loadExecPolicy(null);
+  assert.equal(p.requiredQuality.reconfigure, 80);
+  assert.equal(p.electricityPricePerKwh, 0.30);
+});
+
+test('run report records tier selections, candidates, costs and flags', () => {
+  const registry = registryWith({ cheap: { provider: 'qoder', model: 'Qwen3.8-Flash' } });
+  const catalog = cat([
+    m('qoder', 'Qwen3.8-Flash', { status: 'free-campaign', cost: { free: true }, postCampaignCost: { usdPerMInput: 0.3, usdPerMOutput: 1.2 }, scores: { coding: 30, reasoning: 25, speed: 85 } }),
+    CHEAP_OK,
+  ]);
+  const events = [{ type: 'free-campaign-ended', provider: 'qoder', model: 'Qwen3.8-Flash' }];
+  const run = runMaintenancePlan({ routing, registry, agents, catalog, events });
+  for (const t of run.tiers) {
+    assert.ok(t.selected?.model, `${t.role} selected`);
+    assert.ok(t.selectionReason);
+    assert.ok(Array.isArray(t.candidates) && t.candidates.length);
+    assert.ok(typeof t.selected.effectiveCostUsd === 'number');
+    assert.ok(typeof t.fallbackOccurred === 'boolean');
+  }
+  assert.ok(run.estimatedDecisionCostUsd > 0);
+});

package/tests/maintenance-runner.test.mjs

@@ -0,0 +1,222 @@
+// Phase 3: maintenance runner — real (stubbed) tier invocation, schema validation,
+// one retry, ordered fallback, deterministic authority, and per-run cost profiles.
+// Cases A–J from the Phase 3 spec. No real model is invoked; `invoke` is scripted.
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { resolve, dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import {
+  invokeTier, runMaintenanceLive, validateStructuredOutput, extractJson,
+  buildTierPrompt, TIER_SCHEMAS, MONITOR_SCHEMA,
+} from '../lib/maintenance-runner.mjs';
+import { selectTierModel, estimatedCostPerRun, DEFAULT_POLICY } from '../lib/maintenance-exec.mjs';
+import { effectiveCatalog } from '../lib/maintenance.mjs';
+import { loadRouting } from '../lib/routing.mjs';
+import { loadAgents } from '../lib/agents.mjs';
+import { mergeRegistries } from '../lib/registry.mjs';
+
+const kit = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const routing = loadRouting(join(kit, 'routing/routing.json'));
+const { agents } = loadAgents(join(kit, 'agents'), routing);
+
+const m = (provider, model, over = {}) => ({
+  provider, model, status: 'active', cost: { usdPerMInput: 0.5, usdPerMOutput: 2 },
+  contextK: 256, vision: false, toolUse: 'good', location: 'cloud',
+  scores: { coding: 60, reasoning: 55, speed: 70 }, ...over,
+});
+const cat = models => ({ version: 1, updatedAt: '2026-03-01', models });
+const registryWith = backends => mergeRegistries(null, { version: 1, backends });
+
+const FREE_GOOD = m('freep', 'free-good', { cost: { free: true }, scores: { coding: 75, reasoning: 68, speed: 80 } });
+const CHEAP2 = m('cloudp', 'cheap-2', { cost: { usdPerMInput: 0.1, usdPerMOutput: 0.4 }, scores: { coding: 66, reasoning: 60, speed: 78 } });
+const PREMIUM = m('cloudp', 'premium-huge', { premium: true, cost: { usdPerMInput: 10, usdPerMOutput: 40 }, scores: { coding: 95, reasoning: 95, speed: 40 } });
+const LOCAL = m('localp', 'local-mid', { location: 'local', cost: { free: true }, local: { powerWatts: 400, taskMinutes: 10 }, scores: { coding: 45, reasoning: 40, speed: 50 }, toolUse: 'basic' });
+
+const json = o => JSON.stringify(o);
+const ok = text => ({ ok: true, text, durationMs: 5 });
+const fail = error => ({ ok: false, error, durationMs: 3 });
+const scriptInvoker = (script, calls = []) => async req => {
+  calls.push(req.modelId);
+  const v = typeof script === 'function' ? script(req) : script[req.modelId.split(':')[0]] ?? script['*'];
+  return typeof v === 'function' ? v(req) : v;
+};
+
+const MON_OK = () => json({ decision: 'changed', confidence: 0.9, reasoningSummary: 'event diffs found', affectedModels: ['qoder/Qwen3.8-Flash'], severity: 'medium', evaluateNeeded: true, recommendedActions: [] });
+const EVAL_OK = (c = 0.85) => json({ decision: 'keep', confidence: c, reasoningSummary: 'within margin', affectedCapabilities: ['cheap-code'], recommendedActions: [], recommendedModels: [] });
+const REC_OK = () => json({ decision: 'propose', confidence: 0.8, reasoningSummary: 'structural', affectedCapabilities: ['cheap-code', 'strong-code'], recommendedActions: [], recommendedModels: [], routingNotes: [] });
+
+// Post-campaign Qwen3.8-Flash stays mid-quality: the switch is simple, no escalation.
+const dyingCatalog = () => cat([
+  m('qoder', 'Qwen3.8-Flash', { status: 'free-campaign', cost: { free: true }, postCampaignCost: { usdPerMInput: 0.3, usdPerMOutput: 1.2 }, scores: { coding: 62, reasoning: 55, speed: 85 } }),
+  CHEAP2, FREE_GOOD,
+]);
+
+// --- A: no change -> monitor only ------------------------------------------
+test('A: no-change -> only monitor is invoked, run stops', async () => {
+  const calls = [];
+  const run = await runMaintenanceLive({
+    routing, registry: registryWith({ cheap: { provider: 'cloudp', model: 'cheap-2' } }),
+    agents, catalog: cat([CHEAP2, FREE_GOOD]), events: [],
+    invoke: scriptInvoker({ '*': () => ok(json({ decision: 'no-change', confidence: 0.95, reasoningSummary: 'quiet', severity: 'none', evaluateNeeded: false })) }, calls),
+  });
+  assert.equal(run.outcome, 'no-change');
+  assert.equal(calls.length, 1);
+  assert.equal(run.invocations.length, 1);
+  assert.equal(run.invocations[0].tier, 'monitor');
+});
+
+// --- B: simple change -> monitor + evaluate, no reconfigure -----------------
+test('B: simple change -> monitor and evaluate invoked, no reconfigure', async () => {
+  const calls = [];
+  const run = await runMaintenanceLive({
+    routing, registry: registryWith({ cheap: { provider: 'qoder', model: 'Qwen3.8-Flash' } }),
+    agents, catalog: dyingCatalog(),
+    events: [{ type: 'free-campaign-ended', provider: 'qoder', model: 'Qwen3.8-Flash' }],
+    invoke: scriptInvoker({ '*': req => ok(req.prompt.includes('STAGE: monitor') ? MON_OK() : EVAL_OK()) }, calls),
+  });
+  assert.equal(calls.length, 2);
+  assert.deepEqual(run.invocations.map(i => i.tier), ['monitor', 'evaluate']);
+  assert.equal(run.escalation, null);
+});
+
+// --- C: complex change -> all three tiers -----------------------------------
+test('C: multi-capability structural change -> reconfigure is invoked', async () => {
+  const calls = [];
+  const catalog = cat([
+    m('qoder', 'Qwen3.8-Flash', { scores: { coding: 30, reasoning: 25, speed: 85 } }),
+    CHEAP2, FREE_GOOD, m('cloudp', 'vis', { vision: true, cost: { usdPerMInput: 3, usdPerMOutput: 12 }, scores: { coding: 85, reasoning: 88, speed: 55 } }),
+  ]);
+  const registry = registryWith({
+    cheap: { provider: 'qoder', model: 'Qwen3.8-Flash' },
+    sol: { provider: 'qoder', model: 'Qwen3.8-Flash' },
+    astra: { provider: 'qoder', model: 'Qwen3.8-Flash' },
+  });
+  const run = await runMaintenanceLive({
+    routing, registry, agents, catalog,
+    events: [{ type: 'removed', provider: 'qoder', model: 'Qwen3.8-Flash' }],
+    invoke: scriptInvoker({ '*': req => ok(req.prompt.includes('STAGE: monitor') ? MON_OK() : req.prompt.includes('STAGE: evaluate') ? EVAL_OK() : REC_OK()) }, calls),
+  });
+  assert.deepEqual(run.invocations.map(i => i.tier), ['monitor', 'evaluate', 'reconfigure']);
+  assert.ok(run.escalation.escalationReason.length > 0);
+  assert.equal(run.escalation.targetTier, 'reconfigure');
+});
+
+// --- D: monitor model fails -> next cheap candidate --------------------------
+test('D: monitor invocation failure falls back to the next ordered candidate', async () => {
+  const calls = [];
+  const run = await runMaintenanceLive({
+    routing, registry: registryWith({ cheap: { provider: 'cloudp', model: 'cheap-2' } }),
+    agents, catalog: cat([FREE_GOOD, CHEAP2]), events: [],
+    invoke: scriptInvoker(req => req.modelId.startsWith('freep/') ? fail('rate limit exceeded') : ok(json({ decision: 'no-change', confidence: 0.9, reasoningSummary: 'ok', severity: 'none', evaluateNeeded: false })), calls),
+  });
+  assert.deepEqual(calls.map(c => c.split('/')[0]), ['freep', 'cloudp']);
+  const inv = run.invocations[0];
+  assert.equal(inv.fallbackOccurred, true);
+  assert.equal(inv.selectedModel, 'cloudp/cheap-2');
+  assert.equal(inv.invocations[0].failureClass, 'invoke');
+});
+
+// --- E: malformed JSON -> one retry -> success -------------------------------
+test('E: malformed structured output retries once then succeeds', async () => {
+  let n = 0;
+  const inv = await invokeTier('monitor', selectTierModel(effectiveCatalog(cat([FREE_GOOD])), 'monitor', DEFAULT_POLICY), {
+    invoke: async () => (++n === 1 ? ok('not json at all') : ok(MON_OK())),
+    catalogKeys: new Set(['freep/free-good']),
+  });
+  assert.equal(n, 2);
+  assert.equal(inv.output.decision, 'changed');
+  assert.equal(inv.attempts.length, 2);
+  assert.equal(inv.attempts[0].schemaValid, false);
+  assert.equal(inv.attempts[1].schemaValid, true);
+});
+
+// --- F: still invalid after retry -> deterministic continues -----------------
+test('F: invalid output after retry -> tier degrades to deterministic result', async () => {
+  const calls = [];
+  const run = await runMaintenanceLive({
+    routing, registry: registryWith({ cheap: { provider: 'cloudp', model: 'cheap-2' } }),
+    agents, catalog: cat([FREE_GOOD]), events: [],
+    invoke: scriptInvoker({ '*': () => ok('garbage {no json') }, calls),
+  });
+  const inv = run.invocations[0];
+  assert.equal(inv.degradedToDeterministic, true);
+  assert.equal(calls.length, 2); // initial + one retry, then stop
+  assert.equal(run.outcome, 'no-change'); // deterministic monitor still decides
+});
+
+// --- G/H: per-run cost comparison --------------------------------------------
+test('G: short profile -> cloud cheaper than local electricity -> cloud selected', () => {
+  const models = [LOCAL, CHEAP2];
+  const r = selectTierModel(models, 'monitor', DEFAULT_POLICY); // 8K in / 800 out
+  assert.equal(r.selected.location, 'cloud');
+  const localCost = estimatedCostPerRun(LOCAL, DEFAULT_POLICY, models, 'monitor');
+  const cloudCost = estimatedCostPerRun(CHEAP2, DEFAULT_POLICY, models, 'monitor');
+  assert.ok(cloudCost.api < localCost.electricity, `api ${cloudCost.api} vs elec ${localCost.electricity}`);
+});
+
+test('H: long coding profile -> token cost grows, local becomes cheaper', () => {
+  const expensiveCloud = m('cloudp', 'pricey', { cost: { usdPerMInput: 8, usdPerMOutput: 30 }, scores: { coding: 66, reasoning: 60, speed: 70 } });
+  const local = m('localp', 'local-ok', { location: 'local', cost: { free: true }, local: { powerWatts: 300, taskMinutes: 2 }, scores: { coding: 45, reasoning: 40, speed: 60 }, toolUse: 'basic' });
+  const policy = structuredClone(DEFAULT_POLICY);
+  policy.taskProfiles.monitor = { estimatedInputTokens: 500000, estimatedOutputTokens: 50000, estimatedTaskMinutes: 20 };
+  const models = [expensiveCloud, local];
+  const r = selectTierModel(models, 'monitor', policy);
+  const cloudApi = estimatedCostPerRun(expensiveCloud, policy, models, 'monitor').api; // ~$5.5
+  const localElec = estimatedCostPerRun(local, policy, models, 'monitor').electricity; // 0.3kWh*0.3=$0.03
+  assert.ok(cloudApi > localElec);
+  assert.equal(r.selected.location, 'local');
+  assert.equal(r.selectionPath, 'cheapest-sufficient-local'); // local wins on cost, not as a fallback
+});
+
+// --- I: premium not used for monitor -----------------------------------------
+test('I: premium models are excluded from monitor/evaluate selection', () => {
+  const r = selectTierModel([PREMIUM, CHEAP2], 'monitor', DEFAULT_POLICY);
+  assert.equal(r.selected.model, 'cloudp/cheap-2');
+  assert.ok(!r.candidates.some(c => c.model === 'cloudp/premium-huge'));
+  const rec = selectTierModel([PREMIUM, CHEAP2], 'reconfigure', DEFAULT_POLICY); // reconfigure may see it
+  assert.ok(rec.candidates.some(c => c.model === 'cloudp/premium-huge'));
+});
+
+// --- J: LLM recommends a non-catalog model -> rejected ------------------------
+test('J: recommendedModels outside the catalog are rejected and recorded', async () => {
+  const out = json({ decision: 'keep', confidence: 0.8, reasoningSummary: 'x', affectedCapabilities: [], recommendedActions: [], recommendedModels: ['evil/ghost-model', 'freep/free-good'] });
+  const inv = await invokeTier('evaluate', selectTierModel(effectiveCatalog(cat([FREE_GOOD, CHEAP2])), 'evaluate', DEFAULT_POLICY), {
+    invoke: async () => ok(out),
+    catalogKeys: new Set(['freep/free-good', 'cloudp/cheap-2']),
+  });
+  assert.deepEqual(inv.output.recommendedModels, ['freep/free-good']);
+  assert.deepEqual(inv.output.rejectedRecommendations, ['evil/ghost-model']);
+});
+
+// --- schema / parsing --------------------------------------------------------
+test('schema validation: required keys, enums, ranges', () => {
+  assert.deepEqual(validateStructuredOutput(MONITOR_SCHEMA, { decision: 'changed', confidence: 0.5, reasoningSummary: 'x' }), []);
+  assert.ok(validateStructuredOutput(MONITOR_SCHEMA, { decision: 'maybe', confidence: 0.5, reasoningSummary: 'x' }).some(e => e.includes('decision')));
+  assert.ok(validateStructuredOutput(MONITOR_SCHEMA, { decision: 'changed', confidence: 2, reasoningSummary: 'x' }).some(e => e.includes('confidence')));
+  assert.ok(validateStructuredOutput(MONITOR_SCHEMA, { confidence: 0.5 }).some(e => e.includes('decision')));
+  assert.ok(validateStructuredOutput(MONITOR_SCHEMA, 'text').some(e => e.includes('object')));
+});
+
+test('extractJson tolerates fences and surrounding prose', () => {
+  assert.equal(extractJson('```json\n{"a":1}\n```').a, 1);
+  assert.equal(extractJson('here is the result: {"a":{"b":2}} done').a.b, 2);
+  assert.equal(extractJson('no json'), null);
+});
+
+test('tier prompts embed deterministic facts and forbid invented models', () => {
+  const p = buildTierPrompt('evaluate', { evaluation: { changes: 1 }, catalogKeys: ['a/b'] });
+  assert.ok(p.includes('STAGE: evaluate'));
+  assert.ok(p.includes('a/b'));
+  assert.ok(p.includes('Never invent provider/model ids'));
+  const mp = buildTierPrompt('monitor', { monitor: { changed: true, reasons: ['e'], affectedModels: ['p/m'], severity: 'low' } });
+  assert.ok(mp.includes('Do NOT propose routing changes'));
+});
+
+test('quota failure is classified distinctly', async () => {
+  const inv = await invokeTier('monitor', selectTierModel(effectiveCatalog(cat([FREE_GOOD, CHEAP2])), 'monitor', DEFAULT_POLICY), {
+    invoke: async req => req.modelId.startsWith('freep/') ? fail('Error: usage limit has been reached') : ok(MON_OK()),
+    catalogKeys: new Set(),
+  });
+  assert.equal(inv.attempts[0].failureClass, 'quota');
+  assert.equal(inv.selected.model, 'cloudp/cheap-2');
+});

package/tests/maintenance.test.mjs

@@ -0,0 +1,195 @@
+// Model-provisioning maintenance: catalog/events validation, scoring, and the
+// Qoder/Qwen3.8-Flash free-campaign-end scenario. Proposals only — nothing is applied.
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { resolve, dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import {
+  evaluateMaintenance, loadCatalog, loadEvents, loadAvailabilityFile,
+  validateCatalog, validateEvents, scoreModel, DEFAULT_MARGIN,
+} from '../lib/maintenance.mjs';
+import { loadRouting } from '../lib/routing.mjs';
+import { loadAgents } from '../lib/agents.mjs';
+import { mergeRegistries } from '../lib/registry.mjs';
+import { parseModelList } from '../adapters/pi/lib/list-models.mjs';
+
+const kit = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const fx = join(kit, 'tests/fixtures/maintenance');
+const routing = loadRouting(join(kit, 'routing/routing.json'));
+const { agents } = loadAgents(join(kit, 'agents'), routing);
+const catalog = loadCatalog(join(fx, 'catalog.json'));
+const events = loadEvents(join(fx, 'events.json')); // qoder/Qwen3.8-Flash free-campaign-ended
+
+const baseRegistry = () => mergeRegistries(null, {
+  version: 1,
+  backends: {
+    local: { provider: 'freetoken', model: 'Qwen3.6-35B-A3B-NVFP4', thinking: 'off' },
+    cheap: { provider: 'openai-codex', model: 'gpt-5.6-luna', thinking: 'low' },
+    qoder: { provider: 'qoder', model: 'Qwen3.8-Flash', thinking: 'low' },
+    devin: { provider: 'devin', model: 'swe-2-high', thinking: 'high' },
+    sol: { provider: 'openai-codex', model: 'gpt-5.6-sol', thinking: 'medium' },
+    astra: { provider: 'openai-codex', model: 'gpt-6-astra', thinking: 'medium', vision: true },
+    codex: { provider: 'openai-codex', model: 'gpt-5.5', thinking: 'high' },
+  },
+});
+const decide = (report, backend) => report.decisions.find(d => d.backend === backend);
+
+test('catalog and events validators reject malformed input', () => {
+  assert.ok(validateCatalog({ version: 1, models: [{ provider: 'p', model: 'm', status: 'gone' }] }).some(e => e.includes('status')));
+  assert.ok(validateCatalog({ version: 1, models: [{ provider: 'p', model: 'm', status: 'active', cost: { usdPerMInput: -1 } }] }).some(e => e.includes('usdPerMInput')));
+  assert.ok(validateCatalog({ version: 1, models: [
+    { provider: 'p', model: 'm', status: 'active' }, { provider: 'p', model: 'm', status: 'active' },
+  ] }).some(e => e.includes('duplicates')));
+  assert.ok(validateEvents({ version: 1, events: [{ type: 'exploded', provider: 'p', model: 'm' }] }).some(e => e.includes('type')));
+  assert.ok(validateEvents({ version: 1, events: [{ type: 'price-changed', provider: 'p' }] }).some(e => e.includes('provider and model')));
+});
+
+test('scoring is tier-weighted: the high tier values quality far more than the low tier', () => {
+  const free = { cost: { free: true }, contextK: 64, toolUse: 'basic', scores: { coding: 40, reasoning: 35, speed: 70 } };
+  const strong = { cost: { usdPerMInput: 1.5, usdPerMOutput: 6 }, contextK: 400, toolUse: 'good', scores: { coding: 85, reasoning: 82, speed: 55 } };
+  const lowDelta = scoreModel(strong, 'low').score - scoreModel(free, 'low').score;
+  const highDelta = scoreModel(strong, 'high').score - scoreModel(free, 'high').score;
+  assert.ok(highDelta > lowDelta, `high ${highDelta} should exceed low ${lowDelta}`);
+  assert.ok(scoreModel(strong, 'high').score > scoreModel(free, 'high').score);
+  assert.equal(scoreModel({ status: 'active' }, 'low').score, null); // nothing known
+});
+
+test('free-campaign end alone does not force a change: current model is re-scored and compared', () => {
+  const report = evaluateMaintenance({ routing, registry: baseRegistry(), agents, catalog, events });
+  const cheap = decide(report, 'qoder');
+  assert.equal(cheap.current.model, 'Qwen3.8-Flash');
+  assert.ok(cheap.trigger.some(t => t.includes('free-campaign-ended')), JSON.stringify(cheap.trigger));
+  // Post-campaign Qwen3.8-Flash (~$0.75/M) scores close to the paid alternatives under
+  // low-tier weights: advantage < margin -> keep. The event alone changed nothing.
+  assert.equal(cheap.decision, 'keep', cheap.reason);
+  assert.match(cheap.reason, /below margin/);
+  assert.equal(report.changes.length, 0);
+});
+
+test('a forced migration (removed model) produces a full proposal with rollback', () => {
+  const removed = [{ type: 'removed', provider: 'qoder', model: 'Qwen3.8-Flash', asOf: '2026-03-15' }];
+  const report = evaluateMaintenance({ routing, registry: baseRegistry(), agents, catalog, events: removed });
+  const change = report.changes.find(c => c.affected.backend === 'qoder');
+  assert.ok(change.changeReason.includes('removed'));
+  assert.deepEqual(change.affected.capabilities, ['cheap-code', 'orchestration']);
+  assert.deepEqual(change.affected.agents, ['orchestrator', 'scout', 'tester']);
+  assert.equal(change.currentModel.provider, 'qoder');
+  assert.equal(change.proposedModel.model, 'Qwen3.9-Flash'); // best eligible under low-tier weights
+  assert.match(change.expectedCostImpact, /\$0 \(free\) -> proposed ~\$/);
+  assert.match(change.expectedQualityImpact, /coding 62->70/);
+  assert.equal(change.rollback.method, 'manual-edit');
+  assert.deepEqual(change.rollback.previousBinding, { provider: 'qoder', model: 'Qwen3.8-Flash', thinking: 'low' });
+  assert.ok(change.rollback.steps.some(s => s.includes('models.local.json')));
+});
+
+test('a free-campaign end proposes a switch when a clearly better candidate exists', () => {
+  // Same event, but the cheap backend is bound to a weak model: the margin is exceeded.
+  const reg = baseRegistry();
+  const cat = structuredClone(catalog);
+  cat.models.find(m => m.model === 'Qwen3.8-Flash').scores = { coding: 30, reasoning: 25, speed: 60 };
+  const report = evaluateMaintenance({ routing, registry: reg, agents, catalog: cat, events });
+  const cheap = decide(report, 'qoder');
+  assert.equal(cheap.decision, 'propose', cheap.reason);
+  assert.equal(cheap.proposed.model, 'Qwen3.9-Flash');
+  assert.ok(report.changes.length >= 1);
+});
+
+test('a free-campaign end with a comparable successor keeps the current model', () => {
+  const cat = structuredClone(catalog);
+  cat.models.find(m => m.model === 'Qwen3.8-Flash').postCampaignCost = { usdPerMInput: 0.3, usdPerMOutput: 1.2 };
+  const report = evaluateMaintenance({ routing, registry: baseRegistry(), agents, catalog: cat, events });
+  // Qwen3.9-Flash (0.25/1.0, coding 70) vs post-campaign Qwen3.8 (0.3/1.2, coding 62):
+  // advantage is below the margin -> keep.
+  const cheap = decide(report, 'qoder');
+  assert.equal(cheap.decision, 'keep', cheap.reason);
+  assert.match(cheap.reason, /below margin/);
+  assert.equal(report.changes.length, 0);
+});
+
+test('margin is configurable and forced migration ignores it when the model is removed', () => {
+  const reg = baseRegistry();
+  const removed = { version: 1, events: [{ type: 'removed', provider: 'qoder', model: 'Qwen3.8-Flash', asOf: '2026-03-01' }] };
+  const report = evaluateMaintenance({ routing, registry: reg, agents, catalog, events: removed.events, margin: 90 });
+  const cheap = decide(report, 'qoder');
+  assert.equal(cheap.decision, 'propose'); // forced: removed model must move even with margin 90
+  assert.match(cheap.reason, /migration required/);
+});
+
+test('availability listing: a bound model absent from its provider listing is forced to migrate', () => {
+  const availability = { models: new Set(['openai-codex/gpt-5.6-sol', 'openai-codex/gpt-6-astra', 'openai-codex/gpt-5.5', 'openai-codex/gpt-5.6-luna']), providers: new Set(['openai-codex', 'qoder']), source: 'test' };
+  const report = evaluateMaintenance({ routing, registry: baseRegistry(), agents, catalog, events: [], availability });
+  const cheap = decide(report, 'qoder'); // qoder listed but Qwen3.8-Flash absent -> gone
+  assert.equal(cheap.decision, 'propose');
+  assert.ok(cheap.trigger.some(t => t.includes('not')));
+  // provider not in the listing at all -> unknown, never "missing"
+  const availability2 = { models: new Set(['openai-codex/gpt-5.6-sol']), providers: new Set(['openai-codex']), source: 'test' };
+  const report2 = evaluateMaintenance({ routing, registry: baseRegistry(), agents, catalog, events: [], availability: availability2 });
+  assert.equal(decide(report2, 'qoder').decision, 'keep');
+  assert.equal(decide(report2, 'local').decision, 'not-primary'); // fallback-only backend: reported, not evaluated
+});
+
+test('vision-requiring capability only proposes vision-capable candidates', () => {
+  const reg = baseRegistry();
+  const removed = [{ type: 'removed', provider: 'openai-codex', model: 'gpt-6-astra', asOf: '2026-03-01' }];
+  const report = evaluateMaintenance({ routing, registry: reg, agents, catalog, events: removed });
+  const astra = decide(report, 'astra');
+  assert.equal(astra.decision, 'insufficient-data'); // no other vision model in catalog
+  assert.match(astra.reason, /no eligible candidate/);
+});
+
+test('unbound/placeholder backends are skipped; fallback-only backends are reported not evaluated', () => {
+  const reg = mergeRegistries({ version: 1, backends: { qoder: { provider: 'TODO-provider', model: 'TODO-model' } } }, {
+    version: 1, backends: { codex: { provider: 'openai-codex', model: 'gpt-5.5' } },
+  });
+  const report = evaluateMaintenance({ routing, registry: reg, agents, catalog, events });
+  assert.equal(decide(report, 'qoder').decision, 'skip');
+  // codex is primary of no capability but fallback of strong-code/browser/orchestration
+  assert.equal(decide(report, 'codex').decision, 'not-primary');
+});
+
+test('model-added triggers comparison but keeps when the new model is not better', () => {
+  const added = [{ type: 'model-added', provider: 'qoder', model: 'Qwen3.9-Flash', asOf: '2026-03-05' }];
+  const report = evaluateMaintenance({ routing, registry: baseRegistry(), agents, catalog, events: added });
+  const cheap = decide(report, 'qoder');
+  // Qwen3.9-Flash is better than free Qwen3.8-Flash but not by the margin -> keep.
+  assert.equal(cheap.decision, 'keep', cheap.reason);
+  assert.ok(cheap.trigger.some(t => t.includes('Qwen3.9-Flash')));
+});
+
+test('no events and healthy catalog -> every bound primary keeps, zero changes', () => {
+  const report = evaluateMaintenance({ routing, registry: baseRegistry(), agents, catalog, events: [] });
+  assert.equal(report.changes.length, 0);
+  assert.deepEqual(report.decisions.filter(d => d.decision === 'keep').map(d => d.backend).sort(), ['astra', 'devin', 'qoder', 'sol']);
+  assert.equal(decide(report, 'local').decision, 'not-primary');
+  assert.equal(decide(report, 'codex').decision, 'not-primary');
+});
+
+test('parseModelList extracts provider/model pairs and tolerates noise', () => {
+  const { models, providers } = parseModelList('openai-codex/gpt-5.6-sol\nqoder / Qwen3.8-Flash\nnot a model line\n');
+  assert.ok(models.has('openai-codex/gpt-5.6-sol'));
+  assert.ok(models.has('qoder/Qwen3.8-Flash'));
+  assert.ok(providers.has('qoder'));
+});
+
+test('parseModelList reads the real `pi --list-models` table (space-separated columns, header skipped)', () => {
+  // Regression: the table format is `provider  model  context  max-out  thinking  images`,
+  // not `provider/model`. The old regex silently matched nothing -> probe reported null.
+  const table = 'provider      model              context  max-out  thinking  images\n'
+    + 'qoder         Qwen3.8-Flash        1M       131.1K   yes       yes\n'
+    + 'devin         swe-2-high           256K     128K     yes       yes\n'
+    + 'freetoken     Qwen3.6-35B-A3B-NVFP4 262.1K 4.1K     no        yes\n'
+    + 'openai-codex  gpt-5.6-luna         272K     128K     yes       yes\n';
+  const { models, providers } = parseModelList(table);
+  assert.ok(models.has('qoder/Qwen3.8-Flash'));
+  assert.ok(models.has('devin/swe-2-high'));
+  assert.ok(models.has('freetoken/Qwen3.6-35B-A3B-NVFP4'));
+  assert.ok(models.has('openai-codex/gpt-5.6-luna'));
+  assert.ok(!providers.has('provider')); // header row excluded
+});
+
+test('loadAvailabilityFile produces the same shape', () => {
+  const tmp = join(fx, 'availability.txt');
+  const a = loadAvailabilityFile(tmp);
+  assert.ok(a.providers.has('qoder'));
+  assert.ok(a.models.has('qoder/Qwen3.9-Flash'));
+});