@ludi-uni/ludi-agent-kit 0.1.0

package/tests/sqlite-concurrency.test.mjs

@@ -0,0 +1,178 @@
+// SQLite concurrency and exception-safe run termination: busy_timeout, two writers on
+// one state.db, injected exceptions, and no `running` zombie in list / latest-run.
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { spawn, spawnSync } from 'node:child_process';
+import { mkdtempSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, resolve, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { loadRouting } from '../lib/routing.mjs';
+import { loadAgents } from '../lib/agents.mjs';
+import { DEFAULT_POLICY } from '../lib/orchestrator/policy.mjs';
+import { openStore, BUSY_TIMEOUT_MS } from '../lib/orchestrator/store.mjs';
+import { orchestrate } from '../lib/orchestrator/orchestrator.mjs';
+
+const kit = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const routing = loadRouting(join(kit, 'routing/routing.json'));
+const { agents } = loadAgents(join(kit, 'agents'), routing);
+const REG = { version: 1, backends: {
+  local: { provider: 'pl', model: 'ml', thinking: 'off' },
+  cheap: { provider: 'pc', model: 'mc', thinking: 'low' },
+  sol: { provider: 'ps', model: 'ms', thinking: 'high' },
+  codex: { provider: 'px', model: 'mx', thinking: 'high' },
+} };
+const spec = (id, agent, dependencies = []) => ({ id, title: `task ${id}`, goal: `goal ${id}`, agent, dependencies, acceptance: ['done'] });
+const completed = task => ({ ok: true, structured: true, result: { status: 'completed', summary: `did ${task.id}`, artifacts: [], verification: [], acceptance: [{ id: 'A1', met: true, evidence: 'observed' }], remainingIssues: [], decisions: [], newTasks: [] } });
+const dbPath = () => join(mkdtempSync(join(tmpdir(), 'ludi-sql-')), 'state.db');
+const child = join(kit, 'tests/fixtures/orch-concurrent-child.mjs');
+const cli = join(kit, 'scripts/orchestrate.mjs');
+
+const spawnChild = args => new Promise(res => {
+  const p = spawn(process.execPath, [child, ...args], { encoding: 'utf8' });
+  let out = '', err = '';
+  p.stdout.on('data', d => { out += d; });
+  p.stderr.on('data', d => { err += d; });
+  p.on('close', code => res({ code, out, err, json: (() => { try { return JSON.parse(out.trim().split(/\r?\n/).at(-1)); } catch { return null; } })() }));
+});
+
+test('busy_timeout is set to a bounded value and WAL is kept', () => {
+  const session = openStore(dbPath());
+  const p = session.pragmas();
+  assert.equal(p.journalMode, 'wal');
+  assert.equal(p.busyTimeoutMs, BUSY_TIMEOUT_MS);
+  assert.ok(BUSY_TIMEOUT_MS >= 1000 && BUSY_TIMEOUT_MS <= 30000, 'bounded');
+  session.close();
+});
+
+test('concurrent writer: another process holding the write lock makes us wait (busy_timeout), not throw', async () => {
+  const path = dbPath();
+  const session = openStore(path);
+  const holdMs = 600;
+  // Separate process: BEGIN IMMEDIATE, hold for holdMs, COMMIT.
+  const holder = spawn(process.execPath, ['--input-type=module', '-e', `
+    import { DatabaseSync } from 'node:sqlite';
+    const db = new DatabaseSync(${JSON.stringify(path)});
+    db.exec('BEGIN IMMEDIATE');
+    db.prepare("INSERT INTO runs (id, request, status, created_at, updated_at, policy_snapshot, counters, scope_key) VALUES ('hold','h','completed','2000','2000','{}','{}','k')").run();
+    console.log('locked');
+    await new Promise(r => setTimeout(r, ${holdMs}));
+    db.exec('COMMIT'); db.close();
+  `], { encoding: 'utf8' });
+  await new Promise(res => holder.stdout.on('data', d => { if (String(d).includes('locked')) res(); }));
+  const started = Date.now();
+  const runId = session.createRun({ request: 'waiter', policy: DEFAULT_POLICY }); // would throw SQLITE_BUSY without busy_timeout
+  const waited = Date.now() - started;
+  assert.ok(waited >= 200, `expected to wait for the lock, waited ${waited}ms`);
+  assert.ok(waited < BUSY_TIMEOUT_MS, `waited ${waited}ms`);
+  assert.equal(session.getRun(runId).status, 'running');
+  await new Promise(res => holder.on('close', res));
+  assert.equal(session.listRuns().length, 2);
+  session.close();
+});
+
+test('without busy_timeout the same overlap throws SQLITE_BUSY (documents why the pragma matters)', async () => {
+  const path = dbPath();
+  openStore(path).close();
+  const { DatabaseSync } = await import('node:sqlite');
+  const raw = new DatabaseSync(path); // default busy_timeout = 0
+  const holder = spawn(process.execPath, ['--input-type=module', '-e', `
+    import { DatabaseSync } from 'node:sqlite';
+    const db = new DatabaseSync(${JSON.stringify(path)});
+    db.exec('BEGIN IMMEDIATE'); console.log('locked');
+    await new Promise(r => setTimeout(r, 400)); db.exec('COMMIT'); db.close();
+  `], { encoding: 'utf8' });
+  await new Promise(res => holder.stdout.on('data', d => { if (String(d).includes('locked')) res(); }));
+  assert.throws(() => raw.exec('BEGIN IMMEDIATE'), /SQLITE_BUSY|database is locked/i);
+  raw.close();
+  await new Promise(res => holder.on('close', res));
+});
+
+test('concurrent E2E: two orchestrate processes on one state.db both terminate, no running zombie', async () => {
+  const path = dbPath();
+  openStore(path).close(); // create schema once so both children start on an existing file
+  const [x, y] = await Promise.all([spawnChild([path, 'alpha', '4', '40']), spawnChild([path, 'beta', '4', '40'])]);
+  assert.equal(x.code, 0, x.err + x.out);
+  assert.equal(y.code, 0, y.err + y.out);
+  assert.equal(x.json.runStatus, 'completed');
+  assert.equal(y.json.runStatus, 'completed');
+  assert.notEqual(x.json.runId, y.json.runId);
+  const session = openStore(path);
+  const runs = session.listRuns();
+  assert.equal(runs.length, 2);
+  assert.deepEqual(runs.map(r => r.status), ['completed', 'completed']);
+  assert.ok(runs.every(r => r.completed === 4 && r.total === 4));
+  assert.equal(session.listRuns({ status: 'running' }).length, 0, 'no zombie');
+  for (const r of runs) assert.ok(session.loadTasks(r.id).every(t => t.status === 'completed'));
+  session.close();
+});
+
+test('concurrent E2E with an injected exception: the failing run ends `failed`, the other completes, latest-run is never a zombie', async () => {
+  const path = dbPath();
+  openStore(path).close();
+  const [x, y] = await Promise.all([spawnChild([path, 'alpha', '3', '40', 't2']), spawnChild([path, 'beta', '3', '40'])]);
+  assert.equal(x.code, 3, x.err + x.out);
+  assert.match(x.json.error, /injected failure at t2/);
+  assert.equal(x.json.persisted, true, 'run state was persisted on error');
+  assert.equal(y.code, 0, y.err + y.out);
+  const session = openStore(path);
+  const failed = session.getRun(x.json.runId);
+  assert.equal(failed.status, 'failed');
+  assert.ok(failed.counters.unresolved.some(u => /run aborted/.test(u)));
+  assert.ok(session.loadTrace(failed.id).some(e => e.type === 'error' && /injected/.test(e.message)));
+  assert.ok(session.loadTasks(failed.id).every(t => t.status !== 'running'));
+  assert.equal(session.getRun(y.json.runId).status, 'completed');
+  assert.equal(session.listRuns({ status: 'running' }).length, 0);
+  // list / latest-run: the top row is a terminal state, and --list from the CLI agrees.
+  const list = spawnSync(process.execPath, [cli, '--list', '--store', path], { encoding: 'utf8' });
+  assert.equal(list.status, 0, list.stderr);
+  assert.doesNotMatch(list.stdout, /\srunning\s/);
+  session.close();
+});
+
+test('in-process injected store exception mid-round: in-flight task and run are terminated, updated_at moves, transaction is released', async () => {
+  const path = dbPath();
+  const real = openStore(path);
+  let armed = false;
+  // Wrap the session: appendTrace throws once on the first `result` event, i.e. while
+  // the task is still `running` in the store.
+  const session = { ...real, appendTrace(runId, e) { if (e.type === 'result' && !armed) { armed = true; throw new Error('disk full (injected)'); } return real.appendTrace(runId, e); } };
+  let caught = null;
+  try {
+    await orchestrate({ request: 'boom', plan: [spec('a', 'scout'), spec('b', 'coder', ['a'])], agents, routing, registry: REG, policy: DEFAULT_POLICY, session, runner: { async run(t) { return completed(t); } } });
+  } catch (e) { caught = e; }
+  assert.ok(caught, 'orchestrate rethrows');
+  assert.match(caught.message, /disk full/);
+  assert.equal(caught.persisted, true);
+  const run = real.getRun(caught.runId);
+  assert.equal(run.status, 'failed');
+  assert.ok(run.updatedAt >= run.createdAt);
+  const tasks = real.loadTasks(run.id);
+  assert.equal(tasks.find(t => t.id === 'a').status, 'failed');
+  assert.match(tasks.find(t => t.id === 'a').blockedReason, /run aborted by error/);
+  assert.equal(tasks.find(t => t.id === 'b').status, 'pending');
+  assert.ok(real.loadTrace(run.id).some(e => e.type === 'error'));
+  // The connection is usable (no dangling BEGIN): a fresh run on the same session completes.
+  const ok = await orchestrate({ request: 'after', plan: [spec('a', 'scout')], agents, routing, registry: REG, policy: DEFAULT_POLICY, session: real, runner: { async run(t) { return completed(t); } } });
+  assert.equal(ok.runStatus, 'completed');
+  assert.equal(real.listRuns({ status: 'running' }).length, 0);
+  assert.equal(real.listRuns()[0].id, ok.runId, 'latest run is the completed one');
+  real.close();
+});
+
+test('store unwritable during termination: the error names the persistence limitation instead of hiding it', async () => {
+  const real = openStore(dbPath());
+  let phase = 0;
+  const session = { ...real,
+    appendTrace(runId, e) { if (e.type === 'result') { phase = 1; throw new Error('primary failure'); } if (phase === 1) throw new Error('store closed'); return real.appendTrace(runId, e); },
+    updateRun(id, patch) { if (phase === 1) throw new Error('store closed'); return real.updateRun(id, patch); },
+    saveTask(runId, t) { if (phase === 1) throw new Error('store closed'); return real.saveTask(runId, t); },
+  };
+  let caught = null;
+  try { await orchestrate({ request: 'boom', plan: [spec('a', 'scout')], agents, routing, registry: REG, policy: DEFAULT_POLICY, session, runner: { async run(t) { return completed(t); } } }); }
+  catch (e) { caught = e; }
+  assert.match(caught.message, /primary failure/);
+  assert.match(caught.message, /run state could not be persisted: store closed/);
+  assert.equal(caught.persisted, false);
+  real.close();
+});

package/tests/task-global-e2e.test.mjs

@@ -0,0 +1,63 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { resolve, dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { loadRouting } from '../lib/routing.mjs';
+import { loadAgents } from '../lib/agents.mjs';
+import { DEFAULT_POLICY, mergePolicy } from '../lib/orchestrator/policy.mjs';
+import { createAgentRunner } from '../lib/orchestrator/runner.mjs';
+import { orchestrate, formatReport } from '../lib/orchestrator/orchestrator.mjs';
+
+const root = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const routing = structuredClone(loadRouting(join(root, 'routing/routing.json')));
+routing.capabilities['cheap-code'].fallback = ['local'];
+routing.capabilities['strong-code'].fallback = ['qoder', 'codex', 'local'];
+const { agents } = loadAgents(join(root, 'agents'), routing);
+const registry = { version: 1, backends: {
+  qoder: { provider: 'qoder', model: 'Qwen3.8-Flash' },
+  local: { provider: 'freetoken', model: 'FreeToken' },
+  devin: { provider: 'devin', model: 'SWE-2' },
+  codex: { provider: 'codex', model: 'quota-0' },
+} };
+const policy = mergePolicy(DEFAULT_POLICY, { limits: { max_retries: 3, model_attempts_per_task: 2, max_total_attempts_per_task: 4 } });
+const task = { id: 't1', title: 'fixture', goal: 'complete fixture', agent: 'scout', dependencies: [], acceptance: ['done'] };
+
+test('Codex quota 0 E2E: cheap protocol failures remain excluded on strong-code; three invocations and pipeline continues', async () => {
+  const calls = [];
+  const health = { skip: c => c.provider === 'codex' ? 'quota-0' : null, report() {} };
+  const runner = createAgentRunner({ agents, routing, registry, policy, health,
+    invoke: async () => { throw new Error('unexpected oneshot'); },
+    runSubagent: async c => {
+      calls.push(c.modelId);
+      if (c.provider === 'qoder') return { ok: true, text: 'not structured', child: { toolCalls: 2, turns: 3 } };
+      if (c.provider === 'freetoken') return { ok: false, error: 'turn limit 12', failureClass: 'NO_PROGRESS_TIMEOUT',
+        telemetry: { text: '', toolCalls: 0, turns: 12 }, child: { toolCalls: 0, turns: 12, stopReason: 'no-progress-turn-limit' } };
+      if (c.provider === 'devin') return { ok: true, text: '```json\n{"status":"completed","summary":"done","acceptance":[{"id":"A1","met":true,"evidence":"verified"}]}\n```', child: { toolCalls: 1, turns: 2 } };
+      throw new Error(`unexpected invocation: ${c.modelId}`);
+    },
+  });
+  const result = await orchestrate({ request: 'fixture', plan: [task], agents, routing, registry, policy, runner, health });
+  const t = result.tasks[0];
+  assert.equal(result.status, 'completed');
+  assert.equal(t.status, 'completed');
+  assert.equal(t.totalModelAttempts, 3, JSON.stringify({ calls, trace: result.trace.filter(e => e.type === 'result' || e.type === 'retry' || e.type === 'fallback'), task: t }, null, 2));
+  assert.deepEqual(calls.map(id => id.split('/')[0]), ['qoder', 'freetoken', 'devin']);
+  assert.ok(t.taskGlobalFailedModels.some(id => id.startsWith('qoder/')));
+  assert.ok(t.taskGlobalFailedModels.some(id => id.startsWith('freetoken/')));
+  assert.deepEqual(t.capabilityLocalTriedModels, []);
+  const strong = result.trace.find(e => e.type === 'result' && e.capability === 'strong-code');
+  assert.equal(strong.counters.invocationsStarted, 1);
+  assert.equal(strong.counters.candidatesSkipped, 2);
+  assert.ok(strong.steps.some(s => s.reason === 'task-global-failed' && s.modelId.startsWith('qoder/')));
+  assert.ok(strong.steps.some(s => s.reason === 'task-global-failed' && s.modelId.startsWith('freetoken/')));
+  assert.ok(strong.steps.some(s => s.reason === 'quota-0' && s.modelId.startsWith('codex/')) === false, 'health candidate after success is not reached');
+  const failedChild = result.trace.find(e => e.type === 'child' && e.modelId.startsWith('freetoken/'));
+  assert.equal(failedChild.capability, 'cheap-code');
+  assert.equal(failedChild.provider, 'freetoken');
+  assert.equal(failedChild.toolCalls, 0);
+  assert.equal(failedChild.turns, 12);
+  assert.equal(failedChild.failureClass, 'NO_PROGRESS_TIMEOUT');
+  assert.equal(typeof failedChild.durationMs, 'number');
+  assert.match(formatReport(result), /devin\/SWE-2 on strong-code → completed/);
+  assert.match(formatReport(result), /task-global-failed skip/);
+});

package/tests/task-global-failed.test.mjs

@@ -0,0 +1,134 @@
+// Two-layer tried history: a model that protocol-fails is skipped for the WHOLE
+// task across capabilities, while the capability-local candidate list re-resolves.
+// Regression for run-muf8ula6-8509b1 (cheap-code Qwen re-invoked on strong-code).
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { resolve, dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { loadRouting } from '../lib/routing.mjs';
+import { loadAgents } from '../lib/agents.mjs';
+import { DEFAULT_POLICY, mergePolicy } from '../lib/orchestrator/policy.mjs';
+import { createAgentRunner } from '../lib/orchestrator/runner.mjs';
+import { orchestrate } from '../lib/orchestrator/orchestrator.mjs';
+import { shouldMarkTaskGlobalFailure } from '../lib/orchestrator/failures.mjs';
+import { withEscalation } from '../lib/pipeline.mjs';
+import { createScriptedInvoker } from '../adapters/pi/lib/invoke.mjs';
+
+const kit = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const routing = loadRouting(join(kit, 'routing/routing.json'));
+const { agents } = loadAgents(join(kit, 'agents'), routing);
+// cheap-code: pc/mc -> pl/ml -> ps/ms ; strong-code: devin(pd/md) -> pc/mc -> ps/ms -> px/mx -> pl/ml
+const REG = { version: 1, backends: {
+  local: { provider: 'pl', model: 'ml', thinking: 'off' },
+  cheap: { provider: 'pc', model: 'mc', thinking: 'low' },
+  sol:   { provider: 'ps', model: 'ms', thinking: 'high' },
+  codex: { provider: 'px', model: 'mx', thinking: 'high' },
+  devin: { provider: 'pd', model: 'md', thinking: 'high' },
+} };
+const policyWith = over => mergePolicy(DEFAULT_POLICY, over ?? {});
+const jsonReply = obj => `notes\n\n\`\`\`json\n${JSON.stringify(obj)}\n\`\`\``;
+const okReply = jsonReply({ status: 'completed', summary: 'done', acceptance: [{ id: 'A1', met: true, evidence: 'e' }] });
+const spec = (id, agent, dependencies = [], extra = {}) => ({ id, title: `task ${id}`, goal: `goal ${id}`, agent, dependencies, acceptance: ['done'], ...extra });
+const healthSkip = (skipIds) => ({ skip: c => (skipIds.includes(c.modelId) ? 'usage_exhausted' : null), report() {} });
+
+async function runScenario(script, { policy: over = {}, health = null } = {}) {
+  const policy = policyWith(over);
+  const calls = [];
+  const invoke = createScriptedInvoker(script, calls);
+  const runner = createAgentRunner({ invoke, agents, routing, registry: REG, policy, health });
+  const r = await orchestrate({ request: 'r', plan: [spec('a', 'scout')], agents, routing, registry: REG, policy, runner });
+  return { r, calls };
+}
+const count = (calls, prefix) => calls.filter(c => c.modelId.startsWith(prefix)).length;
+
+// A: cheap-code Qwen malformed -> added to taskGlobalFailedModels.
+test('A: protocol failure records the model in taskGlobalFailedModels', async () => {
+  const { r } = await runScenario({ '*': req => req.modelId.startsWith('pc/') ? 'Done' : okReply });
+  const t = r.tasks[0];
+  // pc/mc failed malformed on attempt 1; task should record it globally.
+  assert.ok((t.taskGlobalFailedModels ?? []).includes('pc/mc:low'), `got ${JSON.stringify(t.taskGlobalFailedModels)}`);
+});
+
+// B: strong-code escalation -> Qwen present in candidate list but skipped task-global.
+test('B: after escalation a task-global-failed model is skipped, not re-invoked', async () => {
+  // pc/mc (cheap primary) malformed; on strong-code pc/mc is also a candidate.
+  // devin (pd/md) must run first; pc/mc must NOT be re-invoked.
+  const { r, calls } = await runScenario({ '*': req => req.modelId.startsWith('pd/') ? okReply : 'Done' },
+    { policy: { limits: { max_retries: 4, model_attempts_per_task: 4, max_total_attempts_per_task: 8 } } });
+  assert.equal(r.status, 'completed');
+  assert.equal(count(calls, 'pc/'), 1, `pc/mc invoked ${count(calls,'pc/')}x — should be exactly once (cheap only)`);
+  assert.ok(count(calls, 'pd/') >= 1, 'devin invoked on strong-code');
+});
+
+// C: no-progress turn-limit TIMEOUT on FreeToken -> not re-invoked on strong-code.
+test('C: no-progress turn-limit marks task-global; not re-invoked after escalation', async () => {
+  assert.equal(shouldMarkTaskGlobalFailure('TIMEOUT', { toolCalls: 0 }), false, 'missing progress evidence must not be guessed');
+  assert.equal(shouldMarkTaskGlobalFailure('TIMEOUT', { toolCalls: 0, turnLimit: true, structuredProgress: false, hasFinalOutput: false }), true);
+  const calls = [];
+  const policy = policyWith({ limits: { max_retries: 4, model_attempts_per_task: 4, max_total_attempts_per_task: 8 } });
+  const runner = createAgentRunner({ invoke: async () => { throw new Error('oneshot used'); }, runSubagent: async req => {
+    calls.push({ modelId: req.modelId });
+    return req.modelId.startsWith('pd/') ? { ok: true, text: okReply, child: { toolCalls: 1, turns: 1 } } :
+      { ok: false, error: 'turn limit 12', failureClass: 'NO_PROGRESS_TIMEOUT', telemetry: { text: '', toolCalls: 0, turns: 12 }, child: { toolCalls: 0, turns: 12, stopReason: 'no-progress-turn-limit' } };
+  }, agents, routing, registry: REG, policy });
+  const r = await orchestrate({ request: 'r', plan: [spec('a', 'scout')], agents, routing, registry: REG, policy, runner });
+  // pl/ml (freetoken-analog) failed once on cheap; must not re-run on strong.
+  assert.ok(count(calls, 'pl/') <= 1, `pl/ml invoked ${count(calls,'pl/')}x`);
+});
+
+// D: transient tool failure does NOT mark task-global; same model may retry.
+test('D: transient tool failure stays eligible after a capability change', async () => {
+  assert.equal(shouldMarkTaskGlobalFailure('TOOL_FAILURE'), false);
+  assert.equal(shouldMarkTaskGlobalFailure('TEST_FAILURE'), false);
+  const called = [];
+  const localRouting = structuredClone(routing);
+  localRouting.capabilities['strong-code'].fallback = ['cheap'];
+  const result = await withEscalation({ routing: localRouting, registry: REG, capability: 'strong-code', agent: 'scout', pack: {}, trace: [], maxAttempts: 2,
+    excludeModels: [], taskGlobalFailedModels: [], fn: async c => {
+      called.push(c.modelId);
+      return { ok: c.modelId.startsWith('pc/'), reason: 'recoverable tool failure on cheap-code must not exclude this model' };
+    },
+  });
+  assert.equal(result.ok, true);
+  assert.deepEqual(called.map(id => id.split('/')[0]), ['pd', 'pc']);
+});
+
+// E: task-global skip consumes zero attempt budget.
+test('E: task-global-failed skip does not consume invocationsStarted', async () => {
+  const { r, calls } = await runScenario({ '*': req => req.modelId.startsWith('pd/') ? okReply : 'Done' },
+    { policy: { limits: { max_retries: 4, model_attempts_per_task: 4, max_total_attempts_per_task: 8 } } });
+  assert.equal(r.status, 'completed');
+  // pc/mc invoked once on cheap; on strong-code it is skipped (not invoked again).
+  assert.equal(count(calls, 'pc/'), 1);
+  // total real invocations = cheap pc/mc + pl/ml + ps/ms + devin (bounded)
+  assert.ok(calls.length <= 5, `invocations ${calls.length}`);
+});
+
+// F: Qwen + FreeToken failed on cheap -> strong-code Devin is invoked first.
+test('F: cheap failures -> strong-code Devin is the first invoked candidate', async () => {
+  const { r, calls } = await runScenario({ '*': req => req.modelId.startsWith('pd/') ? okReply : 'Done' },
+    { policy: { limits: { max_retries: 4, model_attempts_per_task: 4, max_total_attempts_per_task: 8 } } });
+  assert.equal(r.status, 'completed');
+  const strongIdx = calls.findIndex(c => c.modelId.startsWith('pd/'));
+  assert.ok(strongIdx >= 0, 'devin invoked');
+  // After escalation, devin should be invoked before any repeated cheap model.
+  const afterEscalation = calls.slice(strongIdx);
+  assert.ok(afterEscalation[0].modelId.startsWith('pd/'), 'devin first on strong-code');
+});
+
+// G: protocol failure still leaves modelId in the audit/child record.
+test('G: protocol failure keeps modelId in the result/child audit', async () => {
+  const { r } = await runScenario({ '*': req => req.modelId.startsWith('pc/') ? 'Done' : okReply });
+  const resultEvent = r.trace.find(e => e.type === 'result');
+  // invokedModels records which models were actually called this attempt.
+  assert.ok((resultEvent.invokedModels ?? []).includes('pc/mc:low'));
+});
+
+// H: autoDecisions/report show the strong-code progression (capability+model+failure).
+test('H: auto decision reports capability + modelId + failureClass', async () => {
+  const { r } = await runScenario({ '*': req => req.modelId.startsWith('pd/') ? okReply : 'Done' },
+    { policy: { limits: { max_retries: 4, model_attempts_per_task: 4, max_total_attempts_per_task: 8 } } });
+  const decisions = r.autoDecisions.map(d => d.choice).join(' | ');
+  assert.match(decisions, /on cheap-code|escalate to strong-code/, decisions);
+  assert.match(decisions, /MALFORMED_RESULT|TIMEOUT|→/, decisions);
+});

package/tests/telemetry.test.mjs

@@ -0,0 +1,173 @@
+// Phase 6: shadow mode, telemetry aggregation, calibration warnings, counterfactuals,
+// policy calibration proposal, retention. Cases A–L. Temp dirs only.
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtempSync, rmSync, writeFileSync, readFileSync, existsSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { resolve, dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import {
+  emptyTelemetry, recordRun, persistRun, loadTelemetry, calibrationWarnings,
+  counterfactuals, calibrationProposal, compactTelemetry, telemetryPaths,
+} from '../lib/telemetry.mjs';
+import { runMaintenanceJob } from '../lib/job.mjs';
+import { DEFAULT_POLICY } from '../lib/maintenance-exec.mjs';
+import { loadRouting } from '../lib/routing.mjs';
+import { loadAgents } from '../lib/agents.mjs';
+import { mergeRegistries } from '../lib/registry.mjs';
+
+const kit = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const routing = loadRouting(join(kit, 'routing/routing.json'));
+const { agents } = loadAgents(join(kit, 'agents'), routing);
+const tmp = () => mkdtempSync(join(tmpdir(), 'tel-'));
+
+const CAT = { version: 1, models: [
+  { provider: 'cloudp', model: 'cheap-ok', status: 'active', cost: { usdPerMInput: 0.2, usdPerMOutput: 0.8 }, contextK: 256, toolUse: 'good', location: 'cloud', scores: { coding: 68, reasoning: 62, speed: 75 } },
+  { provider: 'cloudp', model: 'vis-ok', status: 'active', cost: { usdPerMInput: 2, usdPerMOutput: 8 }, contextK: 400, vision: true, toolUse: 'good', location: 'cloud', scores: { coding: 80, reasoning: 82, speed: 55 } },
+] };
+const REG = mergeRegistries(null, { version: 1, backends: { cheap: { provider: 'cloudp', model: 'cheap-ok' }, astra: { provider: 'cloudp', model: 'vis-ok' } } });
+const base = dir => ({ outDir: dir, adapterDir: '.', kit: '.', catalog: CAT, routing, registry: REG, agents, policy: DEFAULT_POLICY });
+const manualObs = (dir, observations) => { const p = join(dir, 'o.json'); writeFileSync(p, JSON.stringify({ version: 1, observations })); return p; };
+const OBS = (p, m, changes, over = {}) => ({ provider: p, model: m, observedAt: '2026-03-05T00:00:00Z', source: { type: 'manual', trust: 'manual_verified' }, changes, confidence: 0.9, ...over });
+
+// --- A: shadow mode -> no external notification ------------------------------
+test('A: shadow mode suppresses the external notify-command', async () => {
+  const dir = tmp();
+  const input = manualObs(dir, [OBS('newp', 'Nova-1', { status: 'active' })]);
+  let cmdRan = false;
+  const run = await runMaintenanceJob({ ...base(dir), source: 'manual', input, shadow: true, notifyCommand: `node -e "require('fs').writeFileSync('${join(dir, 'cmd-ran').replace(/\\/g, '\\\\')}','1')"` });
+  assert.equal(run.shadow, true);
+  assert.equal(run.notification.sent, true); // decision still made + file sink
+  assert.ok(run.notification.shadowSuppressed);
+  assert.equal(existsSync(join(dir, 'cmd-ran')), false);
+  assert.ok(existsSync(join(dir, 'model-maintenance.notification.json'))); // file sink still wrote
+  rmSync(dir, { recursive: true, force: true });
+});
+
+test('A2: --shadow --shadow-notify DOES run the command', async () => {
+  const dir = tmp();
+  const input = manualObs(dir, [OBS('newp', 'Nova-1', { status: 'active' })]);
+  const marker = join(dir, 'cmd-ran').replace(/\\/g, '\\\\');
+  const run = await runMaintenanceJob({ ...base(dir), source: 'manual', input, shadow: true, shadowNotify: true, notifyCommand: `node -e "require('fs').writeFileSync('${marker}','1')"` });
+  assert.equal(existsSync(join(dir, 'cmd-ran')), true);
+  rmSync(dir, { recursive: true, force: true });
+});
+
+// --- B: telemetry aggregation -------------------------------------------------
+test('B: telemetry aggregates runs, costs, severities, decisions', async () => {
+  const dir = tmp();
+  const input = manualObs(dir, [OBS('newp', 'Nova-1', { status: 'active' })]);
+  await runMaintenanceJob({ ...base(dir), source: 'manual', input });
+  await runMaintenanceJob(base(dir)); // second run -> deduped notification
+  const t = loadTelemetry(dir);
+  assert.equal(t.runs, 2);
+  assert.equal(t.meaningfulRuns >= 1, true);
+  assert.ok(t.notifications >= 1);
+  assert.ok(t.dedupedNotifications >= 1);
+  assert.ok(t.cost.totalUsd > 0);
+  assert.ok(t.decisions.length >= 1);
+  assert.ok(t.decisions[0].model);
+  rmSync(dir, { recursive: true, force: true });
+});
+
+// --- C: mostly quiet -> no noise warning --------------------------------------
+test('C: quiet-dominated history produces no too-noisy warning', () => {
+  const t = emptyTelemetry();
+  for (let i = 0; i < 20; i++) recordRun(t, { runId: `r${i}`, quiet: true, completedAt: '2026-03-01' });
+  recordRun(t, { runId: 'm1', quiet: false, completedAt: '2026-03-02', notification: { severity: 'info', sent: true } });
+  const w = calibrationWarnings(t, DEFAULT_POLICY);
+  assert.equal(w.filter(x => x.kind === 'too-noisy').length, 0);
+});
+
+// --- D: notification-heavy -> noise warning ------------------------------------
+test('D: notification-per-meaningful > 0.8 -> too-noisy warning', () => {
+  const t = emptyTelemetry();
+  for (let i = 0; i < 5; i++) recordRun(t, { runId: `m${i}`, quiet: false, completedAt: '2026-03-01', notification: { severity: 'info', sent: true } });
+  const w = calibrationWarnings(t, DEFAULT_POLICY);
+  assert.ok(w.some(x => x.kind === 'too-noisy'));
+});
+
+// --- E: premium escalation heavy -> warning ------------------------------------
+test('E: frequent reconfigure escalations -> too-many-premium-escalations', () => {
+  const t = emptyTelemetry();
+  for (let i = 0; i < 6; i++) recordRun(t, { runId: `r${i}`, quiet: false, completedAt: '2026-03-01', escalation: { escalationReason: 'structural', targetTier: 'reconfigure' } });
+  const w = calibrationWarnings(t, DEFAULT_POLICY);
+  assert.ok(w.some(x => x.kind === 'too-many-premium-escalations'));
+});
+
+// --- F: fallback heavy -> warning ----------------------------------------------
+test('F: frequent fallbacks -> too-many-fallbacks warning', () => {
+  const t = emptyTelemetry();
+  for (let i = 0; i < 6; i++) recordRun(t, { runId: `r${i}`, quiet: false, completedAt: '2026-03-01', tiers: [{ role: 'monitor', fallbackOccurred: true, selected: { model: 'p/m', location: 'cloud', quality: 70, effectiveCostUsd: 0.001 }, requiredQuality: 40, candidates: [] }] });
+  const w = calibrationWarnings(t, DEFAULT_POLICY);
+  assert.ok(w.some(x => x.kind === 'too-many-fallbacks'));
+});
+
+// --- G: small quality margins -> warning ---------------------------------------
+test('G: chronically small quality margins -> quality-margin-too-small', () => {
+  const t = emptyTelemetry();
+  for (let i = 0; i < 6; i++) recordRun(t, { runId: `r${i}`, quiet: false, completedAt: '2026-03-01', tiers: [{ role: 'evaluate', selected: { model: 'p/m', location: 'cloud', quality: 66, effectiveCostUsd: 0.001 }, requiredQuality: 65, candidates: [] }] });
+  const w = calibrationWarnings(t, DEFAULT_POLICY);
+  assert.ok(w.some(x => x.kind === 'quality-margin-too-small'));
+});
+
+// --- H: below minimum data -> insufficient-observation-data --------------------
+test('H: too few runs -> calibration proposal withheld', () => {
+  const t = emptyTelemetry();
+  for (let i = 0; i < 3; i++) recordRun(t, { runId: `r${i}`, quiet: true, completedAt: '2026-03-01' });
+  const p = calibrationProposal(t, DEFAULT_POLICY);
+  assert.equal(p.status, 'insufficient-observation-data');
+  assert.equal(p.proposals.length, 0);
+});
+
+// --- I: enough data -> calibration proposals ------------------------------------
+test('I: sufficient history -> policy calibration proposal emitted', () => {
+  const t = emptyTelemetry();
+  for (let i = 0; i < 25; i++) recordRun(t, { runId: `r${i}`, quiet: i < 20, completedAt: '2026-03-01', notification: i < 20 ? null : { severity: 'info', sent: true } });
+  const p = calibrationProposal(t, DEFAULT_POLICY);
+  assert.equal(p.status, 'ok');
+  assert.ok(p.proposals.length >= 1);
+  assert.ok(p.proposals.every(x => 'currentValue' in x && 'proposedValue' in x && 'evidence' in x && 'expectedEffect' in x && 'confidence' in x));
+});
+
+// --- J: counterfactual comparison ----------------------------------------------
+test('J: counterfactuals compare selected vs alternatives on cost/quality', () => {
+  const t = emptyTelemetry();
+  recordRun(t, { runId: 'r1', quiet: false, completedAt: '2026-03-01', tiers: [{ role: 'monitor', selected: { model: 'p/mid', location: 'cloud', quality: 70, effectiveCostUsd: 0.002 }, requiredQuality: 40, ordered: [{ model: 'p/mid' }, { model: 'p/cheap', effectiveCostUsd: 0.001, quality: 65 }, { model: 'p/pro', effectiveCostUsd: 0.01, quality: 90 }], candidates: [] }] });
+  const cf = counterfactuals(t);
+  assert.equal(cf.length, 1);
+  const alts = cf[0].counterfactual;
+  assert.equal(alts.find(a => a.model === 'p/cheap').verdict, 'cheaper-lower-quality');
+  assert.equal(alts.find(a => a.model === 'p/pro').verdict, 'better-more-expensive');
+});
+
+// --- K: retention compaction preserves totals -----------------------------------
+test('K: compaction folds old runs but keeps summary + recent data', async () => {
+  const dir = tmp();
+  const { runs } = telemetryPaths(dir);
+  const old = { runId: 'old', quiet: true, completedAt: new Date(Date.now() - 40 * 86400e3).toISOString() };
+  const recent = { runId: 'new', quiet: true, completedAt: new Date().toISOString() };
+  writeFileSync(runs, JSON.stringify(old) + '\n' + JSON.stringify(recent) + '\n');
+  const r = compactTelemetry(dir, { days: 30 });
+  assert.equal(r.compacted, 1);
+  const lines = readFileSync(runs, 'utf8').split(/\r?\n/).filter(Boolean);
+  assert.ok(lines.some(l => l.includes('_compactedSummary')));
+  assert.ok(lines.some(l => l.includes('"new"')));
+  assert.ok(!lines.some(l => l.includes('"old"') && !l.includes('_compactedSummary')));
+  rmSync(dir, { recursive: true, force: true });
+});
+
+// --- L: calibration proposal does not touch real policy -------------------------
+test('L: writing a calibration proposal leaves the real policy file unchanged', async () => {
+  const dir = tmp();
+  const policyPath = join(dir, 'maintenance-policy.json');
+  writeFileSync(policyPath, JSON.stringify({ version: 1, requiredQuality: { evaluate: 65 } }));
+  const before = readFileSync(policyPath, 'utf8');
+  const t = emptyTelemetry();
+  for (let i = 0; i < 25; i++) recordRun(t, { runId: `r${i}`, quiet: i < 20, completedAt: '2026-03-01', notification: i < 20 ? null : { severity: 'info', sent: true } });
+  const p = calibrationProposal(t, DEFAULT_POLICY);
+  writeFileSync(join(dir, 'maintenance-policy.calibration.proposal.json'), JSON.stringify(p));
+  assert.equal(readFileSync(policyPath, 'utf8'), before);
+  assert.equal(p.status, 'ok');
+  rmSync(dir, { recursive: true, force: true });
+});

package/tests/test-sync-pi.ps1

@@ -0,0 +1,56 @@
+#requires -Version 7.0
+# Exercises adapters/pi/sync-pi.ps1 against an isolated temporary agent dir.
+# Dry-run must write nothing to the target; -Apply must create only the planned entries and never touch settings/auth.
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+$kit = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..'))
+$root = Join-Path ([IO.Path]::GetTempPath()) ('ludi-kit-sync-' + [guid]::NewGuid().ToString('N').Substring(0, 8))
+$agent = Join-Path $root 'agent'
+New-Item -ItemType Directory -Path $agent | Out-Null
+$settings = '{"defaultProvider":"fixture","defaultModel":"fixture-model","defaultTools":["read"]}'
+[IO.File]::WriteAllText((Join-Path $agent 'settings.json'), $settings)
+[IO.File]::WriteAllText((Join-Path $agent 'auth.json'), '{"fixture":"not-a-real-secret"}')
+[IO.File]::WriteAllText((Join-Path $agent 'AGENTS.md'), '# user instructions')
+$sync = Join-Path $kit 'adapters/pi/sync-pi.ps1'
+
+# 1. dry-run: nothing changes in target
+$before = Get-ChildItem $agent -Recurse -Force | Select-Object -ExpandProperty FullName | Sort-Object
+& pwsh -NoProfile -File $sync -AgentDir $agent | Out-Null
+if ($LASTEXITCODE -ne 0) { throw 'dry-run failed' }
+$after = Get-ChildItem $agent -Recurse -Force | Select-Object -ExpandProperty FullName | Sort-Object
+if (($before -join '|') -cne ($after -join '|')) { throw 'dry-run modified the agent directory' }
+$plan = Get-Content (Join-Path $kit 'adapters/pi/out/plan.json') -Raw | ConvertFrom-Json
+if ($plan.mode -ne 'dry-run') { throw 'plan mode should be dry-run' }
+if (-not ($plan.plan | Where-Object { $_.entry -eq 'AGENTS.md' -and $_.state -eq 'conflict-existing' })) { throw 'existing AGENTS.md should be reported as conflict' }
+if (-not ($plan.plan | Where-Object { $_.entry -eq 'skills/visual-verification' -and $_.state -eq 'create' })) { throw 'skill link should be planned' }
+if (-not (Test-Path (Join-Path $kit 'adapters/pi/out/AGENTS.md'))) { throw 'generated AGENTS.md missing' }
+
+# 2. apply without -BackupConflicts must refuse and change nothing
+$failed = $false
+try { & pwsh -NoProfile -File $sync -AgentDir $agent -Apply 2>$null | Out-Null; if ($LASTEXITCODE -ne 0) { $failed = $true } } catch { $failed = $true }
+if (-not $failed) { throw 'apply with conflicts should fail' }
+if ((Get-Content (Join-Path $agent 'AGENTS.md') -Raw) -cne '# user instructions') { throw 'conflicting AGENTS.md was modified' }
+
+# 3. apply with -BackupConflicts: junctions + AGENTS, backup retained, settings/auth untouched
+& pwsh -NoProfile -File $sync -AgentDir $agent -Apply -BackupConflicts | Out-Null
+if ($LASTEXITCODE -ne 0) { throw 'apply failed' }
+foreach ($rel in 'skills/visual-verification', 'skills/project-management', 'skills/pi-workflow', 'agents/ludi-agent-kit', 'extensions/ludi-agent-kit', 'extensions/ludi-orchestrator') {
+    $item = Get-Item -LiteralPath (Join-Path $agent $rel) -Force
+    if ($item.LinkType -ne 'Junction') { throw "$rel is not a Junction" }
+}
+if ((Get-Content (Join-Path $agent 'AGENTS.md') -Raw) -notmatch '^<!-- Generated by ludi-agent-kit') { throw 'AGENTS.md not generated' }
+$backups = @(Get-ChildItem (Join-Path $agent 'ludi-agent-kit') -Directory -Filter 'backup-*')
+if ($backups.Count -ne 1 -or -not (Test-Path (Join-Path $backups[0].FullName 'AGENTS.md'))) { throw 'AGENTS.md backup missing' }
+if ([IO.File]::ReadAllText((Join-Path $agent 'settings.json')) -cne $settings) { throw 'settings.json modified' }
+if ([IO.File]::ReadAllText((Join-Path $agent 'auth.json')) -cne '{"fixture":"not-a-real-secret"}') { throw 'auth.json modified' }
+
+# 4. repeat apply is a no-op
+& pwsh -NoProfile -File $sync -AgentDir $agent -Apply | Out-Null
+if ($LASTEXITCODE -ne 0) { throw 'repeat apply failed' }
+$plan = Get-Content (Join-Path $kit 'adapters/pi/out/plan.json') -Raw | ConvertFrom-Json
+if ($plan.plan | Where-Object { $_.state -ne 'ok' }) { throw 'repeat apply should report all ok' }
+
+# cleanup: remove junctions (links only) then fixture
+foreach ($j in Get-ChildItem $agent -Recurse -Force -Attributes ReparsePoint) { $j.Delete() }
+Remove-Item $root -Recurse -Force
+Write-Output 'PASS: sync-pi dry-run writes nothing; apply is bounded, backed up and idempotent; settings/auth untouched'