@north-light/crouter 0.3.154

package/dist/core/__tests__/respawn-throttle.test.js

@@ -0,0 +1,154 @@
+// Run with: node --import tsx/esm --test src/core/__tests__/respawn-throttle.test.ts
+//
+// The event-driven respawn throttle (design flow B) — the replacement for the
+// deleted stranded-relaunch counters. Each respawn action consults the node's
+// consecutive short-lived-exit count: attempt n defers min(5s·2^(n-1), 10min)
+// via a REAL backoff timer that re-enqueues a fresh policy job; at
+// MAX_RESPAWN_ATTEMPTS (6) the loop terminalizes (transition('crash') +
+// surfaceBootFailure + doctrine wake, counter cleared). An exit with uptime ≥
+// HEALTHY_UPTIME_MS (60s) resets the counter BEFORE classification, restoring
+// the immediate (sub-second) respawn.
+//
+// The backoff timers are real setTimeout — only the throttle's uptime clock is
+// injected — so this file asserts the DEFERRAL (the emitted
+// `broker.respawn.deferred` event + no launch), never timer maturity: the
+// shortest deferral is 5s, well past this file's lifetime, and dispose()
+// cancels every pending timer.
+import { test, before, after } from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtempSync, readFileSync, rmSync, mkdirSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { createNode, getNode, subscribe } from '../canvas/canvas.js';
+import { closeDb } from '../canvas/db.js';
+import { headlessBrokerHost } from '../runtime/host.js';
+import { readInboxSince } from '../feed/inbox.js';
+import { bindDaemonEventSource, resetEventSourceForTests } from '../events/source.js';
+import { resetFleetForTests } from '../runtime/fleet.js';
+import { HEALTHY_UPTIME_MS } from '../../daemon/fleet.js';
+import { bindTestFleet } from './helpers/fleet.js';
+let home;
+let tf;
+let nowMs = 5_000_000;
+function node(id, over = {}) {
+    return {
+        node_id: id,
+        name: id,
+        created: new Date().toISOString(),
+        cwd: home,
+        kind: 'developer',
+        mode: 'base',
+        lifecycle: 'terminal',
+        status: 'active',
+        ...over,
+    };
+}
+/** Every daemon event this process emitted, oldest first. */
+function events() {
+    let raw;
+    try {
+        raw = readFileSync(join(home, 'crtrd.log'), 'utf8');
+    }
+    catch {
+        return [];
+    }
+    return raw
+        .split('\n')
+        .filter((l) => l.trim() !== '')
+        .map((l) => JSON.parse(l));
+}
+before(() => {
+    home = mkdtempSync(join(tmpdir(), 'crtr-respawn-throttle-'));
+    mkdirSync(home, { recursive: true });
+    process.env['CRTR_HOME'] = home;
+    resetEventSourceForTests();
+    bindDaemonEventSource(); // events land in <home>/crtrd.log, like the daemon's
+    tf = bindTestFleet({ now: () => nowMs });
+});
+after(() => {
+    tf.fleet.dispose(); // cancels every pending real backoff timer
+    resetFleetForTests();
+    resetEventSourceForTests();
+    closeDb();
+    rmSync(home, { recursive: true, force: true });
+    delete process.env['CRTR_HOME'];
+});
+// ===========================================================================
+// Short-lived exits 1-5 defer with doubling backoff; the 6th terminalizes.
+// ===========================================================================
+test('consecutive short-lived exits defer respawn on a doubling backoff, then terminalize at the cap', async () => {
+    // A booted row (saved session, no crash markers) — table row 6, respawn-resume.
+    createNode(node('T', { pi_session_id: 'sess-T' }));
+    // Five synthetic exits (no known launch time → each counts as short-lived).
+    for (let i = 0; i < 5; i++) {
+        tf.fleet.deliverExit('T', { code: 1, signal: null });
+        await tf.drain();
+    }
+    const deferred = events().filter((e) => e.event === 'broker.respawn.deferred');
+    assert.deepEqual(deferred.map((e) => e.fields?.['backoff_ms']), [5_000, 10_000, 20_000, 40_000, 80_000], 'attempts 1-5 defer on the doubling 5s·2^(n-1) schedule');
+    assert.deepEqual(deferred.map((e) => e.fields?.['consecutive_failures']), [1, 2, 3, 4, 5], 'the counter grows once per short-lived exit');
+    assert.equal(getNode('T').status, 'active', 'the node stays revivable while attempts remain');
+    assert.equal(events().some((e) => e.event === 'broker.respawn.started'), false, 'no launch fires while every attempt is deferred');
+    // The 6th short-lived exit exhausts the budget: crash + surface, no respawn.
+    tf.fleet.deliverExit('T', { code: 1, signal: null });
+    await tf.drain();
+    assert.equal(getNode('T').status, 'dead', 'the respawn loop terminalizes at MAX_RESPAWN_ATTEMPTS');
+    assert.equal(events().some((e) => e.event === 'broker.respawn.exhausted'), true, 'exhaustion is surfaced as its own event');
+});
+// ===========================================================================
+// A healthy-uptime exit resets the counter: the next respawn is immediate.
+// ===========================================================================
+test('a pre-launch respawn refusal terminalizes the stranded node and wakes its parent', async () => {
+    const missingCwd = join(home, 'reaped-worktree');
+    createNode(node('P', { lifecycle: 'resident' }));
+    createNode(node('R', { cwd: missingCwd, parent: 'P', pi_session_id: 'sess-R' }));
+    subscribe('P', 'R', true);
+    // A healthy real entry makes this exit respawn immediately (rather than
+    // waiting for the throttle), reaching reviveNode's cwd preflight through
+    // DaemonFleet.#respawn.
+    nowMs = 10_000_000;
+    tf.fleet.register('R', { pid: 424_244, exited: new Promise(() => { }) });
+    nowMs += HEALTHY_UPTIME_MS;
+    tf.fleet.deliverExit('R', { code: 1, signal: null });
+    await tf.drain();
+    assert.equal(getNode('R').status, 'dead', 'a pre-launch refusal cannot strand an active node without a fleet entry');
+    const inbox = readInboxSince('P');
+    assert.equal(inbox.length, 2, 'the terminal outcome surfaces through both the urgent failure and doctrine wake');
+    const urgent = inbox.find((entry) => entry.tier === 'urgent');
+    assert.ok(urgent, 'the parent receives the recovery-refusal surface');
+    assert.match(urgent.label, /Recovery failed/, 'the surface describes the refusal instead of falsely claiming a fresh boot failed');
+    assert.ok(inbox.some((entry) => entry.tier === 'normal'), 'the parent receives the doctrine wake');
+});
+test('an exit after HEALTHY_UPTIME_MS of uptime resets the counter and respawns immediately', async () => {
+    const originalLaunch = headlessBrokerHost.launch;
+    const launches = [];
+    try {
+        createNode(node('H', { pi_session_id: 'sess-H' }));
+        // Build up one failure first: a short-lived exit defers (5s timer, no launch).
+        tf.fleet.deliverExit('H', { code: 1, signal: null });
+        await tf.drain();
+        assert.equal(events().filter((e) => e.event === 'broker.respawn.deferred' && e.fields?.['backoff_ms'] === 5_000).length >= 1, true, 'precondition: the first short-lived exit deferred at 5s');
+        // A live registration (a manual revive winning the race) supersedes the
+        // pending backoff timer and stamps the current failure count on the entry.
+        const t0 = nowMs;
+        tf.fleet.register('H', { pid: 424_242, exited: new Promise(() => { }) });
+        assert.equal(tf.fleet.get('H').consecutiveFailures, 1, 'the entry mirrors the throttle count at registration');
+        // The relaunch runs healthy past the 60s bar, then exits: the counter
+        // resets BEFORE classification, so the respawn is immediate (backoff 0).
+        headlessBrokerHost.launch = (nodeId) => {
+            launches.push(nodeId);
+            const handle = { pid: 424_243, exited: new Promise(() => { }) };
+            tf.fleet.register(nodeId, handle); // the real host contract: registration at the launch seam
+            return handle;
+        };
+        nowMs = t0 + 61_000;
+        tf.fleet.deliverExit('H', { code: 0, signal: null });
+        await tf.drain();
+        assert.deepEqual(launches, ['H'], 'a healthy-uptime exit respawns immediately — no deferral');
+        assert.equal(tf.fleet.get('H').consecutiveFailures, 0, 'the healthy run reset the consecutive-failure counter');
+        assert.equal(getNode('H').status, 'active', 'the node is live again, never terminalized');
+    }
+    finally {
+        headlessBrokerHost.launch = originalLaunch;
+    }
+});

package/dist/core/__tests__/review-model-floor.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/core/__tests__/review-model-floor.test.js

@@ -0,0 +1,104 @@
+// Run with: node --import tsx/esm --test src/core/__tests__/review-model-floor.test.ts
+//
+// Review workers must not be forced down to light by an explicit `--model`
+// override. Base review work uses the medium floor; the coordinating review
+// persona keeps the stronger default needed to synthesise a broad review.
+import { after, before, test } from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { defaultKindsConfig, defaultModelLaddersConfig } from '../../types.js';
+import { buildLaunchSpec, normalizeModel } from '../runtime/launch.js';
+const priorHome = process.env['HOME'];
+const ladders = defaultModelLaddersConfig();
+// Review's own kind default IS its floor — an explicit override weaker than
+// this rung is replaced by it, provider and all.
+const REVIEW_FLOOR = defaultKindsConfig()["review"].model;
+let home = '';
+before(() => {
+    home = mkdtempSync(join(tmpdir(), 'crtr-review-model-floor-'));
+    process.env['HOME'] = home;
+});
+after(() => {
+    if (priorHome === undefined)
+        delete process.env['HOME'];
+    else
+        process.env['HOME'] = priorHome;
+    rmSync(home, { recursive: true, force: true });
+});
+function expected(model) {
+    return normalizeModel(model, ladders);
+}
+function spec(kind, model) {
+    return buildLaunchSpec(kind, 'base', {
+        lifecycle: 'terminal',
+        hasManager: true,
+        cwd: home,
+        profileId: null,
+        ...(model !== undefined ? { model } : {}),
+    }).launch.model;
+}
+test('base review ignores a light override and keeps its medium floor', () => {
+    assert.equal(spec('review', 'light'), expected(REVIEW_FLOOR));
+    assert.equal(spec('review', 'haiku'), expected(REVIEW_FLOOR));
+});
+test('no base kind defaults below the medium rung', () => {
+    const kinds = defaultKindsConfig();
+    for (const kind of [
+        'general', 'developer', 'design', 'plan', 'spec', 'product', 'review',
+        'plan/reviewers/requirements-coverage', 'plan/reviewers/pattern-consistency',
+        'plan/reviewers/code-smells', 'plan/reviewers/security', 'plan/reviewers/architecture-fit',
+        'spec/requirements', 'product/teardown',
+    ]) {
+        // Kind defaults are provider-qualified and differ per kind; what must hold
+        // for every one of these is the rung, not the provider or the concrete id.
+        const rung = kinds[kind].model.split('/').pop();
+        assert.ok(rung === 'medium' || rung === 'strong' || rung === 'ultra', `${kind} defaults to ${kinds[kind].model}`);
+    }
+});
+test('plan reviewer sub-kinds also keep their medium worker floor', () => {
+    assert.equal(spec('plan/reviewers/security', 'light'), expected('medium'));
+    assert.equal(spec('plan/reviewers/requirements-coverage', 'light'), expected('medium'));
+});
+test('review orchestrator uses its stronger coordinating model', () => {
+    assert.equal(buildLaunchSpec('review', 'orchestrator', { lifecycle: 'terminal', hasManager: true, cwd: home, profileId: null }).launch.model, expected(defaultKindsConfig()['review'].orchestratorModel));
+});
+test('review still permits equal-or-stronger explicit models', () => {
+    assert.equal(spec('review', 'strong'), expected('strong'));
+    assert.equal(spec('review', 'ultra'), expected('ultra'));
+});
+test('explicit provider-qualified overrides preserve their provider pin through launch construction', () => {
+    const qualified = buildLaunchSpec('developer', 'base', {
+        lifecycle: 'terminal', hasManager: true, cwd: home, profileId: null, model: 'anthropic/ultra',
+    }).launch;
+    const exact = buildLaunchSpec('developer', 'base', {
+        lifecycle: 'terminal', hasManager: true, cwd: home, profileId: null, model: 'openai/gpt-5',
+    }).launch;
+    const bare = buildLaunchSpec('developer', 'base', {
+        lifecycle: 'terminal', hasManager: true, cwd: home, profileId: null, model: 'ultra',
+    }).launch;
+    assert.equal(qualified.modelProviderPinned, true);
+    assert.equal(exact.modelProviderPinned, true);
+    assert.equal(bare.modelProviderPinned, false);
+});
+test('explore remains cheap by default and may explicitly run light', () => {
+    assert.equal(spec('explore'), expected(defaultKindsConfig()['explore'].model));
+    assert.equal(spec('explore', 'light'), expected('light'));
+});
+// `node new --model` widened to the full spec grammar (provider/tier, exact
+// provider/id, family alias) alongside the bare tier — the review floor must
+// still hold no matter which shape the requested model arrives in.
+test('review honors an explicit provider/tier override at or above its floor', () => {
+    assert.equal(spec('review', 'openai/ultra'), expected('openai/ultra'));
+});
+test('review still floors a provider/tier override below its floor', () => {
+    assert.equal(spec('review', 'openai/light'), expected(REVIEW_FLOOR));
+});
+test('review passes an unrecognized concrete provider/id straight through, unfloored', () => {
+    // A fully concrete id that matches no ladder cell resolves to no known
+    // strength, so floorReviewModel trusts the explicit pin as-given rather than
+    // guessing — same trust boundary as `node config --model` on a dormant node.
+    const concrete = 'anthropic/claude-does-not-exist-9000';
+    assert.equal(spec('review', concrete), concrete);
+});

package/dist/core/__tests__/revive-all.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/core/__tests__/revive-all.test.js

@@ -0,0 +1,147 @@
+// Run with: node --import tsx/esm --test src/core/__tests__/revive-all.test.ts
+//
+// REGRESSION LOCK — gh issue #9 (revive-all over the canvas). The heart of the
+// feature is the SELECTION predicate `isDisconnected`: which nodes a one-shot
+// "resume everything" sweep relaunches. Getting that wrong is the whole risk —
+// too narrow strands survivors, too broad resurrects finished/closed
+// conversations nobody asked to reopen, or double-spawns a live node.
+//
+// (1) WHY PURE, NOT TMUX — isDisconnected is a total function of (meta, isAlive
+//     probe, scope): zero process, zero tmux, instant. The liveness probe is
+//     injected so the alive/dead branch is exercised without a real pid. The
+//     reviveAll orchestration over it is a thin select-then-reviveNode loop;
+//     reviveNode is covered (double-spawn guard) by revive.test.ts.
+//
+// (2) THE INVARIANTS LOCKED:
+//     - engine running (live pid) → NOT disconnected (never double-spawn).
+//     - no resumable saved session (no pi_session_id AND no pi_session_file) →
+//       NOT disconnected (nothing to resume).
+//     - dead/crashed with a session → disconnected (swept by default).
+//     - done/canceled (terminal-by-choice) → always excluded (no opt-in).
+//     - kind:'human' (the human bridge, never a pi engine) → never disconnected.
+//
+// (3) HOW IT FAILS IF THE LOGIC REGRESSES — drop the terminal-by-choice guard
+//     and the done/canceled exclusion asserts go RED; drop the session
+//     requirement and the no-session assert goes RED; invert the liveness branch
+//     and the live-pid assert goes RED.
+import { test, before, after, beforeEach } from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { spawnSync } from 'node:child_process';
+import { isDisconnected, TERMINAL_BY_CHOICE, listDisconnected } from '../runtime/revive-all.js';
+import { createNode } from '../canvas/canvas.js';
+import { closeDb } from '../canvas/db.js';
+const ALIVE = () => true;
+const DEAD = () => false;
+function meta(over = {}) {
+    return {
+        node_id: 'n',
+        name: 'n',
+        created: new Date().toISOString(),
+        cwd: '/tmp/work',
+        kind: 'developer',
+        mode: 'base',
+        lifecycle: 'terminal',
+        status: 'active',
+        pi_pid: 1234,
+        pi_session_id: 'uuid-1',
+        pi_session_file: '/abs/sess.jsonl',
+        ...over,
+    };
+}
+// --- engine liveness branch -------------------------------------------------
+test('a node whose engine is RUNNING (live pid) is NOT disconnected — never double-spawn', () => {
+    assert.equal(isDisconnected(meta({ pi_pid: 1234 }), ALIVE), false);
+});
+test('a node whose engine is DEAD but has a saved session IS disconnected', () => {
+    assert.equal(isDisconnected(meta({ pi_pid: 1234 }), DEAD), true);
+});
+test('a node with a null pid (never/ no longer recorded) and a session IS disconnected', () => {
+    assert.equal(isDisconnected(meta({ pi_pid: null }), DEAD), true);
+});
+// --- resumable-session requirement -----------------------------------------
+test('a dead node with NO resumable saved session is NOT disconnected', () => {
+    assert.equal(isDisconnected(meta({ pi_session_id: null, pi_session_file: null }), DEAD), false);
+});
+test('a session FILE alone (older node, no id) is enough to be disconnected', () => {
+    assert.equal(isDisconnected(meta({ pi_session_id: null }), DEAD), true);
+});
+// --- terminal-by-choice scope ----------------------------------------------
+for (const status of TERMINAL_BY_CHOICE) {
+    test(`a dead-engine '${status}' node is EXCLUDED (terminal-by-choice, no opt-in)`, () => {
+        assert.equal(isDisconnected(meta({ status: status }), DEAD), false);
+    });
+}
+test("a crashed 'dead' node is swept by default (a crash is involuntary, not terminal-by-choice)", () => {
+    assert.equal(isDisconnected(meta({ status: 'dead' }), DEAD), true);
+});
+// Regression (stuck-node incident): a node that already pushed its FINAL REPORT
+// FINISHED its own work and must NEVER be mass-revived, EVEN when a crash
+// mid-reap left it 'dead' (not terminal-by-choice) with the report committed.
+// Resurrecting it as 'active' stranded it re-reviving forever (it can't
+// re-reap). final_report is decisive regardless of status.
+test('a node with final_report set is EXCLUDED even when status is dead (finished its own work)', () => {
+    assert.equal(isDisconnected(meta({ status: 'dead', final_report: 'done report' }), DEAD), false);
+    assert.equal(isDisconnected(meta({ status: 'active', final_report: 'done report' }), DEAD), false);
+});
+test("dormant 'idle' nodes with a saved session are swept by default", () => {
+    assert.equal(isDisconnected(meta({ status: 'idle', intent: 'idle-release' }), DEAD), true);
+});
+// --- human-bridge carve-out -------------------------------------------------
+test("kind:'human' rows (the human bridge, never a pi engine) are never disconnected", () => {
+    assert.equal(isDisconnected(meta({ kind: 'human', pi_pid: null }), DEAD), false);
+});
+// --- listDisconnected over a live canvas (the sweep's selection) -------------
+//
+// Locks that the canvas-wide select returns EXACTLY the disconnected set against
+// the REAL isPidAlive probe — a live-engine node (this process's pid) is kept
+// out, a crashed one with a session is in, a done one is excluded. No broker is
+// launched: listDisconnected is the side-effect-free preview the command gates on.
+let home;
+function deadPid() {
+    const r = spawnSync('true', [], { stdio: 'ignore' });
+    return r.pid ?? 0x7ffffffe;
+}
+function fabricate(id, over) {
+    createNode({
+        node_id: id,
+        name: id,
+        created: new Date().toISOString(),
+        cwd: '/tmp/work',
+        host_kind: 'broker',
+        kind: 'developer',
+        mode: 'base',
+        lifecycle: 'terminal',
+        status: 'active',
+        pi_session_id: 'uuid-' + id,
+        ...over,
+    });
+    closeDb();
+}
+before(() => {
+    home = mkdtempSync(join(tmpdir(), 'crtr-revive-all-'));
+    process.env['CRTR_HOME'] = home;
+});
+beforeEach(() => {
+    closeDb();
+    rmSync(home, { recursive: true, force: true });
+});
+after(() => {
+    closeDb();
+    rmSync(home, { recursive: true, force: true });
+    delete process.env['CRTR_HOME'];
+});
+test('listDisconnected returns EXACTLY the disconnected nodes (crashed-with-session), excluding live + done', () => {
+    // disconnected: dead pid + a saved session.
+    fabricate('crashed', { status: 'active', pi_pid: deadPid() });
+    // connected: a LIVE engine pid (this process — read-only here) → excluded.
+    fabricate('live', { status: 'active', pi_pid: process.pid });
+    // terminal-by-choice: done + dead pid + session → excluded.
+    fabricate('finished', { status: 'done', pi_pid: deadPid() });
+    // no resumable session → excluded.
+    fabricate('sessionless', { status: 'active', pi_pid: deadPid(), pi_session_id: null });
+    const ids = listDisconnected().map((m) => m.node_id).sort();
+    assert.deepEqual(ids, ['crashed'], 'only the crashed-with-session node is swept');
+});

package/dist/core/__tests__/revive.test.d.ts

@@ -0,0 +1 @@
+export {};