@north-light/crouter 0.3.303 → 0.3.304

package/dist/daemon/manage.js

@@ -19,7 +19,7 @@ import { clearDaemonStartupBlocked, daemonStartupBlocked } from './startup-block
 import { exclusiveLockOwnerPid } from '../core/exclusive-lock.js';
 import { onDiskMigrationLockPath } from '../core/canvas/paths.js';
 import { envNoDaemonAutostart } from '../shared/env.js';
-import { CrtrClient } from '../api/index.js';
+import { CrtrClient, waitForDaemonAvailability } from '../api/index.js';
 // Daemon env sanitization
 /** Env keys that must NEVER reach the daemon process. Restarting crtrd is
  *  overwhelmingly done from inside an agent node's own bash tool (`crtr sys
@@ -208,16 +208,20 @@ const DAEMON_VERIFY_POLL_MS = 10;
 /** A status/start readiness probe must not hang behind a daemon that accepts a
  * Unix connection but never answers it. */
 const DAEMON_HEALTH_PROBE_TIMEOUT_MS = 1_000;
-/** True only when the mandatory Unix API socket answers `/healthz`. This is
- * deliberately separate from `isDaemonRunning()`: a pidfile remains useful for
- * identifying the process an operator must stop, but it is not readiness. */
+/** Make one `/healthz` observation without client-side recovery. The caller
+ * owns the availability window, so this probe cannot extend it. */
+export async function probeDaemonServing(timeoutMs = DAEMON_HEALTH_PROBE_TIMEOUT_MS) {
+    const probeTimeoutMs = Math.min(timeoutMs, DAEMON_HEALTH_PROBE_TIMEOUT_MS);
+    const health = await CrtrClient.forLocalSocket({ autostart: false, timeoutMs: probeTimeoutMs }).probeHealthz(probeTimeoutMs);
+    if (!health.ok)
+        throw new Error('crtrd health check reported not ready');
+}
+/** True only when one mandatory Unix API socket probe answers `/healthz`.
+ * A pidfile identifies an owner; it is never readiness. */
 export async function isDaemonServing() {
     try {
-        const health = await CrtrClient.forLocalSocket({
-            autostart: false,
-            timeoutMs: DAEMON_HEALTH_PROBE_TIMEOUT_MS,
-        }).healthz();
-        return health.ok;
+        await probeDaemonServing();
+        return true;
     }
     catch {
         return false;
@@ -255,32 +259,47 @@ export async function verifyDaemonStartup(pid, timeoutMs = DAEMON_STARTUP_WINDOW
     const read = deps.readPidfile ?? readPidfile;
     const alive = deps.isPidAlive ?? isPidAlive;
     const exited = deps.childExited;
-    const serving = deps.isDaemonServing ?? isDaemonServing;
+    const serving = deps.isDaemonServing;
+    const probeServing = deps.probeDaemonServing ?? (async (remainingMs) => {
+        if (serving !== undefined) {
+            if (await serving())
+                return;
+            throw new Error('crtrd health check reported not ready');
+        }
+        await probeDaemonServing(remainingMs);
+    });
     const sleep = deps.sleepMs ?? sleepMs;
     const now = deps.now ?? Date.now;
-    const deadline = now() + timeoutMs;
-    while (now() <= deadline) {
-        const owner = read();
-        if (owner !== null && alive(owner) && await serving())
-            return owner === pid ? null : owner;
-        const exitState = exited?.();
-        if (exitState !== null && exitState !== undefined) {
-            if (exitState.signal !== null) {
-                throw new Error(`daemon ${pid} exited before becoming ready (by ${exitState.signal})`);
-            }
-            if (exitState.code === DAEMON_EXIT_STARTUP_BLOCKED) {
-                throw new DaemonStartupBlockedError(pid);
-            }
-            if (exitState.code !== 0) {
-                throw new Error(`daemon ${pid} exited before becoming ready (with exit code ${exitState.code ?? '?'})`);
+    let owner = null;
+    let terminalError = null;
+    await waitForDaemonAvailability({
+        windowMs: timeoutMs,
+        pollIntervalMs: DAEMON_VERIFY_POLL_MS,
+        now,
+        sleep,
+        retry: (error) => error !== terminalError,
+        probe: async (remainingMs) => {
+            const candidate = read();
+            if (candidate === null || !alive(candidate)) {
+                const exitState = exited?.();
+                if (exitState !== null && exitState !== undefined) {
+                    if (exitState.signal !== null)
+                        terminalError = new Error(`daemon ${pid} exited before becoming ready (by ${exitState.signal})`);
+                    else if (exitState.code === DAEMON_EXIT_STARTUP_BLOCKED)
+                        terminalError = new DaemonStartupBlockedError(pid);
+                    else if (exitState.code !== 0)
+                        terminalError = new Error(`daemon ${pid} exited before becoming ready (with exit code ${exitState.code ?? '?'})`);
+                    // A clean exit can have lost the ownership claim, so keep observing.
+                    if (terminalError !== null)
+                        throw terminalError;
+                }
+                throw new Error(`daemon ${pid} has no live pidfile owner`);
             }
-            // Clean code-0 exit: a possible ownership-claim loser, not readiness.
-            // Keep polling the same deadline for a different live pidfile owner; the
-            // loop's timeout throw below fires if none appears.
-        }
-        await sleep(DAEMON_VERIFY_POLL_MS);
-    }
-    throw new Error(`daemon ${pid} did not become ready within ${timeoutMs}ms`);
+            await probeServing(remainingMs);
+            owner = candidate;
+        },
+    });
+    return owner === pid ? null : owner;
 }
 /** Wait until a signaled daemon has exited and relinquished its pidfile. This
  *  makes a following start safe to claim the singleton rather than racing its
@@ -355,15 +374,19 @@ export async function stopDaemonProcess(pid, timeoutMs = DAEMON_SHUTDOWN_WINDOW_
 export async function spawnDaemon() {
     const recordedPid = readPidfile();
     if (recordedPid !== null && isPidAlive(recordedPid)) {
-        if (await isDaemonServing())
+        try {
+            await probeDaemonServing();
             return { started: false, existing_pid: recordedPid };
-        return {
-            started: false,
-            existing_pid: recordedPid,
-            running: true,
-            serving: false,
-            next: 'Run `crtr sys daemon stop` to release the non-serving daemon before starting it again.',
-        };
+        }
+        catch (error) {
+            return {
+                started: false,
+                existing_pid: recordedPid,
+                running: true,
+                serving: false,
+                probe_error: error.message,
+            };
+        }
     }
     // Ensure the canvas home directory exists so the daemon can write its pidfile.
     mkdirSync(crtrHome(), { recursive: true });

package/dist/daemon/reconcilers/managed-worktree-sweep.js

@@ -30,6 +30,12 @@ const TERMINAL_STATUSES = ['done', 'dead', 'canceled'];
 function isTerminal(row) {
     return TERMINAL_STATUSES.includes(row.status);
 }
+/** A closed record already has an explicit close disposition. An open record
+ *  reaches the sweep only when its owner published a final result; a crash or
+ *  cancellation leaves its managed checkout available for explicit recovery. */
+function hasSweepDisposition(row, wt) {
+    return wt?.state === 'closed' || (wt?.state === 'open' && row.status === 'done' && row.final_report !== null);
+}
 /** Is this node's engine definitively gone?
  *
  *  Terminal status alone is not enough: `push final` marks a node `done` while
@@ -49,6 +55,8 @@ function engineIsGone(row, fleet, snapshot) {
         return false;
     if (!isTerminal(row))
         return false;
+    if (!hasSweepDisposition(row, getNode(row.node_id)?.managed_worktree))
+        return false;
     return recordedPidLiveness(row.pi_pid, row.pi_pid_identity, snapshot) === 'dead';
 }
 /** The eligibility recheck the reconciliation runs immediately before EVERY Git
@@ -70,6 +78,8 @@ function eligibilityGuard(row, fleet) {
             const current = getRow(nodeId);
             if (current === null || !isTerminal(current) || current.frozen_at !== null)
                 return false;
+            if (!hasSweepDisposition(current, getNode(nodeId)?.managed_worktree))
+                return false;
             return current.pi_pid === pid && current.pi_pid_identity === identity;
         },
     };
@@ -120,6 +130,8 @@ export class ManagedWorktreeSweepReconciler {
             }
             if (!isReconcilable(wt) || wt == null)
                 continue;
+            if (!hasSweepDisposition(row, wt))
+                continue;
             if (!isDueForSweep(wt, now))
                 continue;
             due.push(row.node_id);

package/dist/daemon/reconcilers/node-lifecycle/tick.d.ts

@@ -7,12 +7,6 @@ export interface NodeLifecycleContext {
     rows: readonly NodeRow[];
     fleet: FleetRegistry;
 }
-/** Whether this row should have a live engine right now. An active row always
- *  should — that status IS the claim that it is executing. An idle row was
- *  deliberately released and holds no engine on purpose: only mail it has not
- *  seen brings one back, because a live obligation keeps a node WAKEABLE rather
- *  than awake, and relaunching on one would re-release it every tick. */
-export declare function wantsExecution(row: NodeRow): boolean;
 export declare class NodeLifecycleReconciler {
     private readonly freezeLane;
     run(now: number, ctx: NodeLifecycleContext): Promise<FrozenCounts>;

package/dist/daemon/reconcilers/node-lifecycle/tick.js

@@ -12,7 +12,6 @@ import { isSafeNodeId } from '../../../core/canvas/paths.js';
 import { recordedPidLiveness } from '../../../core/canvas/pid.js';
 import { emitEvent } from '../../../core/events/emit.js';
 import { operationIdContext } from '../../../core/events/operation-id.js';
-import { hasUnseenWakeMail } from '../../../core/feed/inbox.js';
 import { hasCleanAbort, isBusy } from '../../../core/runtime/busy.js';
 import { isProfilePaused } from '../../../core/profiles/manifest.js';
 import { classifyDeadNode } from '../../fleet.js';
@@ -20,21 +19,10 @@ import { applyParkEvent, hasPendingWake, parkEvent } from '../live-obligation.js
 import { FreezeLaneReconciler } from './freeze-lane.js';
 import { isInBackoff } from './respawn-policy.js';
 import { handleTerminatingRow } from './terminating.js';
+import { wantsExecution } from './wants-execution.js';
 function isTerminal(row) {
     return row.status === 'done' || row.status === 'canceled' || row.status === 'dead';
 }
-/** Whether this row should have a live engine right now. An active row always
- *  should — that status IS the claim that it is executing. An idle row was
- *  deliberately released and holds no engine on purpose: only mail it has not
- *  seen brings one back, because a live obligation keeps a node WAKEABLE rather
- *  than awake, and relaunching on one would re-release it every tick. */
-export function wantsExecution(row) {
-    if (row.status === 'active')
-        return true;
-    if (row.status !== 'idle')
-        return false;
-    return hasUnseenWakeMail(row.node_id);
-}
 export class NodeLifecycleReconciler {
     freezeLane = new FreezeLaneReconciler();
     async run(now, ctx) {

package/dist/daemon/reconcilers/node-lifecycle/wants-execution.d.ts

@@ -0,0 +1,5 @@
+import type { NodeRow } from '../../../core/canvas/index.js';
+/** Whether the lifecycle tick must ensure this row has an engine. The durable
+ * message handler calls this after appending its entry: that append makes the
+ * idle branch's unseen-wake-mail check true for a wake-capable entry. */
+export declare function wantsExecution(row: Pick<NodeRow, 'node_id' | 'status'>): boolean;

package/dist/daemon/reconcilers/node-lifecycle/wants-execution.js

@@ -0,0 +1,11 @@
+import { hasUnseenWakeMail } from '../../../core/feed/inbox.js';
+/** Whether the lifecycle tick must ensure this row has an engine. The durable
+ * message handler calls this after appending its entry: that append makes the
+ * idle branch's unseen-wake-mail check true for a wake-capable entry. */
+export function wantsExecution(row) {
+    if (row.status === 'active')
+        return true;
+    if (row.status !== 'idle')
+        return false;
+    return hasUnseenWakeMail(row.node_id);
+}

package/dist/pi-extensions/__tests__/canvas-context-intro.test.js

@@ -5,12 +5,16 @@
 import { test, before, after, beforeEach, afterEach } from 'node:test';
 import assert from 'node:assert/strict';
 import { existsSync, mkdtempSync, mkdirSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
+import { createServer } from 'node:http';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { registerCanvasContextIntro, CONTEXT_INTRO_CUSTOM_TYPE, } from '../canvas-context-intro.js';
 import { registerCanvasDocSubstrate } from '../canvas-doc-substrate.js';
 import { buildContextBearings as buildContextIntro } from '../../core/runtime/bearings.js';
 import { installPersonaTransitionGate } from '../../core/runtime/broker-persona-guidance.js';
+import { FaultRetry } from '../../core/runtime/broker/fault-retry.js';
+import { readFault } from '../../core/runtime/fault.js';
+import { ApiError } from '../../api/errors.js';
 import { createNode, deleteNode, getNode, updateNode } from '../../core/canvas/canvas.js';
 import { apiSocketPath } from '../../core/canvas/paths.js';
 import { createApiServer } from '../../daemon/api/server.js';
@@ -32,6 +36,23 @@ async function waitForSocket(path) {
     }
     throw new Error(`api socket never bound at ${path}`);
 }
+async function listenUnavailableApi(path) {
+    const unavailable = createServer((_request, response) => {
+        response.writeHead(503, { 'content-type': 'application/json' });
+        response.end(JSON.stringify({ error: { code: 'daemon_unavailable', message: 'local daemon connection refused' } }));
+    });
+    await new Promise((resolve, reject) => {
+        unavailable.once('error', reject);
+        unavailable.listen(path, () => {
+            unavailable.off('error', reject);
+            resolve();
+        });
+    });
+    return unavailable;
+}
+function closeHttpServer(server) {
+    return new Promise((resolve, reject) => server.close((error) => error === undefined ? resolve() : reject(error)));
+}
 function node(id, over = {}) {
     return {
         node_id: id,
@@ -322,6 +343,68 @@ test('child /new commits its baseline before ack and the provider gate delivers
             process.env['CRTR_PROFILE_ID'] = previousProfile;
     }
 });
+test('a typed local API error remains a retryable broker-to-daemon fault after settlement', async () => {
+    createNode(node('persona-transport', {
+        persona_ack: { kind: 'review', mode: 'base', lifecycle: 'terminal' },
+    }));
+    await server.close();
+    let unavailable;
+    try {
+        unavailable = await listenUnavailableApi(apiSocketPath());
+        const messages = [];
+        const agent = {
+            state: { messages },
+            transformContext: async (context) => context,
+        };
+        installPersonaTransitionGate('persona-transport', {
+            agent,
+            sessionManager: {
+                appendCustomMessageEntry: () => undefined,
+            },
+        });
+        await assert.rejects(() => agent.transformContext([{ role: 'user', content: 'continue' }]), (error) => error instanceof ApiError && error.status === 503 && error.code === 'daemon_unavailable');
+        assert.deepEqual(readFault('persona-transport'), {
+            link: 'broker↔crtrd',
+            op: 'persona transition',
+            kind: 'connection',
+            retry: { disposition: 'manual' },
+            message: 'local daemon connection refused',
+            since: readFault('persona-transport')?.since,
+            operation_id: readFault('persona-transport')?.operation_id,
+        }, 'the typed daemon transport error is recorded before Pi turns it into a generic agent error');
+        const session = {
+            sessionManager: { getLeafId: () => 'persona-transport-leaf' },
+            model: undefined,
+        };
+        const generation = {
+            id: 1,
+            session,
+            fallback: { blocked: false },
+            stagedProviderAgentEnd: {
+                messages: [{ role: 'assistant', stopReason: 'error', errorMessage: 'local daemon connection refused' }],
+            },
+            stagedWatchdogAbort: false,
+            stagedRefreshAbort: false,
+            stagedOverflowFailure: null,
+            executingTools: new Set(),
+        };
+        new FaultRetry({
+            nodeId: 'persona-transport',
+            currentSession: () => session,
+            installedGeneration: () => generation,
+        }).publishSettledProviderFault(generation);
+        const fault = readFault('persona-transport');
+        assert.equal(fault?.link, 'broker↔crtrd');
+        assert.equal(fault?.kind, 'connection');
+        assert.equal(fault?.retry.disposition, 'manual');
+    }
+    finally {
+        if (unavailable !== undefined)
+            await closeHttpServer(unavailable);
+        server = createApiServer();
+        await waitForSocket(apiSocketPath());
+    }
+});
 test('resume idempotency uses the exact details stamp independent of block text', async () => {
     createNode(node('stampD'));
     // The exact node-id stamp is authoritative even when prose carries no id.

package/dist/pi-extensions/__tests__/integration/canvas-bash-valve.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/pi-extensions/__tests__/integration/canvas-bash-valve.test.js

@@ -0,0 +1,133 @@
+import assert from 'node:assert/strict';
+import { spawn } from 'node:child_process';
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { test } from 'node:test';
+import { backgroundRunningBashJobs, bashJobsDir, bashJobPaths, readJobPgid, stopBackgroundBashJob } from '../../../core/bash-jobs.js';
+import { capturePidIdentities, isPidAlive } from '../../../core/canvas/pid.js';
+import { BACKGROUND_BASH_LOG_TRUNCATION_MARKER, MAX_BACKGROUND_BASH_LOG_BYTES, createValveOperations } from '../../canvas-bash-valve.js';
+const POLL_MS = 20;
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function waitFor(condition, message) {
+    const deadline = Date.now() + 4_000;
+    while (Date.now() < deadline) {
+        if (condition())
+            return;
+        await sleep(POLL_MS);
+    }
+    throw new Error(message);
+}
+async function backgroundOneJob(contextDir) {
+    await waitFor(() => backgroundRunningBashJobs(contextDir).length === 1, 'bash job did not start');
+    await waitFor(() => {
+        try {
+            return readdirSync(bashJobsDir(contextDir)).length === 1;
+        }
+        catch {
+            return false;
+        }
+    }, 'background job directory was not created');
+    return readdirSync(bashJobsDir(contextDir))[0];
+}
+function killOwnedProcess(pid) {
+    if (pid === undefined)
+        return;
+    try {
+        process.kill(pid, 'SIGKILL');
+    }
+    catch {
+        // The test already stopped this exact process.
+    }
+}
+test('background cancellation reaches a descendant that changed process groups', { timeout: 10_000 }, async () => {
+    const root = mkdtempSync(join(tmpdir(), 'crtr-bash-tree-'));
+    const contextDir = join(root, 'context');
+    const escapedPidPath = join(root, 'escaped.pid');
+    let escapedPid;
+    try {
+        const python = [
+            'import os, pathlib, time',
+            'os.setpgrp()',
+            `pathlib.Path(${JSON.stringify(escapedPidPath)}).write_text(f"{os.getpid()} {os.getpgrp()}")`,
+            'time.sleep(30)',
+        ].join('; ');
+        const run = createValveOperations('test-node', contextDir).exec(`python3 -c ${JSON.stringify(python)}`, root, { onData: () => { } });
+        const jobId = await backgroundOneJob(contextDir);
+        await run;
+        await waitFor(() => existsSync(escapedPidPath), 'escaped descendant did not start');
+        const [pidText, pgidText] = readFileSync(escapedPidPath, 'utf8').split(' ');
+        escapedPid = Number(pidText);
+        const escapedPgid = Number(pgidText);
+        assert.ok(Number.isInteger(escapedPid) && escapedPid > 1);
+        assert.notEqual(escapedPgid, readJobPgid(contextDir, jobId));
+        const stopped = await stopBackgroundBashJob(contextDir, jobId);
+        assert.equal(stopped.kind, 'stopped');
+        assert.equal(stopped.signaled, true);
+        await waitFor(() => {
+            try {
+                process.kill(escapedPid, 0);
+                return false;
+            }
+            catch {
+                return true;
+            }
+        }, 'escaped descendant survived background cancellation');
+    }
+    finally {
+        killOwnedProcess(escapedPid);
+        rmSync(root, { recursive: true, force: true });
+    }
+});
+test('stale background metadata never signals a reused supervisor PID', async () => {
+    const root = mkdtempSync(join(tmpdir(), 'crtr-bash-stale-pid-'));
+    const contextDir = join(root, 'context');
+    const child = spawn(process.execPath, ['-e', 'setTimeout(() => {}, 30_000)']);
+    try {
+        assert.notEqual(child.pid, undefined);
+        const actualIdentity = capturePidIdentities([child.pid])?.get(child.pid);
+        assert.notEqual(actualIdentity, undefined);
+        const paths = bashJobPaths(contextDir, '1234567890-stale');
+        mkdirSync(paths.dir, { recursive: true });
+        writeFileSync(paths.jobBg, '');
+        writeFileSync(paths.jobPgid, String(child.pid));
+        writeFileSync(paths.jobPgidIdentity, 'stale-supervisor');
+        const stopped = await stopBackgroundBashJob(contextDir, paths.jobId);
+        assert.deepEqual(stopped, { kind: 'stopped', signaled: false, pgid: child.pid, logPath: paths.jobLog });
+        assert.equal(isPidAlive(child.pid), true, 'a process with a reused supervisor PID was signaled');
+    }
+    finally {
+        child.kill('SIGKILL');
+        rmSync(root, { recursive: true, force: true });
+    }
+});
+test('background log retains a capped prefix, truncation marker, and does not grow afterward', { timeout: 10_000 }, async () => {
+    const root = mkdtempSync(join(tmpdir(), 'crtr-bash-log-'));
+    const contextDir = join(root, 'context');
+    try {
+        const writer = [
+            'const chunk = Buffer.alloc(1024 * 1024, 120)',
+            'setTimeout(() => {',
+            '  for (let i = 0; i < 11; i += 1) process.stdout.write(chunk)',
+            '  setTimeout(() => process.stdout.write("after-cap"), 200)',
+            '}, 200)',
+            'setTimeout(() => {}, 1_000)',
+        ].join('; ');
+        const run = createValveOperations('test-node', contextDir).exec(`${process.execPath} -e ${JSON.stringify(writer)}`, root, { onData: () => { } });
+        const jobId = await backgroundOneJob(contextDir);
+        await run;
+        const { jobExit, jobLog } = bashJobPaths(contextDir, jobId);
+        await waitFor(() => existsSync(jobLog) && readFileSync(jobLog, 'utf8').includes(BACKGROUND_BASH_LOG_TRUNCATION_MARKER), 'log did not record truncation');
+        const sizeAtCap = statSync(jobLog).size;
+        assert.ok(sizeAtCap <= MAX_BACKGROUND_BASH_LOG_BYTES);
+        await sleep(500);
+        assert.equal(statSync(jobLog).size, sizeAtCap, 'discarded output continued growing the log');
+        await waitFor(() => existsSync(jobExit), 'writer did not finish');
+        assert.equal(readFileSync(jobExit, 'utf8').trim(), '0', 'collector changed the writer exit status');
+    }
+    finally {
+        rmSync(root, { recursive: true, force: true });
+    }
+});

package/dist/pi-extensions/canvas-bash-valve.d.ts

@@ -1,5 +1,7 @@
 import { createBashToolDefinition, type BashOperations } from '@earendil-works/pi-coding-agent';
 import type { ExtensionAPI } from '@earendil-works/pi-coding-agent';
+export declare const MAX_BACKGROUND_BASH_LOG_BYTES: number;
+export declare const BACKGROUND_BASH_LOG_TRUNCATION_MARKER = "\n[background bash job output truncated at 10 MiB; remaining output discarded]\n";
 /** The valve's BashOperations backend. Each tool execution closes over its own
  * `takePurpose` and its own preview-result path, so concurrent calls cannot
  * exchange labels or result mirrors. The purpose is consumed even when the