@north-light/crouter 0.3.278 → 0.3.279

package/dist/commands/node/bash.js

@@ -2,7 +2,6 @@
 import { envNodeId } from '../../shared/env.js';
 import { defineBranch, defineLeaf } from '../../core/command.js';
 import { InputError } from '../../core/io.js';
-import { displayMessage } from '../../core/runtime/tmux-chrome.js';
 import { contextDir } from '../../core/canvas/paths.js';
 import { existsSync } from 'node:fs';
 import { activeBackgroundBashJobs, backgroundRunningBashJobs, bashJobCommandLine, bashJobPaths, clearJobDeadline, formatBashElapsed, isSafeBashJobId, jobStartedAtMs, readJobPgid, stopBackgroundBashJob, tailJobLog, writeJobDeadlineMs } from '../../core/bash-jobs.js';
@@ -41,19 +40,17 @@ const nodeBashBackground = defineLeaf({
         effects: ['Signals every foreground bash command currently running for the node to return from its tool call immediately while its detached supervisor keeps the command running.', 'Each handed-off command retains its log under the node context jobs directory and sends the node an urgent inbox message when it exits, which steers the node mid-turn.'],
     },
     run: async (input) => {
-        const { id, pane } = resolveBashTarget(input, 'no node found for the bash handoff');
+        const { id } = resolveBashTarget(input, 'no node found for the bash handoff');
         if ((await getNodeOrNull(id)) === null) {
             throw new InputError({ error: 'not_found', message: `no node: ${id}`, next: 'List nodes with `crtr node inspect list`.' });
         }
         const jobs = backgroundRunningBashJobs(contextDir(id));
         const now = Date.now();
         const elapsed = jobs.map((job) => ({ job_id: job.jobId, elapsed_ms: now - (jobStartedAtMs(job.jobId) ?? now) }));
-        const toast = jobs.length === 0
-            ? 'crtr: no running bash command'
-            : jobs.length === 1
-                ? `crtr: bash handed off · ${formatBashElapsed(elapsed[0].elapsed_ms)}`
-                : `crtr: bash handed off (${jobs.length})`;
-        displayMessage(toast, pane);
+        // No tmux toast. A tmux status-line message commandeers the whole client's
+        // status bar for seconds over an action the user just took deliberately;
+        // the viewer reports its own handoff in its own status line, which is where
+        // someone watching that pane is already looking.
         return { node_id: id, jobs: elapsed };
     },
     render: (result) => {

package/dist/core/__tests__/helpers/fleet.d.ts

@@ -9,4 +9,4 @@ export interface TestFleet {
  *  process's fleet. Call in `before`/`beforeEach`; `bindFleet` throws on a
  *  conflicting rebind, so the previous binding is reset first (same idiom as
  *  resetEventSourceForTests). Pass `now` to inject the throttle clock. */
-export declare function bindTestFleet(opts?: Partial<Pick<DaemonFleetOpts, 'epoch' | 'now'>>): TestFleet;
+export declare function bindTestFleet(opts?: Partial<Pick<DaemonFleetOpts, 'epoch' | 'now' | 'probeOnline'>>): TestFleet;

package/dist/core/__tests__/helpers/fleet.js

@@ -30,6 +30,11 @@ export function bindTestFleet(opts = {}) {
             });
         },
         ...(opts.now !== undefined ? { now: opts.now } : {}),
+        // Respawn policy consults network reachability — an offline box parks a
+        // node rather than spending its respawn budget. Default it online so tests
+        // stay deterministic off the network; pass `probeOnline` to drive the
+        // outage branch.
+        probeOnline: opts.probeOnline ?? (async () => true),
     });
     bindFleet(fleet);
     return {

package/dist/core/__tests__/helpers/harness.js

@@ -252,7 +252,11 @@ export async function createHarness(opts = {}) {
             await tail;
         } while (tail !== laneTail);
     }
-    const fleet = new DaemonFleet({ epoch: 'harness-epoch', enqueue: enqueueLaneJob });
+    // Respawn policy probes network reachability (an offline box parks a node
+    // instead of spending its respawn budget). Reachability is an environment
+    // condition, not runtime behavior under test — pin it online so a sandboxed
+    // or offline runner exercises the same recovery path a connected one does.
+    const fleet = new DaemonFleet({ epoch: 'harness-epoch', enqueue: enqueueLaneJob, probeOnline: async () => true });
     bindFleet(fleet);
     // Neutralize ensureDaemon (spawn.ts calls it on every `node new`): write a
     // fake-live-daemon pidfile pointing at OUR (always-alive) pid, so

package/dist/core/runtime/busy.d.ts

@@ -1,3 +1,7 @@
+export type TransportPoisonVerdict = {
+    consecutivePasses: number;
+    routes: string[];
+};
 /** Mark a node mid-turn (pi entered a turn). Best-effort. A new turn makes any
  * prior clean-abort certificate stale before it can affect a later crash. */
 export declare function markBusy(nodeId: string): void;
@@ -10,6 +14,15 @@ export declare function clearBusy(nodeId: string): void;
 export declare function markCleanAbort(nodeId: string): void;
 /** Remove a consumed clean-abort certificate as a broker boots. */
 export declare function clearCleanAbort(nodeId: string): void;
+/** Record the provider-rotation verdict immediately before it terminates this broker.
+ * It survives the broker exit so the fleet can surface the cause if this launch
+ * exhausts respawn recovery. */
+export declare function recordTransportPoisonVerdict(nodeId: string, verdict: TransportPoisonVerdict): void;
+/** A verdict belongs to one broker launch. Clear it before every replacement so
+ * a later unrelated crash loop cannot inherit an old transport diagnosis. */
+export declare function clearTransportPoisonVerdict(nodeId: string): void;
+/** Read only a complete, structurally valid provider-rotation verdict. */
+export declare function readTransportPoisonVerdict(nodeId: string): TransportPoisonVerdict | null;
 /** Whether the dead broker left a persisted clean-abort turn witness. */
 export declare function hasCleanAbort(nodeId: string): boolean;
 /** Is the node currently inside a turn? AND this with `pidAlive` at the call

package/dist/core/runtime/busy.js

@@ -13,7 +13,7 @@
 // read site, so a stale marker (process crashed mid-turn without firing
 // agent_end) is harmless — the dead pid fails the AND and the node is reaped.
 // No db migration, atomic touch/unlink, best-effort (never throws).
-import { existsSync, writeFileSync, rmSync, mkdirSync, statSync } from 'node:fs';
+import { existsSync, readFileSync, writeFileSync, rmSync, mkdirSync, statSync } from 'node:fs';
 import { join } from 'node:path';
 import { jobDir } from '../canvas/paths.js';
 function busyPath(nodeId) {
@@ -22,6 +22,9 @@ function busyPath(nodeId) {
 function cleanAbortPath(nodeId) {
     return join(jobDir(nodeId), 'clean-abort');
 }
+function transportPoisonPath(nodeId) {
+    return join(jobDir(nodeId), 'transport-poison.json');
+}
 /** Mark a node mid-turn (pi entered a turn). Best-effort. A new turn makes any
  * prior clean-abort certificate stale before it can affect a later crash. */
 export function markBusy(nodeId) {
@@ -65,6 +68,45 @@ export function clearCleanAbort(nodeId) {
         /* best-effort */
     }
 }
+/** Record the provider-rotation verdict immediately before it terminates this broker.
+ * It survives the broker exit so the fleet can surface the cause if this launch
+ * exhausts respawn recovery. */
+export function recordTransportPoisonVerdict(nodeId, verdict) {
+    try {
+        mkdirSync(jobDir(nodeId), { recursive: true });
+        writeFileSync(transportPoisonPath(nodeId), JSON.stringify(verdict));
+    }
+    catch {
+        /* best-effort: no marker means the crash report remains explicitly unknown */
+    }
+}
+/** A verdict belongs to one broker launch. Clear it before every replacement so
+ * a later unrelated crash loop cannot inherit an old transport diagnosis. */
+export function clearTransportPoisonVerdict(nodeId) {
+    try {
+        rmSync(transportPoisonPath(nodeId), { force: true });
+    }
+    catch {
+        /* best-effort */
+    }
+}
+/** Read only a complete, structurally valid provider-rotation verdict. */
+export function readTransportPoisonVerdict(nodeId) {
+    try {
+        const value = JSON.parse(readFileSync(transportPoisonPath(nodeId), 'utf8'));
+        if (typeof value !== 'object' || value === null)
+            return null;
+        const verdict = value;
+        const consecutivePasses = verdict.consecutivePasses;
+        const routes = verdict.routes;
+        if (typeof consecutivePasses !== 'number' || !Number.isInteger(consecutivePasses) || consecutivePasses < 1 || !Array.isArray(routes) || routes.length === 0 || !routes.every((route) => typeof route === 'string' && route !== ''))
+            return null;
+        return { consecutivePasses, routes };
+    }
+    catch {
+        return null;
+    }
+}
 /** Whether the dead broker left a persisted clean-abort turn witness. */
 export function hasCleanAbort(nodeId) {
     return existsSync(cleanAbortPath(nodeId));

package/dist/core/runtime/revive.js

@@ -38,6 +38,7 @@ import { encodeKickoffOrigin, KICKOFF_ORIGIN_ENV } from './stamp/protocol.js';
 import { headlessBrokerHost } from './host.js';
 import { boundFleet, brokerCapReached, brokerThresholdsForDaemon } from './fleet.js';
 import { clearFault, beginBootFaultAttempt } from './fault.js';
+import { clearTransportPoisonVerdict } from './busy.js';
 import { clearContextExposureState, initializeForkContextExposure, } from '../substrate/injected-store.js';
 import { rootOfSpine } from './nodes.js';
 import { isReviewCompanionBound } from '../review/companion.js';
@@ -468,6 +469,7 @@ async function launchRevive(nodeId, meta, opts) {
     if (opts.recovery !== true)
         cancelCronsOnWake(nodeId);
     ensureNodeDirs(nodeId);
+    clearTransportPoisonVerdict(nodeId);
     const lifecycleHooks = discoverLifecycleHookRegistry(meta.cwd, meta.profile_id).plans.get('node:start') ?? [];
     await invokeLifecycleHooks(lifecycleHooks, {
         node: {

package/dist/daemon/fleet.d.ts

@@ -52,6 +52,9 @@ export interface DaemonFleetOpts {
     enqueue: (job: () => void | Promise<void>) => void;
     /** Injectable clock for tests. */
     now?: () => number;
+    /** Injectable network-reachability probe. Every respawn decision consults it,
+     *  so tests drive the offline branch without touching the real network. */
+    probeOnline?: () => Promise<boolean>;
 }
 export declare class DaemonFleet implements FleetRegistry {
     #private;

package/dist/daemon/fleet.js

@@ -16,9 +16,10 @@ import { fullName } from '../core/canvas/labels.js';
 import { isSafeNodeId } from '../core/canvas/paths.js';
 import { transition } from '../core/runtime/lifecycle.js';
 import { fanDoctrineWake } from '../core/runtime/close.js';
-import { hasCleanAbort, isBusy } from '../core/runtime/busy.js';
+import { hasCleanAbort, isBusy, readTransportPoisonVerdict } from '../core/runtime/busy.js';
 import { hasPendingWake } from './reconcilers/live-obligation.js';
-import { clearRespawnPolicy, MAX_RESPAWN_ATTEMPTS, recordRespawnPolicy, respawnBackoffMs } from './reconcilers/node-lifecycle/respawn-policy.js';
+import { clearRespawnPolicy, deferRespawnWhileOffline, MAX_RESPAWN_ATTEMPTS, OFFLINE_RESPAWN_RETRY_MS, recordRespawnPolicy, respawnBackoffMs, } from './reconcilers/node-lifecycle/respawn-policy.js';
+import { probeOnline as probeOnlineReal } from '../core/runtime/connectivity.js';
 import { reviveNode } from '../core/runtime/revive.js';
 import { pushUrgent } from '../core/feed/feed.js';
 import { emitEvent } from '../core/events/emit.js';
@@ -42,9 +43,14 @@ export async function surfaceBootFailure(meta) {
     await pushUrgent(meta.node_id, body, { from: meta.node_id });
 }
 async function surfaceCrashLoop(meta) {
+    const transportPoison = readTransportPoisonVerdict(meta.node_id);
+    const observedCause = transportPoison === null
+        ? ''
+        : `\n\nProvider rotation observed ${transportPoison.consecutivePasses} consecutive route-queue passes with no HTTP response (tried ${transportPoison.routes.join(', ')}). Check network and provider transport before reviving.`;
     const body = `⚠ Crash loop — \`${meta.name}\` (${meta.kind}) kept dying without settling a turn.\n\n` +
-        `Its session started successfully, but six consecutive relaunches made no progress, so the daemon stopped retrying.\n\n` +
-        `Fix the crash cause, then revive the node if the work still needs doing.`;
+        `Its session started successfully, but six consecutive relaunches made no progress, so the daemon stopped retrying.` +
+        observedCause +
+        `\n\nFix the crash cause, then revive the node if the work still needs doing.`;
     await pushUrgent(meta.node_id, body, { from: meta.node_id });
 }
 /** Surface a recovery refusal that happened before a broker could launch. */
@@ -121,6 +127,7 @@ export class DaemonFleet {
     #epoch;
     #enqueue;
     #now;
+    #probeOnline;
     #map = new Map();
     /** Slots claimed by launches that have not registered a broker yet. */
     #reservations = new Set();
@@ -129,6 +136,7 @@ export class DaemonFleet {
         this.#epoch = opts.epoch;
         this.#enqueue = opts.enqueue;
         this.#now = opts.now ?? Date.now;
+        this.#probeOnline = opts.probeOnline ?? (() => probeOnlineReal());
     }
     epoch() {
         return this.#epoch;
@@ -292,8 +300,30 @@ export class DaemonFleet {
         const row = getRow(nodeId);
         if (row === null)
             return;
-        const assess = row.respawn_not_before == null;
         const priorFailures = row.respawn_failures ?? 0;
+        // A box with no network fails every relaunch for a reason that has nothing
+        // to do with this node: with no route reachable, the broker's provider queue
+        // exhausts with no HTTP response at all, it declares its own transport
+        // poisoned, and it exits for a fresh process that fails identically. Six of
+        // those and the node is dead a few minutes into an outage, holding an
+        // unfinished turn that only a human revive resumes. Park instead of counting
+        // — the outage is a condition to wait out, not a failure sequence to pace.
+        if (!(await this.#probeOnline())) {
+            deferRespawnWhileOffline(nodeId, priorFailures, this.#now());
+            emitEvent({
+                level: 'info',
+                event: 'broker.respawn.offline_deferred',
+                ...(isSafeNodeId(nodeId) ? { node_id: nodeId } : {}),
+                fields: {
+                    ...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
+                    action,
+                    retry_in_ms: OFFLINE_RESPAWN_RETRY_MS,
+                    consecutive_failures: priorFailures,
+                },
+            });
+            return;
+        }
+        const assess = row.respawn_not_before == null;
         const attempt = assess
             ? recordRespawnPolicy(row, this.#now())
             : { failures: priorFailures, notBefore: row.respawn_not_before, exhausted: priorFailures >= MAX_RESPAWN_ATTEMPTS };

package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.d.ts

@@ -1,5 +1,11 @@
 import { type NodeRow } from '../../../core/canvas/index.js';
 export declare const MAX_RESPAWN_ATTEMPTS = 6;
+/** How long an offline box waits before re-testing the network and relaunching.
+ *  Flat, not exponential: this is not a failure sequence being paced, it is a
+ *  wait for a condition outside the node — the same "my wifi is back" gate the
+ *  broker's connection-fault retry uses, sampled less often because each sample
+ *  here can cost a whole broker launch. */
+export declare const OFFLINE_RESPAWN_RETRY_MS = 30000;
 export declare function respawnBackoffMs(consecutiveFailures: number): number;
 export interface RespawnAttempt {
     failures: number;
@@ -11,6 +17,14 @@ export declare function isInBackoff(row: NodeRow, now: number): boolean;
 /** The only respawn-failure policy: progress since launch clears the sequence;
  * every other missing-engine observation advances it and records its backoff. */
 export declare function decideRespawnAttempt(row: Pick<NodeRow, 'last_settled_at' | 'launched_at' | 'respawn_failures'>, now: number): RespawnAttempt;
+/** Hold a relaunch while the box has no network, WITHOUT advancing the failure
+ *  sequence. An outage makes every launch fail identically, so counting those
+ *  launches spends the whole respawn budget on a condition the node did not
+ *  cause and cannot fix — the node terminalizes minutes into an outage and only
+ *  a human revive brings it back. Re-arming instead parks it: the row stays
+ *  active, the tick re-enters this policy after the window, and the first
+ *  launch made once the network returns resumes the interrupted turn. */
+export declare function deferRespawnWhileOffline(nodeId: string, failures: number, now: number): RespawnAttempt;
 /** Persist the policy decision through the daemon-only canvas writer. */
 export declare function recordRespawnPolicy(row: NodeRow, now: number): RespawnAttempt;
 export declare function clearRespawnPolicy(nodeId: string): void;

package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.js

@@ -2,6 +2,12 @@ import { recordRespawnAttempt } from '../../../core/canvas/index.js';
 export const MAX_RESPAWN_ATTEMPTS = 6;
 const RESPAWN_BACKOFF_BASE_MS = 5_000;
 const RESPAWN_BACKOFF_MAX_MS = 10 * 60_000;
+/** How long an offline box waits before re-testing the network and relaunching.
+ *  Flat, not exponential: this is not a failure sequence being paced, it is a
+ *  wait for a condition outside the node — the same "my wifi is back" gate the
+ *  broker's connection-fault retry uses, sampled less often because each sample
+ *  here can cost a whole broker launch. */
+export const OFFLINE_RESPAWN_RETRY_MS = 30_000;
 export function respawnBackoffMs(consecutiveFailures) {
     if (consecutiveFailures <= 0)
         return 0;
@@ -30,6 +36,18 @@ export function decideRespawnAttempt(row, now) {
         exhausted: failures >= MAX_RESPAWN_ATTEMPTS,
     };
 }
+/** Hold a relaunch while the box has no network, WITHOUT advancing the failure
+ *  sequence. An outage makes every launch fail identically, so counting those
+ *  launches spends the whole respawn budget on a condition the node did not
+ *  cause and cannot fix — the node terminalizes minutes into an outage and only
+ *  a human revive brings it back. Re-arming instead parks it: the row stays
+ *  active, the tick re-enters this policy after the window, and the first
+ *  launch made once the network returns resumes the interrupted turn. */
+export function deferRespawnWhileOffline(nodeId, failures, now) {
+    const notBefore = new Date(now + OFFLINE_RESPAWN_RETRY_MS).toISOString();
+    recordRespawnAttempt(nodeId, { failures, notBefore });
+    return { failures, notBefore, exhausted: false };
+}
 /** Persist the policy decision through the daemon-only canvas writer. */
 export function recordRespawnPolicy(row, now) {
     const attempt = decideRespawnAttempt(row, now);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@north-light/crouter",
-  "version": "0.3.278",
+  "version": "0.3.279",
   "description": "crtr — agent runtime with memory, plugins, and marketplaces",
   "type": "module",
   "main": "dist/index.js",

package/runtime.lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@north-light/crouter",
-  "version": "0.3.278",
+  "version": "0.3.279",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@north-light/crouter",
-      "version": "0.3.278",
+      "version": "0.3.279",
       "hasInstallScript": true,
       "license": "MIT",
       "dependencies": {