@north-light/crouter 0.3.302 → 0.3.304

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/dist/api/__tests__/integration/client.test.js +37 -36
  2. package/dist/api/client.d.ts +32 -55
  3. package/dist/api/client.js +102 -98
  4. package/dist/api/dto/messages.d.ts +6 -5
  5. package/dist/api/errors.d.ts +3 -0
  6. package/dist/api/errors.js +10 -0
  7. package/dist/api/index.d.ts +1 -1
  8. package/dist/api/index.js +1 -1
  9. package/dist/builtin-memory/internal/plugins.md +1 -1
  10. package/dist/clients/attach/viewer.js +532 -532
  11. package/dist/commands/__tests__/seam/daemon-status.test.d.ts +1 -0
  12. package/dist/commands/__tests__/seam/daemon-status.test.js +29 -0
  13. package/dist/commands/api-client.d.ts +3 -3
  14. package/dist/commands/api-client.js +3 -3
  15. package/dist/commands/memory/read.js +2 -1
  16. package/dist/commands/node/bash.js +6 -6
  17. package/dist/commands/node/message.js +3 -3
  18. package/dist/commands/sys/daemon.js +11 -13
  19. package/dist/core/__tests__/bash-guard.test.d.ts +1 -0
  20. package/dist/core/__tests__/bash-guard.test.js +190 -0
  21. package/dist/core/__tests__/broker-stream-watchdog-floor.test.js +0 -2
  22. package/dist/core/__tests__/daemon-boot.test.js +1 -1
  23. package/dist/core/__tests__/fixtures/fake-engine.d.ts +6 -0
  24. package/dist/core/__tests__/fixtures/fake-engine.js +51 -11
  25. package/dist/core/__tests__/helpers/harness.d.ts +2 -0
  26. package/dist/core/__tests__/helpers/harness.js +9 -0
  27. package/dist/core/__tests__/integration/worktree-land.test.js +50 -0
  28. package/dist/core/__tests__/integration/worktree-reap.test.js +184 -5
  29. package/dist/core/__tests__/parse-argv-stdin-secret.test.js +14 -0
  30. package/dist/core/__tests__/seam/broker-attach-stream.test.js +13 -0
  31. package/dist/core/__tests__/seam/broker-provider-retry.test.js +57 -0
  32. package/dist/core/__tests__/seam/broker-startup-diagnostics.test.d.ts +1 -0
  33. package/dist/core/__tests__/seam/broker-startup-diagnostics.test.js +82 -0
  34. package/dist/core/bash-guard.d.ts +6 -0
  35. package/dist/core/bash-guard.js +393 -0
  36. package/dist/core/bash-jobs.d.ts +20 -8
  37. package/dist/core/bash-jobs.js +40 -21
  38. package/dist/core/canvas/types.d.ts +4 -2
  39. package/dist/core/command.js +1 -1
  40. package/dist/core/fault-classifier.d.ts +1 -1
  41. package/dist/core/runtime/broker/event-projection.d.ts +0 -3
  42. package/dist/core/runtime/broker/event-projection.js +2 -15
  43. package/dist/core/runtime/broker/fault-retry.d.ts +4 -0
  44. package/dist/core/runtime/broker/fault-retry.js +74 -6
  45. package/dist/core/runtime/broker-persona-guidance.js +12 -0
  46. package/dist/core/runtime/broker.js +0 -5
  47. package/dist/core/runtime/fault.js +1 -1
  48. package/dist/core/runtime/host.js +10 -1
  49. package/dist/core/runtime/spawn.js +5 -6
  50. package/dist/core/shell-segments.d.ts +42 -0
  51. package/dist/core/shell-segments.js +169 -0
  52. package/dist/core/substrate/surface-match.d.ts +0 -9
  53. package/dist/core/substrate/surface-match.js +4 -160
  54. package/dist/core/worktree-close.d.ts +4 -0
  55. package/dist/core/worktree-close.js +225 -0
  56. package/dist/core/worktree-containment.d.ts +14 -0
  57. package/dist/core/worktree-containment.js +48 -0
  58. package/dist/core/worktree-mutation-async.d.ts +13 -0
  59. package/dist/core/worktree-mutation-async.js +281 -0
  60. package/dist/core/worktree-sweep.js +17 -79
  61. package/dist/core/worktree.d.ts +1 -0
  62. package/dist/core/worktree.js +1 -1
  63. package/dist/daemon/__tests__/integration/api-startup-readiness.test.js +10 -8
  64. package/dist/daemon/api/__tests__/seam/api-server.test.js +70 -1
  65. package/dist/daemon/api/handlers/bash-jobs.js +1 -1
  66. package/dist/daemon/api/handlers/messages.js +7 -5
  67. package/dist/daemon/api/handlers/reports.js +3 -2
  68. package/dist/daemon/api/handlers/worktree.js +7 -6
  69. package/dist/daemon/fleet.d.ts +1 -1
  70. package/dist/daemon/fleet.js +30 -12
  71. package/dist/daemon/manage.d.ts +8 -5
  72. package/dist/daemon/manage.js +63 -40
  73. package/dist/daemon/reconcilers/managed-worktree-sweep.js +12 -0
  74. package/dist/daemon/reconcilers/node-lifecycle/tick.d.ts +0 -6
  75. package/dist/daemon/reconcilers/node-lifecycle/tick.js +1 -13
  76. package/dist/daemon/reconcilers/node-lifecycle/wants-execution.d.ts +5 -0
  77. package/dist/daemon/reconcilers/node-lifecycle/wants-execution.js +11 -0
  78. package/dist/pi-extensions/__tests__/canvas-context-intro.test.js +83 -0
  79. package/dist/pi-extensions/__tests__/integration/canvas-bash-valve.test.d.ts +1 -0
  80. package/dist/pi-extensions/__tests__/integration/canvas-bash-valve.test.js +133 -0
  81. package/dist/pi-extensions/canvas-bash-valve.d.ts +2 -3
  82. package/dist/pi-extensions/canvas-bash-valve.js +75 -77
  83. package/dist/pi-extensions/canvas-inbox-watcher.js +0 -2
  84. package/package.json +1 -1
  85. package/runtime.lock.json +5 -5
@@ -15,7 +15,8 @@ import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
15
15
  import { join } from 'node:path';
16
16
  import { getNode } from '../../../core/canvas/canvas.js';
17
17
  import { push, FinalizationError } from '../../../core/feed/feed.js';
18
- import { autoDropCleanManagedWorktree, branchIsPushedAtCurrentTip, hasOpenManagedWorktree, missingWorktreeRecoveryHint } from '../../../core/worktree.js';
18
+ import { branchIsPushedAtCurrentTip, hasOpenManagedWorktree, missingWorktreeRecoveryHint } from '../../../core/worktree.js';
19
+ import { autoDropCleanManagedWorktreeAsync } from '../../../core/worktree-mutation-async.js';
19
20
  import { reportsDir } from '../../../core/canvas/paths.js';
20
21
  import { notFound, usage } from '../../../core/errors.js';
21
22
  import { InputError } from '../../../core/io.js';
@@ -89,7 +90,7 @@ async function handlePush(ctx) {
89
90
  async function guardFinalWorktree(id) {
90
91
  if (!hasOpenManagedWorktree(getNode(id)))
91
92
  return undefined;
92
- const dropped = autoDropCleanManagedWorktree(id);
93
+ const dropped = await autoDropCleanManagedWorktreeAsync(id);
93
94
  if (dropped !== null)
94
95
  return dropped.worktree_path;
95
96
  if (await branchIsPushedAtCurrentTip(getNode(id).managed_worktree))
@@ -12,21 +12,22 @@
12
12
  // all of that onto `CrtrError` for every route that can raise it — so the CLI
13
13
  // leaf can reconstruct the original agent-facing contract instead of collapsing
14
14
  // to a generic status code.
15
- import { abandonManagedWorktree, closeManagedWorktree } from '../../../core/worktree.js';
15
+ import { closeManagedWorktreeAsync } from '../../../core/worktree-close.js';
16
+ import { abandonManagedWorktreeAsync } from '../../../core/worktree-mutation-async.js';
16
17
  import { listQuarantinedManagedWorktrees } from '../../../core/worktree-quarantine.js';
17
18
  import { usage } from '../../../core/errors.js';
18
- function handleWorktreeClose(ctx) {
19
+ async function handleWorktreeClose(ctx) {
19
20
  const id = ctx.params['id'];
20
- // No requireMeta: closeManagedWorktree already throws the precise
21
+ // No requireMeta: closeManagedWorktreeAsync already throws the precise
21
22
  // `unknown_node` / `no_managed_worktree` WorktreeError for a bad id.
22
- return { status: 200, body: closeManagedWorktree(id) };
23
+ return { status: 200, body: await closeManagedWorktreeAsync(id) };
23
24
  }
24
- function handleWorktreeAbandon(ctx) {
25
+ async function handleWorktreeAbandon(ctx) {
25
26
  const rawBy = ctx.body?.['by'];
26
27
  const by = typeof rawBy === 'string' ? rawBy.trim() : '';
27
28
  if (by === '')
28
29
  throw usage('abandon requires a non-empty invoking identity', { received: ctx.body, next: 'Pass the invoking node id or human identity.' });
29
- return { status: 200, body: abandonManagedWorktree(ctx.params['id'], by) };
30
+ return { status: 200, body: await abandonManagedWorktreeAsync(ctx.params['id'], by) };
30
31
  }
31
32
  async function handleQuarantinedWorktrees() {
32
33
  return { status: 200, body: await listQuarantinedManagedWorktrees() };
@@ -2,7 +2,7 @@ import { type NodeMeta } from '../core/canvas/index.js';
2
2
  import { type FleetEntry, type FleetExitStatus, type FleetRegistry } from '../core/runtime/fleet.js';
3
3
  import type { NodeOutcomeDetailV1, TerminalReason } from '../core/canvas/types.js';
4
4
  import type { HostHandle } from '../core/runtime/host.js';
5
- export declare function surfaceBootFailure(meta: NodeMeta): Promise<void>;
5
+ export declare function surfaceBootFailure(meta: NodeMeta, exitStatus?: FleetExitStatus): Promise<void>;
6
6
  /** Whether a dead relaunch should resume the saved session strictly, or retry
7
7
  * as a fresh cycle instead. A cycling attempt (cycle_pending) that dies before
8
8
  * session_start must retry AS a cycle: the fresh branch it was opening is not
@@ -34,16 +34,18 @@ import { brokerThresholdsForDaemon } from '../core/runtime/fleet.js';
34
34
  * When the daemon observes that death it errors LOUDLY up the spine: an
35
35
  * urgent push so the parent learns the child failed to launch instead of just
36
36
  * seeing a silent `dead`. */
37
- function bootFailureBody(meta) {
37
+ function bootFailureBody(meta, exitStatus) {
38
+ const status = exitStatus === undefined
39
+ ? 'The daemon did not observe an exit status.'
40
+ : `The observed exit status was code ${exitStatus.code ?? 'null'}, signal ${exitStatus.signal ?? 'null'}.`;
38
41
  return (`⚠ Spawn failed — \`${meta.name}\` (${meta.kind}) never started.\n\n` +
39
42
  `Its pi vehicle exited before the session came up (no pi_session_id was ever ` +
40
- `recorded), so the node produced no output. This is almost always a transient ` +
41
- `launch failure — e.g. resource pressure when several nodes boot at once — not ` +
42
- `a fault in the task itself.\n\n` +
43
- `If the work still needs doing, re-spawn it; if spawns keep dying, spawn fewer at a time.`);
43
+ `recorded), so the node produced no output. ${status}\n\n` +
44
+ `Inspect canonical event diagnostics with \`crtr sys logs --node ${meta.node_id}\`.\n\n` +
45
+ `If the work still needs doing, re-spawn it.`);
44
46
  }
45
- export async function surfaceBootFailure(meta) {
46
- await pushUrgent(meta.node_id, bootFailureBody(meta), { from: meta.node_id });
47
+ export async function surfaceBootFailure(meta, exitStatus) {
48
+ await pushUrgent(meta.node_id, bootFailureBody(meta, exitStatus), { from: meta.node_id });
47
49
  }
48
50
  async function surfaceCrashLoop(meta) {
49
51
  const transportPoison = readTransportPoisonVerdict(meta.node_id);
@@ -137,6 +139,8 @@ export class DaemonFleet {
137
139
  /** Brokers deliberately interrupted by profile pause. This belongs to the
138
140
  * exact child, so a resume cannot turn its later SIGTERM exit into a crash. */
139
141
  #profilePauseInterruptions = new Set();
142
+ /** Exit status held only until dead-row policy surfaces this broker's boot failure. */
143
+ #exitStatuses = new Map();
140
144
  #lastCapacityLogAt = Number.NEGATIVE_INFINITY;
141
145
  constructor(opts) {
142
146
  this.#epoch = opts.epoch;
@@ -155,6 +159,7 @@ export class DaemonFleet {
155
159
  throw new Error(`fleet: refusing to register ${nodeId} with no pid`);
156
160
  }
157
161
  this.#reservations.delete(nodeId);
162
+ this.#exitStatuses.delete(nodeId);
158
163
  this.#map.set(nodeId, { pid: handle.pid, handle });
159
164
  }
160
165
  has(nodeId) {
@@ -215,6 +220,7 @@ export class DaemonFleet {
215
220
  forget(nodeId) {
216
221
  this.#map.delete(nodeId);
217
222
  this.#reservations.delete(nodeId);
223
+ this.#exitStatuses.delete(nodeId);
218
224
  }
219
225
  onChildExit(nodeId, status) {
220
226
  // The real child exit is the liveness boundary. Its queued observation only
@@ -225,6 +231,10 @@ export class DaemonFleet {
225
231
  const uptimeMs = Number.isFinite(launchedMs) ? Math.max(0, this.#now() - launchedMs) : null;
226
232
  const consecutiveFailures = meta?.respawn_failures ?? 0;
227
233
  this.#map.delete(nodeId);
234
+ if (meta?.status === 'done' || meta?.status === 'canceled' || meta?.status === 'dead')
235
+ this.#exitStatuses.delete(nodeId);
236
+ else
237
+ this.#exitStatuses.set(nodeId, status);
228
238
  this.#enqueue(() => this.#emitExitObserved(nodeId, status, uptimeMs, consecutiveFailures));
229
239
  }
230
240
  deliverExit(nodeId, status) {
@@ -232,7 +242,14 @@ export class DaemonFleet {
232
242
  }
233
243
  /** The node-lifecycle tick's enactment entry. */
234
244
  async applyDeadRowPolicy(nodeId) {
235
- await this.#applyPolicy(nodeId);
245
+ const exitStatus = this.#exitStatuses.get(nodeId);
246
+ try {
247
+ await this.#applyPolicy(nodeId, exitStatus);
248
+ }
249
+ finally {
250
+ if (this.#exitStatuses.get(nodeId) === exitStatus)
251
+ this.#exitStatuses.delete(nodeId);
252
+ }
236
253
  }
237
254
  /** A revive refusal before its launch try/catch leaves the row active but
238
255
  * with no fleet entry. Convert that otherwise-silent strand into the same
@@ -267,10 +284,10 @@ export class DaemonFleet {
267
284
  },
268
285
  });
269
286
  }
270
- async #applyPolicy(nodeId) {
271
- return this.#decideAndEnact(nodeId);
287
+ async #applyPolicy(nodeId, exitStatus) {
288
+ return this.#decideAndEnact(nodeId, exitStatus);
272
289
  }
273
- async #decideAndEnact(nodeId) {
290
+ async #decideAndEnact(nodeId, exitStatus) {
274
291
  const meta = getNode(nodeId);
275
292
  if (meta !== null && isProfilePaused(meta.profile_id)) {
276
293
  clearRespawnPolicy(nodeId);
@@ -319,9 +336,10 @@ export class DaemonFleet {
319
336
  await this.#terminalize(nodeId, meta, {
320
337
  event: 'broker.boot.failed',
321
338
  reason: 'boot_failed',
322
- detail: { schema: 'crtr.node-outcome-detail/v1', message: bootFailureBody(meta) },
339
+ detail: { schema: 'crtr.node-outcome-detail/v1', message: bootFailureBody(meta, exitStatus) },
323
340
  wake: `Child crashed — ${fullName(meta)} (${nodeId}) exited before its session ever came up and is now dead. ` +
324
341
  `It stays dead until you revive it — \`crtr node lifecycle revive ${nodeId}\`.`,
342
+ surface: () => surfaceBootFailure(meta, exitStatus),
325
343
  });
326
344
  break;
327
345
  case 'respawn-fresh':
@@ -50,12 +50,14 @@ export interface SpawnDaemonResult {
50
50
  running?: true;
51
51
  /** Present with `running:true` when the Unix API socket is not serving. */
52
52
  serving?: false;
53
- /** Recovery instruction for a live but non-serving daemon. */
54
- next?: string;
53
+ /** The final `/healthz` probe failure when a live owner misses readiness. */
54
+ probe_error?: string;
55
55
  }
56
- /** True only when the mandatory Unix API socket answers `/healthz`. This is
57
- * deliberately separate from `isDaemonRunning()`: a pidfile remains useful for
58
- * identifying the process an operator must stop, but it is not readiness. */
56
+ /** Make one `/healthz` observation without client-side recovery. The caller
57
+ * owns the availability window, so this probe cannot extend it. */
58
+ export declare function probeDaemonServing(timeoutMs?: number): Promise<void>;
59
+ /** True only when one mandatory Unix API socket probe answers `/healthz`.
60
+ * A pidfile identifies an owner; it is never readiness. */
59
61
  export declare function isDaemonServing(): Promise<boolean>;
60
62
  export interface DaemonWaitDeps {
61
63
  readPidfile?: () => number | null;
@@ -69,6 +71,7 @@ export interface DaemonStartupDeps extends DaemonWaitDeps {
69
71
  signal: NodeJS.Signals | null;
70
72
  } | null;
71
73
  isDaemonServing?: () => Promise<boolean>;
74
+ probeDaemonServing?: (timeoutMs: number) => Promise<void>;
72
75
  }
73
76
  /** Thrown when the spawned daemon exited on `DAEMON_EXIT_STARTUP_BLOCKED` — a
74
77
  * standing condition (a blocked on-disk migration) that the very next spawn
@@ -19,7 +19,7 @@ import { clearDaemonStartupBlocked, daemonStartupBlocked } from './startup-block
19
19
  import { exclusiveLockOwnerPid } from '../core/exclusive-lock.js';
20
20
  import { onDiskMigrationLockPath } from '../core/canvas/paths.js';
21
21
  import { envNoDaemonAutostart } from '../shared/env.js';
22
- import { CrtrClient } from '../api/index.js';
22
+ import { CrtrClient, waitForDaemonAvailability } from '../api/index.js';
23
23
  // Daemon env sanitization
24
24
  /** Env keys that must NEVER reach the daemon process. Restarting crtrd is
25
25
  * overwhelmingly done from inside an agent node's own bash tool (`crtr sys
@@ -208,16 +208,20 @@ const DAEMON_VERIFY_POLL_MS = 10;
208
208
  /** A status/start readiness probe must not hang behind a daemon that accepts a
209
209
  * Unix connection but never answers it. */
210
210
  const DAEMON_HEALTH_PROBE_TIMEOUT_MS = 1_000;
211
- /** True only when the mandatory Unix API socket answers `/healthz`. This is
212
- * deliberately separate from `isDaemonRunning()`: a pidfile remains useful for
213
- * identifying the process an operator must stop, but it is not readiness. */
211
+ /** Make one `/healthz` observation without client-side recovery. The caller
212
+ * owns the availability window, so this probe cannot extend it. */
213
+ export async function probeDaemonServing(timeoutMs = DAEMON_HEALTH_PROBE_TIMEOUT_MS) {
214
+ const probeTimeoutMs = Math.min(timeoutMs, DAEMON_HEALTH_PROBE_TIMEOUT_MS);
215
+ const health = await CrtrClient.forLocalSocket({ autostart: false, timeoutMs: probeTimeoutMs }).probeHealthz(probeTimeoutMs);
216
+ if (!health.ok)
217
+ throw new Error('crtrd health check reported not ready');
218
+ }
219
+ /** True only when one mandatory Unix API socket probe answers `/healthz`.
220
+ * A pidfile identifies an owner; it is never readiness. */
214
221
  export async function isDaemonServing() {
215
222
  try {
216
- const health = await CrtrClient.forLocalSocket({
217
- autostart: false,
218
- timeoutMs: DAEMON_HEALTH_PROBE_TIMEOUT_MS,
219
- }).healthz();
220
- return health.ok;
223
+ await probeDaemonServing();
224
+ return true;
221
225
  }
222
226
  catch {
223
227
  return false;
@@ -255,32 +259,47 @@ export async function verifyDaemonStartup(pid, timeoutMs = DAEMON_STARTUP_WINDOW
255
259
  const read = deps.readPidfile ?? readPidfile;
256
260
  const alive = deps.isPidAlive ?? isPidAlive;
257
261
  const exited = deps.childExited;
258
- const serving = deps.isDaemonServing ?? isDaemonServing;
262
+ const serving = deps.isDaemonServing;
263
+ const probeServing = deps.probeDaemonServing ?? (async (remainingMs) => {
264
+ if (serving !== undefined) {
265
+ if (await serving())
266
+ return;
267
+ throw new Error('crtrd health check reported not ready');
268
+ }
269
+ await probeDaemonServing(remainingMs);
270
+ });
259
271
  const sleep = deps.sleepMs ?? sleepMs;
260
272
  const now = deps.now ?? Date.now;
261
- const deadline = now() + timeoutMs;
262
- while (now() <= deadline) {
263
- const owner = read();
264
- if (owner !== null && alive(owner) && await serving())
265
- return owner === pid ? null : owner;
266
- const exitState = exited?.();
267
- if (exitState !== null && exitState !== undefined) {
268
- if (exitState.signal !== null) {
269
- throw new Error(`daemon ${pid} exited before becoming ready (by ${exitState.signal})`);
270
- }
271
- if (exitState.code === DAEMON_EXIT_STARTUP_BLOCKED) {
272
- throw new DaemonStartupBlockedError(pid);
273
- }
274
- if (exitState.code !== 0) {
275
- throw new Error(`daemon ${pid} exited before becoming ready (with exit code ${exitState.code ?? '?'})`);
273
+ let owner = null;
274
+ let terminalError = null;
275
+ await waitForDaemonAvailability({
276
+ windowMs: timeoutMs,
277
+ pollIntervalMs: DAEMON_VERIFY_POLL_MS,
278
+ now,
279
+ sleep,
280
+ retry: (error) => error !== terminalError,
281
+ probe: async (remainingMs) => {
282
+ const candidate = read();
283
+ if (candidate === null || !alive(candidate)) {
284
+ const exitState = exited?.();
285
+ if (exitState !== null && exitState !== undefined) {
286
+ if (exitState.signal !== null)
287
+ terminalError = new Error(`daemon ${pid} exited before becoming ready (by ${exitState.signal})`);
288
+ else if (exitState.code === DAEMON_EXIT_STARTUP_BLOCKED)
289
+ terminalError = new DaemonStartupBlockedError(pid);
290
+ else if (exitState.code !== 0)
291
+ terminalError = new Error(`daemon ${pid} exited before becoming ready (with exit code ${exitState.code ?? '?'})`);
292
+ // A clean exit can have lost the ownership claim, so keep observing.
293
+ if (terminalError !== null)
294
+ throw terminalError;
295
+ }
296
+ throw new Error(`daemon ${pid} has no live pidfile owner`);
276
297
  }
277
- // Clean code-0 exit: a possible ownership-claim loser, not readiness.
278
- // Keep polling the same deadline for a different live pidfile owner; the
279
- // loop's timeout throw below fires if none appears.
280
- }
281
- await sleep(DAEMON_VERIFY_POLL_MS);
282
- }
283
- throw new Error(`daemon ${pid} did not become ready within ${timeoutMs}ms`);
298
+ await probeServing(remainingMs);
299
+ owner = candidate;
300
+ },
301
+ });
302
+ return owner === pid ? null : owner;
284
303
  }
285
304
  /** Wait until a signaled daemon has exited and relinquished its pidfile. This
286
305
  * makes a following start safe to claim the singleton rather than racing its
@@ -355,15 +374,19 @@ export async function stopDaemonProcess(pid, timeoutMs = DAEMON_SHUTDOWN_WINDOW_
355
374
  export async function spawnDaemon() {
356
375
  const recordedPid = readPidfile();
357
376
  if (recordedPid !== null && isPidAlive(recordedPid)) {
358
- if (await isDaemonServing())
377
+ try {
378
+ await probeDaemonServing();
359
379
  return { started: false, existing_pid: recordedPid };
360
- return {
361
- started: false,
362
- existing_pid: recordedPid,
363
- running: true,
364
- serving: false,
365
- next: 'Run `crtr sys daemon stop` to release the non-serving daemon before starting it again.',
366
- };
380
+ }
381
+ catch (error) {
382
+ return {
383
+ started: false,
384
+ existing_pid: recordedPid,
385
+ running: true,
386
+ serving: false,
387
+ probe_error: error.message,
388
+ };
389
+ }
367
390
  }
368
391
  // Ensure the canvas home directory exists so the daemon can write its pidfile.
369
392
  mkdirSync(crtrHome(), { recursive: true });
@@ -30,6 +30,12 @@ const TERMINAL_STATUSES = ['done', 'dead', 'canceled'];
30
30
  function isTerminal(row) {
31
31
  return TERMINAL_STATUSES.includes(row.status);
32
32
  }
33
+ /** A closed record already has an explicit close disposition. An open record
34
+ * reaches the sweep only when its owner published a final result; a crash or
35
+ * cancellation leaves its managed checkout available for explicit recovery. */
36
+ function hasSweepDisposition(row, wt) {
37
+ return wt?.state === 'closed' || (wt?.state === 'open' && row.status === 'done' && row.final_report !== null);
38
+ }
33
39
  /** Is this node's engine definitively gone?
34
40
  *
35
41
  * Terminal status alone is not enough: `push final` marks a node `done` while
@@ -49,6 +55,8 @@ function engineIsGone(row, fleet, snapshot) {
49
55
  return false;
50
56
  if (!isTerminal(row))
51
57
  return false;
58
+ if (!hasSweepDisposition(row, getNode(row.node_id)?.managed_worktree))
59
+ return false;
52
60
  return recordedPidLiveness(row.pi_pid, row.pi_pid_identity, snapshot) === 'dead';
53
61
  }
54
62
  /** The eligibility recheck the reconciliation runs immediately before EVERY Git
@@ -70,6 +78,8 @@ function eligibilityGuard(row, fleet) {
70
78
  const current = getRow(nodeId);
71
79
  if (current === null || !isTerminal(current) || current.frozen_at !== null)
72
80
  return false;
81
+ if (!hasSweepDisposition(current, getNode(nodeId)?.managed_worktree))
82
+ return false;
73
83
  return current.pi_pid === pid && current.pi_pid_identity === identity;
74
84
  },
75
85
  };
@@ -120,6 +130,8 @@ export class ManagedWorktreeSweepReconciler {
120
130
  }
121
131
  if (!isReconcilable(wt) || wt == null)
122
132
  continue;
133
+ if (!hasSweepDisposition(row, wt))
134
+ continue;
123
135
  if (!isDueForSweep(wt, now))
124
136
  continue;
125
137
  due.push(row.node_id);
@@ -7,12 +7,6 @@ export interface NodeLifecycleContext {
7
7
  rows: readonly NodeRow[];
8
8
  fleet: FleetRegistry;
9
9
  }
10
- /** Whether this row should have a live engine right now. An active row always
11
- * should — that status IS the claim that it is executing. An idle row was
12
- * deliberately released and holds no engine on purpose: only mail it has not
13
- * seen brings one back, because a live obligation keeps a node WAKEABLE rather
14
- * than awake, and relaunching on one would re-release it every tick. */
15
- export declare function wantsExecution(row: NodeRow): boolean;
16
10
  export declare class NodeLifecycleReconciler {
17
11
  private readonly freezeLane;
18
12
  run(now: number, ctx: NodeLifecycleContext): Promise<FrozenCounts>;
@@ -12,7 +12,6 @@ import { isSafeNodeId } from '../../../core/canvas/paths.js';
12
12
  import { recordedPidLiveness } from '../../../core/canvas/pid.js';
13
13
  import { emitEvent } from '../../../core/events/emit.js';
14
14
  import { operationIdContext } from '../../../core/events/operation-id.js';
15
- import { hasUnseenWakeMail } from '../../../core/feed/inbox.js';
16
15
  import { hasCleanAbort, isBusy } from '../../../core/runtime/busy.js';
17
16
  import { isProfilePaused } from '../../../core/profiles/manifest.js';
18
17
  import { classifyDeadNode } from '../../fleet.js';
@@ -20,21 +19,10 @@ import { applyParkEvent, hasPendingWake, parkEvent } from '../live-obligation.js
20
19
  import { FreezeLaneReconciler } from './freeze-lane.js';
21
20
  import { isInBackoff } from './respawn-policy.js';
22
21
  import { handleTerminatingRow } from './terminating.js';
22
+ import { wantsExecution } from './wants-execution.js';
23
23
  function isTerminal(row) {
24
24
  return row.status === 'done' || row.status === 'canceled' || row.status === 'dead';
25
25
  }
26
- /** Whether this row should have a live engine right now. An active row always
27
- * should — that status IS the claim that it is executing. An idle row was
28
- * deliberately released and holds no engine on purpose: only mail it has not
29
- * seen brings one back, because a live obligation keeps a node WAKEABLE rather
30
- * than awake, and relaunching on one would re-release it every tick. */
31
- export function wantsExecution(row) {
32
- if (row.status === 'active')
33
- return true;
34
- if (row.status !== 'idle')
35
- return false;
36
- return hasUnseenWakeMail(row.node_id);
37
- }
38
26
  export class NodeLifecycleReconciler {
39
27
  freezeLane = new FreezeLaneReconciler();
40
28
  async run(now, ctx) {
@@ -0,0 +1,5 @@
1
+ import type { NodeRow } from '../../../core/canvas/index.js';
2
+ /** Whether the lifecycle tick must ensure this row has an engine. The durable
3
+ * message handler calls this after appending its entry: that append makes the
4
+ * idle branch's unseen-wake-mail check true for a wake-capable entry. */
5
+ export declare function wantsExecution(row: Pick<NodeRow, 'node_id' | 'status'>): boolean;
@@ -0,0 +1,11 @@
1
+ import { hasUnseenWakeMail } from '../../../core/feed/inbox.js';
2
+ /** Whether the lifecycle tick must ensure this row has an engine. The durable
3
+ * message handler calls this after appending its entry: that append makes the
4
+ * idle branch's unseen-wake-mail check true for a wake-capable entry. */
5
+ export function wantsExecution(row) {
6
+ if (row.status === 'active')
7
+ return true;
8
+ if (row.status !== 'idle')
9
+ return false;
10
+ return hasUnseenWakeMail(row.node_id);
11
+ }
@@ -5,12 +5,16 @@
5
5
  import { test, before, after, beforeEach, afterEach } from 'node:test';
6
6
  import assert from 'node:assert/strict';
7
7
  import { existsSync, mkdtempSync, mkdirSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
8
+ import { createServer } from 'node:http';
8
9
  import { tmpdir } from 'node:os';
9
10
  import { join } from 'node:path';
10
11
  import { registerCanvasContextIntro, CONTEXT_INTRO_CUSTOM_TYPE, } from '../canvas-context-intro.js';
11
12
  import { registerCanvasDocSubstrate } from '../canvas-doc-substrate.js';
12
13
  import { buildContextBearings as buildContextIntro } from '../../core/runtime/bearings.js';
13
14
  import { installPersonaTransitionGate } from '../../core/runtime/broker-persona-guidance.js';
15
+ import { FaultRetry } from '../../core/runtime/broker/fault-retry.js';
16
+ import { readFault } from '../../core/runtime/fault.js';
17
+ import { ApiError } from '../../api/errors.js';
14
18
  import { createNode, deleteNode, getNode, updateNode } from '../../core/canvas/canvas.js';
15
19
  import { apiSocketPath } from '../../core/canvas/paths.js';
16
20
  import { createApiServer } from '../../daemon/api/server.js';
@@ -32,6 +36,23 @@ async function waitForSocket(path) {
32
36
  }
33
37
  throw new Error(`api socket never bound at ${path}`);
34
38
  }
39
+ async function listenUnavailableApi(path) {
40
+ const unavailable = createServer((_request, response) => {
41
+ response.writeHead(503, { 'content-type': 'application/json' });
42
+ response.end(JSON.stringify({ error: { code: 'daemon_unavailable', message: 'local daemon connection refused' } }));
43
+ });
44
+ await new Promise((resolve, reject) => {
45
+ unavailable.once('error', reject);
46
+ unavailable.listen(path, () => {
47
+ unavailable.off('error', reject);
48
+ resolve();
49
+ });
50
+ });
51
+ return unavailable;
52
+ }
53
+ function closeHttpServer(server) {
54
+ return new Promise((resolve, reject) => server.close((error) => error === undefined ? resolve() : reject(error)));
55
+ }
35
56
  function node(id, over = {}) {
36
57
  return {
37
58
  node_id: id,
@@ -322,6 +343,68 @@ test('child /new commits its baseline before ack and the provider gate delivers
322
343
  process.env['CRTR_PROFILE_ID'] = previousProfile;
323
344
  }
324
345
  });
346
+ test('a typed local API error remains a retryable broker-to-daemon fault after settlement', async () => {
347
+ createNode(node('persona-transport', {
348
+ persona_ack: { kind: 'review', mode: 'base', lifecycle: 'terminal' },
349
+ }));
350
+ await server.close();
351
+ let unavailable;
352
+ try {
353
+ unavailable = await listenUnavailableApi(apiSocketPath());
354
+ const messages = [];
355
+ const agent = {
356
+ state: { messages },
357
+ transformContext: async (context) => context,
358
+ };
359
+ installPersonaTransitionGate('persona-transport', {
360
+ agent,
361
+ sessionManager: {
362
+ appendCustomMessageEntry: () => undefined,
363
+ },
364
+ });
365
+ await assert.rejects(() => agent.transformContext([{ role: 'user', content: 'continue' }]), (error) => error instanceof ApiError && error.status === 503 && error.code === 'daemon_unavailable');
366
+ assert.deepEqual(readFault('persona-transport'), {
367
+ link: 'broker↔crtrd',
368
+ op: 'persona transition',
369
+ kind: 'connection',
370
+ retry: { disposition: 'manual' },
371
+ message: 'local daemon connection refused',
372
+ since: readFault('persona-transport')?.since,
373
+ operation_id: readFault('persona-transport')?.operation_id,
374
+ }, 'the typed daemon transport error is recorded before Pi turns it into a generic agent error');
375
+ const session = {
376
+ sessionManager: { getLeafId: () => 'persona-transport-leaf' },
377
+ model: undefined,
378
+ };
379
+ const generation = {
380
+ id: 1,
381
+ session,
382
+ fallback: { blocked: false },
383
+ stagedProviderAgentEnd: {
384
+ messages: [{ role: 'assistant', stopReason: 'error', errorMessage: 'local daemon connection refused' }],
385
+ },
386
+ stagedWatchdogAbort: false,
387
+ stagedRefreshAbort: false,
388
+ stagedOverflowFailure: null,
389
+ executingTools: new Set(),
390
+ };
391
+ new FaultRetry({
392
+ nodeId: 'persona-transport',
393
+ currentSession: () => session,
394
+ installedGeneration: () => generation,
395
+ }).publishSettledProviderFault(generation);
396
+ const fault = readFault('persona-transport');
397
+ assert.equal(fault?.link, 'broker↔crtrd');
398
+ assert.equal(fault?.kind, 'connection');
399
+ assert.equal(fault?.retry.disposition, 'manual');
400
+ }
401
+ finally {
402
+ if (unavailable !== undefined)
403
+ await closeHttpServer(unavailable);
404
+ server = createApiServer();
405
+ await waitForSocket(apiSocketPath());
406
+ }
407
+ });
325
408
  test('resume idempotency uses the exact details stamp independent of block text', async () => {
326
409
  createNode(node('stampD'));
327
410
  // The exact node-id stamp is authoritative even when prose carries no id.