@north-light/crouter 0.3.303 → 0.3.305

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/dist/api/__tests__/integration/client.test.js +37 -36
  2. package/dist/api/client.d.ts +32 -55
  3. package/dist/api/client.js +102 -98
  4. package/dist/api/dto/messages.d.ts +6 -5
  5. package/dist/api/errors.d.ts +3 -0
  6. package/dist/api/errors.js +10 -0
  7. package/dist/api/index.d.ts +1 -1
  8. package/dist/api/index.js +1 -1
  9. package/dist/builtin-memory/internal/plugins.md +1 -1
  10. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/__tests__/integration/provider-rotation.test.ts +87 -0
  11. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js +31 -26
  12. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +33 -26
  13. package/dist/clients/attach/viewer.js +584 -584
  14. package/dist/commands/__tests__/seam/daemon-status.test.d.ts +1 -0
  15. package/dist/commands/__tests__/seam/daemon-status.test.js +29 -0
  16. package/dist/commands/api-client.d.ts +3 -3
  17. package/dist/commands/api-client.js +3 -3
  18. package/dist/commands/memory/read.js +2 -1
  19. package/dist/commands/node/bash.js +6 -6
  20. package/dist/commands/node/create.js +7 -1
  21. package/dist/commands/node/message.js +3 -3
  22. package/dist/commands/node/wait.js +1 -1
  23. package/dist/commands/node-worktree.js +5 -1
  24. package/dist/commands/sys/daemon.js +11 -13
  25. package/dist/core/__tests__/broker-stream-watchdog-floor.test.js +0 -2
  26. package/dist/core/__tests__/daemon-boot.test.js +1 -1
  27. package/dist/core/__tests__/fixtures/fake-engine.js +14 -15
  28. package/dist/core/__tests__/helpers/harness.d.ts +2 -0
  29. package/dist/core/__tests__/helpers/harness.js +9 -0
  30. package/dist/core/__tests__/integration/tmux-surface.test.js +53 -0
  31. package/dist/core/__tests__/integration/worktree-land.test.js +50 -0
  32. package/dist/core/__tests__/integration/worktree-reap.test.js +184 -5
  33. package/dist/core/__tests__/kickoff.test.js +40 -0
  34. package/dist/core/__tests__/parse-argv-stdin-secret.test.js +14 -0
  35. package/dist/core/__tests__/seam/broker-attach-stream.test.js +13 -0
  36. package/dist/core/__tests__/seam/broker-provider-retry.test.js +25 -0
  37. package/dist/core/__tests__/seam/broker-startup-diagnostics.test.d.ts +1 -0
  38. package/dist/core/__tests__/seam/broker-startup-diagnostics.test.js +82 -0
  39. package/dist/core/__tests__/seam/deadline-wake-delivery.test.d.ts +1 -0
  40. package/dist/core/__tests__/seam/deadline-wake-delivery.test.js +138 -0
  41. package/dist/core/__tests__/seam/reopen-invalidated-worktree.test.d.ts +1 -0
  42. package/dist/core/__tests__/seam/reopen-invalidated-worktree.test.js +60 -0
  43. package/dist/core/bash-jobs.d.ts +20 -8
  44. package/dist/core/bash-jobs.js +40 -21
  45. package/dist/core/canvas/types.d.ts +4 -2
  46. package/dist/core/command.js +1 -1
  47. package/dist/core/fault-classifier.d.ts +7 -1
  48. package/dist/core/fault-classifier.js +9 -4
  49. package/dist/core/provider-failure.d.ts +30 -0
  50. package/dist/core/provider-failure.js +86 -0
  51. package/dist/core/runtime/broker/event-projection.d.ts +0 -3
  52. package/dist/core/runtime/broker/event-projection.js +2 -15
  53. package/dist/core/runtime/broker/fault-retry.js +16 -0
  54. package/dist/core/runtime/broker-persona-guidance.js +12 -0
  55. package/dist/core/runtime/broker.js +0 -5
  56. package/dist/core/runtime/fault.d.ts +2 -1
  57. package/dist/core/runtime/fault.js +39 -1
  58. package/dist/core/runtime/host.js +10 -1
  59. package/dist/core/runtime/kickoff.d.ts +3 -0
  60. package/dist/core/runtime/kickoff.js +24 -4
  61. package/dist/core/runtime/reopen.d.ts +2 -2
  62. package/dist/core/runtime/reopen.js +6 -4
  63. package/dist/core/runtime/revive.js +3 -3
  64. package/dist/core/runtime/spawn.js +5 -6
  65. package/dist/core/runtime/tmux-bindings.js +14 -3
  66. package/dist/core/termrender/version.d.ts +1 -1
  67. package/dist/core/termrender/version.js +1 -1
  68. package/dist/core/worktree-close.d.ts +4 -0
  69. package/dist/core/worktree-close.js +228 -0
  70. package/dist/core/worktree-containment.d.ts +14 -0
  71. package/dist/core/worktree-containment.js +48 -0
  72. package/dist/core/worktree-mutation-async.d.ts +13 -0
  73. package/dist/core/worktree-mutation-async.js +281 -0
  74. package/dist/core/worktree-sweep.js +24 -82
  75. package/dist/core/worktree.d.ts +1 -0
  76. package/dist/core/worktree.js +7 -4
  77. package/dist/daemon/__tests__/integration/api-startup-readiness.test.js +10 -8
  78. package/dist/daemon/api/__tests__/reopen-delivery.test.js +31 -0
  79. package/dist/daemon/api/__tests__/seam/api-server.test.js +70 -1
  80. package/dist/daemon/api/__tests__/seam/leaf-api-parity.test.js +36 -4
  81. package/dist/daemon/api/handlers/bash-jobs.js +1 -1
  82. package/dist/daemon/api/handlers/messages.js +33 -14
  83. package/dist/daemon/api/handlers/reports.js +3 -2
  84. package/dist/daemon/api/handlers/worktree.js +7 -6
  85. package/dist/daemon/fleet.d.ts +1 -1
  86. package/dist/daemon/fleet.js +30 -12
  87. package/dist/daemon/manage.d.ts +8 -5
  88. package/dist/daemon/manage.js +63 -40
  89. package/dist/daemon/reconcilers/broker-supervision.js +10 -1
  90. package/dist/daemon/reconcilers/managed-worktree-sweep.js +12 -0
  91. package/dist/daemon/reconcilers/node-lifecycle/tick.d.ts +0 -6
  92. package/dist/daemon/reconcilers/node-lifecycle/tick.js +1 -13
  93. package/dist/daemon/reconcilers/node-lifecycle/wants-execution.d.ts +5 -0
  94. package/dist/daemon/reconcilers/node-lifecycle/wants-execution.js +11 -0
  95. package/dist/pi-extensions/__tests__/canvas-context-intro.test.js +83 -0
  96. package/dist/pi-extensions/__tests__/integration/canvas-bash-valve.test.d.ts +1 -0
  97. package/dist/pi-extensions/__tests__/integration/canvas-bash-valve.test.js +133 -0
  98. package/dist/pi-extensions/canvas-bash-valve.d.ts +2 -0
  99. package/dist/pi-extensions/canvas-bash-valve.js +73 -43
  100. package/dist/pi-extensions/canvas-inbox-watcher.js +0 -2
  101. package/package.json +1 -1
  102. package/runtime.lock.json +5 -5
@@ -26,7 +26,7 @@
26
26
  import { test, before, after } from 'node:test';
27
27
  import assert from 'node:assert/strict';
28
28
  import { spawnSync } from 'node:child_process';
29
- import { statSync, existsSync, readFileSync, mkdtempSync, rmSync, mkdirSync, writeFileSync } from 'node:fs';
29
+ import { chmodSync, statSync, existsSync, readFileSync, mkdtempSync, rmSync, mkdirSync, writeFileSync } from 'node:fs';
30
30
  import { tmpdir } from 'node:os';
31
31
  import { join, dirname } from 'node:path';
32
32
  import WebSocket from 'ws';
@@ -133,6 +133,45 @@ test('A2: POST /nodes boots a live broker; GET /nodes/{id} shows active + live p
133
133
  assert.ok(meta?.pi_pid != null && isPidAlive(meta.pi_pid), 'row records a live broker pid');
134
134
  assert.ok(existsSync(h.brokerSock(id)), 'view.sock exists for the live broker');
135
135
  });
136
+ function installBlockedStartHook(nodeId) {
137
+ const root = join(h.agentHome, '.crouter', 'plugins', `blocked-start-${nodeId}`);
138
+ const started = join(root, 'started');
139
+ const release = join(root, 'release');
140
+ mkdirSync(join(root, '.crouter-plugin'), { recursive: true });
141
+ writeFileSync(join(root, '.crouter-plugin', 'plugin.json'), JSON.stringify({
142
+ name: `blocked-start-${nodeId}`,
143
+ version: '0.1.0',
144
+ description: 'message acknowledgment fixture',
145
+ hooks: 'hooks.json',
146
+ hookExecutable: 'bin/hook.js',
147
+ }));
148
+ writeFileSync(join(root, 'hooks.json'), JSON.stringify({
149
+ schemaVersion: 2,
150
+ lifecycle: [{ event: 'node:start', phase: 'on', op: 'block', description: 'block target launch', effects: ['waits for test release'] }],
151
+ }));
152
+ const executable = join(root, 'bin', 'hook.js');
153
+ mkdirSync(join(root, 'bin'), { recursive: true });
154
+ writeFileSync(executable, `#!/usr/bin/env node
155
+ const { existsSync, writeFileSync } = require('node:fs');
156
+ let input = '';
157
+ process.stdin.on('data', (chunk) => { input += chunk; });
158
+ process.stdin.on('end', () => {
159
+ const request = JSON.parse(input);
160
+ if (request.node.id !== ${JSON.stringify(nodeId)}) {
161
+ process.stdout.write(JSON.stringify({ protocolVersion: 1, ok: true }));
162
+ return;
163
+ }
164
+ writeFileSync(${JSON.stringify(started)}, 'started');
165
+ const timer = setInterval(() => {
166
+ if (!existsSync(${JSON.stringify(release)})) return;
167
+ clearInterval(timer);
168
+ process.stdout.write(JSON.stringify({ protocolVersion: 1, ok: true }));
169
+ }, 10);
170
+ });
171
+ `);
172
+ chmodSync(executable, 0o755);
173
+ return { started, release };
174
+ }
136
175
  // A3 — POST .../messages appends inbox.jsonl; POST .../reports {tier:'final'}
137
176
  // writes the report, notifies subscribers, transitions the row
138
177
  test('A3: messages append inbox.jsonl; final report writes + notifies subscriber + transitions row', async () => {
@@ -153,6 +192,36 @@ test('A3: messages append inbox.jsonl; final report writes + notifies subscriber
153
192
  assert.equal(h.node(B)?.status, 'done', 'row status is done after final');
154
193
  await assert.rejects(() => client.pushReport(B, { tier: 'final', delivery_tier: 'deferred', body: 'invalid tiered final' }), (error) => error instanceof ApiError && error.code === 'usage' && error.message === 'delivery_tier is only valid for update reports', 'the API rejects a delivery tier on final before it can alter final delivery');
155
194
  });
195
+ test('A3b: a durable send acknowledges while the lifecycle tick is blocked launching its idle target', async () => {
196
+ const target = h.fabricateBrokerNode({ lifecycle: 'resident', status: 'idle', intent: 'idle-release' });
197
+ const hook = installBlockedStartHook(target);
198
+ const inboxFile = inboxPath(target);
199
+ const before = countLines(inboxFile);
200
+ const sending = client.sendMessage(target, { body: 'wake after durable acknowledgment' });
201
+ let tick;
202
+ try {
203
+ await h.waitFor(() => countLines(inboxFile) === before + 1, { label: 'durable inbox append' });
204
+ assert.equal(h.node(target)?.cycles ?? 0, 0, 'the send has not launched the target');
205
+ tick = h.tick();
206
+ await h.waitFor(() => existsSync(hook.started), { label: 'blocked lifecycle launch' });
207
+ const acknowledged = await Promise.race([
208
+ sending,
209
+ new Promise((_, reject) => setTimeout(() => reject(new Error('durable send waited for recipient launch')), 500)),
210
+ ]);
211
+ assert.equal(acknowledged.delivered, true, 'the API acknowledges the durable append');
212
+ assert.equal(acknowledged.revived, false, 'the response honestly reports no synchronous revive');
213
+ assert.equal(countLines(inboxFile), before + 1, 'exactly one inbox entry is durable while launch is blocked');
214
+ writeFileSync(hook.release, 'release');
215
+ await tick;
216
+ assert.equal(h.node(target)?.cycles, 1, 'the one queued entry drove lifecycle revival');
217
+ assert.equal(countLines(inboxFile), before + 1, 'revival did not append a second inbox entry');
218
+ }
219
+ finally {
220
+ if (!existsSync(hook.release))
221
+ writeFileSync(hook.release, 'release');
222
+ await tick;
223
+ }
224
+ });
156
225
  // A4 — WS attach reconstructs the same settled broker snapshot as a direct
157
226
  // ViewSocketClient; ?revive=0 on a dormant node → 409 without launching a broker
158
227
  test('A4a: WS attach welcome.snapshot is byte-identical to a direct ViewSocketClient snapshot', async () => {
@@ -19,7 +19,7 @@
19
19
  // in-process server exactly as a real `crtr push` CLI process would.
20
20
  import { test, before, after } from 'node:test';
21
21
  import assert from 'node:assert/strict';
22
- import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
22
+ import { existsSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
23
23
  import { tmpdir } from 'node:os';
24
24
  import { join } from 'node:path';
25
25
  import { createHeadlessHarness } from '../../../../core/__tests__/helpers/harness.js';
@@ -87,16 +87,19 @@ async function runPushAs(caller, tier, body, input = {}) {
87
87
  }
88
88
  }
89
89
  // The `node worktree close` leaf, resolved from the real command tree exactly as
90
- // the CLI dispatcher would, and driven as the given caller node.
91
- async function runCloseAs(caller) {
90
+ // the CLI dispatcher would.
91
+ function closeLeaf() {
92
92
  const worktree = registerNode().children.find((c) => c.name === 'worktree');
93
93
  assert.ok(worktree !== undefined && worktree.kind === 'branch', 'node worktree branch resolves');
94
94
  const close = worktree.children.find((c) => c.name === 'close');
95
95
  assert.ok(close !== undefined && close.kind === 'leaf', 'node worktree close leaf resolves');
96
+ return close;
97
+ }
98
+ async function runCloseAs(caller) {
96
99
  const prev = process.env['CRTR_NODE_ID'];
97
100
  process.env['CRTR_NODE_ID'] = caller;
98
101
  try {
99
- return (await close.run({}));
102
+ return (await closeLeaf().run({}));
100
103
  }
101
104
  finally {
102
105
  if (prev === undefined)
@@ -300,6 +303,35 @@ test('`crtr push final` keeps an absent managed worktree open when its branch is
300
303
  rmSync(repo, { recursive: true, force: true });
301
304
  }
302
305
  });
306
+ test('`crtr node worktree close` warns with the retained stash SHA when base-checkout changes conflict (#1066)', async () => {
307
+ const repo = initRepo();
308
+ try {
309
+ writeFileSync(join(repo, 'conflict.txt'), 'base\n');
310
+ writeFileSync(join(repo, 'disjoint.txt'), 'base\n');
311
+ git(['add', '.'], repo);
312
+ git(['commit', '-m', 'tracked base files'], repo);
313
+ const id = 'wt-close-stash-warning';
314
+ const wt = createManagedWorktree(repo, id);
315
+ createNode(worktreeNode(id, wt.path, wt));
316
+ writeFileSync(join(wt.path, 'conflict.txt'), 'child\n');
317
+ git(['add', 'conflict.txt'], wt.path);
318
+ git(['commit', '-m', 'child conflict'], wt.path);
319
+ writeFileSync(join(repo, 'conflict.txt'), 'base live edit\n');
320
+ writeFileSync(join(repo, 'disjoint.txt'), 'disjoint live edit\n');
321
+ const out = await runCloseAs(id);
322
+ const stashSha = gitSync(['rev-parse', 'refs/stash'], repo).stdout.trim();
323
+ const rendered = closeLeaf().render?.(out) ?? '';
324
+ assert.match(out['base_checkout_stash'], new RegExp(`stash ${stashSha}`), 'the API result identifies the exact retained stash');
325
+ assert.match(rendered, new RegExp(`stash ${stashSha}`), 'the CLI output identifies the exact retained stash');
326
+ assert.match(rendered, new RegExp(`git -C '${realpathSync(repo)}' stash apply ${stashSha}`), 'the CLI output gives the exact recovery command');
327
+ assert.match(rendered, /shared base checkout was reset to landed commit/, 'the CLI output states that the shared checkout was reset');
328
+ assert.doesNotMatch(rendered, /nothing was lost/i, 'the CLI does not claim all concurrent work was preserved');
329
+ assert.equal(readFileSync(join(repo, 'disjoint.txt'), 'utf8'), 'base\n', 'the reset removes the disjoint live edit from the checkout but leaves it in the reported stash');
330
+ }
331
+ finally {
332
+ rmSync(repo, { recursive: true, force: true });
333
+ }
334
+ });
303
335
  // `node worktree close` error parity — the SPECIFIC WorktreeError code + its
304
336
  // `next` remediation round-trip verbatim through the server → ApiError → the
305
337
  // bespoke CLI mapper in node-worktree.ts, instead of collapsing to a generic
@@ -59,7 +59,7 @@ async function handleStop(ctx) {
59
59
  throw new ApiError(404, 'job_not_found', `background job ${jobId} for ${nodeId} is no longer running`, { log_path: stopped.logPath });
60
60
  }
61
61
  if (stopped.kind === 'not-stoppable') {
62
- throw new ApiError(409, 'job_not_stoppable', `job ${jobId} recorded no process group`, { log_path: stopped.logPath });
62
+ throw new ApiError(409, 'job_not_stoppable', `job ${jobId} recorded no trusted process identity`, { log_path: stopped.logPath });
63
63
  }
64
64
  // Match `crtr node bash kill`: retire the job, then send an urgent durable
65
65
  // notice to the owning node. The terminal path sends that notice through the
@@ -1,6 +1,7 @@
1
1
  // Message handler (spec §6.2). `POST /v1/nodes/{id}/messages` delivers an
2
2
  // inbox message now (201) or fresh-revives with no inbox entry. `{id}` is the
3
3
  // target node.
4
+ import { join } from 'node:path';
4
5
  import { appendInbox, cancelPendingHumanMessages } from '../../../core/feed/inbox.js';
5
6
  import { reviveNode } from '../../../core/runtime/revive.js';
6
7
  import { BrokerUnreachableError } from '../../../core/runtime/broker-request.js';
@@ -8,14 +9,16 @@ import { isBrokerLive } from '../../../core/runtime/model-swap.js';
8
9
  import { deliverLive, interruptLive } from '../../../core/runtime/interactive-deliver.js';
9
10
  import { isProfilePaused } from '../../../core/profiles/manifest.js';
10
11
  import { getNode, withFreshTerminalGuard } from '../../../core/canvas/canvas.js';
12
+ import { reportsDir } from '../../../core/canvas/paths.js';
11
13
  import { withCanvasWrite } from '../../../core/canvas/db.js';
12
14
  import { cancelCronsOnWake } from '../../../core/canvas/crons.js';
13
- import { assertNotFinalized, assertFinalizedForReopen, commitReopen, commitReopenResident, } from '../../../core/runtime/reopen.js';
15
+ import { assertNotFinalized, assertFinalizedForReopen, commitReopenResident, } from '../../../core/runtime/reopen.js';
14
16
  import { writeOutputSchema } from '../../../core/runtime/structured-output.js';
15
17
  import { appendSituationalContext, formatSituationalProse } from '../../../core/runtime/situational-context.js';
16
18
  import { formatDataCard } from '../../../shared/generated-context.js';
17
19
  import { hasNoNaturalCycle } from '../../../core/runtime/revive-all.js';
18
20
  import { recordPendingParkActivity } from '../../park-pending.js';
21
+ import { wantsExecution } from '../../reconcilers/node-lifecycle/wants-execution.js';
19
22
  import { hasFreshGroundState } from '../../../core/runtime/revive.js';
20
23
  import { notFound, usage } from '../../../core/errors.js';
21
24
  import { InputError } from '../../../core/io.js';
@@ -161,6 +164,12 @@ function parseSchema(raw) {
161
164
  function label(text) {
162
165
  return text.split('\n')[0].slice(0, 120);
163
166
  }
167
+ function previousFinalReportCard(nodeId, finalReport) {
168
+ if (finalReport === null)
169
+ return '';
170
+ const path = join(reportsDir(nodeId), finalReport);
171
+ return formatDataCard('reopen-final-report', { path }, 'This node previously completed work. Read this final report before acting on the new instruction.');
172
+ }
164
173
  /** The fail-loud rejection for a deferred immediate message aimed at a
165
174
  * target with no natural cycle ahead of it (done/canceled/finalized). Thrown as
166
175
  * an InputError so `toErrorBody` round-trips the FULL structured payload
@@ -211,7 +220,7 @@ async function handleMessage(ctx) {
211
220
  ? parseSchema(req.output_schema)
212
221
  : null;
213
222
  // Fresh revive: no inbox entry, resume:false (spec §7.7).
214
- // Mirrors the CLI's immediate --fresh: validate/clear the finalization latch
223
+ // Mirrors the CLI's immediate --fresh: validate the finalization latch
215
224
  // (--reopen) and upsert the situational sidecar BEFORE the revive, so the
216
225
  // fresh kickoff picks up the update. The goal/roadmap recoverable-state guard
217
226
  // stays a pure CLI-side pre-check.
@@ -223,7 +232,7 @@ async function handleMessage(ctx) {
223
232
  if (hasSidecar)
224
233
  appendSituationalContext(id, sidecar);
225
234
  if (req.reopen)
226
- commitReopen(id, expectedFinalReport);
235
+ commitReopenResident(id, expectedFinalReport);
227
236
  // A fresh revive appends no inbox entry, so nothing survives a freeze for
228
237
  // the daemon to act on later: at the cap the caller is told, not ignored.
229
238
  const result = await reviveNode(id, { resume: false, capacity: 'refuse' });
@@ -242,8 +251,9 @@ async function handleMessage(ctx) {
242
251
  // An interactive reopen commits before even choosing its engine-or-inbox
243
252
  // route. A live transport race then falls back with this same committed state.
244
253
  let reopenCommitted = false;
254
+ let previousFinalCard = '';
245
255
  if (req.delivery === 'interactive' && req.reopen) {
246
- commitReopenResident(id);
256
+ previousFinalCard = previousFinalReportCard(id, commitReopenResident(id));
247
257
  reopenCommitted = true;
248
258
  }
249
259
  // --- Interactive delivery: route via the LIVE broker's serialized frame
@@ -257,7 +267,11 @@ async function handleMessage(ctx) {
257
267
  if (hasSidecar)
258
268
  appendSituationalContext(id, sidecar);
259
269
  await deliverLive(id, req.body, {
260
- cards: [...(hasSidecar ? [sidecar] : []), ...oneShotCards],
270
+ cards: [
271
+ ...(previousFinalCard === '' ? [] : [previousFinalCard]),
272
+ ...(hasSidecar ? [sidecar] : []),
273
+ ...oneShotCards,
274
+ ],
261
275
  // Same authorship the durable route would record for this send, so a
262
276
  // reader cannot tell live from inbox delivery by the stamp.
263
277
  origin: from === 'human'
@@ -294,9 +308,13 @@ async function handleMessage(ctx) {
294
308
  if (hasSidecar)
295
309
  appendSituationalContext(id, sidecar);
296
310
  if (req.reopen && !reopenCommitted) {
297
- commitReopenResident(id);
311
+ previousFinalCard = previousFinalReportCard(id, commitReopenResident(id));
298
312
  }
299
- const cards = oneShotCards.length > 0 ? { cards: oneShotCards } : {};
313
+ const cards = [
314
+ ...(previousFinalCard === '' ? [] : [previousFinalCard]),
315
+ ...oneShotCards,
316
+ ];
317
+ const cardData = cards.length > 0 ? { cards } : {};
300
318
  if (hasBody || schema !== null) {
301
319
  const messageBody = hasBody
302
320
  ? req.body
@@ -306,7 +324,7 @@ async function handleMessage(ctx) {
306
324
  tier,
307
325
  kind: 'message',
308
326
  label: label(messageBody),
309
- data: { body: messageBody, ...(hasSidecar ? { situational: true } : {}), ...cards },
327
+ data: { body: messageBody, ...(hasSidecar ? { situational: true } : {}), ...cardData },
310
328
  });
311
329
  }
312
330
  // Context-only: a hidden wake marker excluded from the visible digest. Its
@@ -317,7 +335,7 @@ async function handleMessage(ctx) {
317
335
  tier,
318
336
  kind: 'message',
319
337
  label: hasSidecar ? '(ambient context updated)' : '(context delivered)',
320
- data: { ...(hasSidecar ? { situational: true } : {}), situationalOnly: true, ...cards },
338
+ data: { ...(hasSidecar ? { situational: true } : {}), situationalOnly: true, ...cardData },
321
339
  });
322
340
  };
323
341
  // Deferred guard: a done/canceled/finalized target has no natural
@@ -355,13 +373,14 @@ async function handleMessage(ctx) {
355
373
  // owes a thaw; consume its deadline when that durable future wake arrives.
356
374
  if (entry.tier !== 'deferred' || meta.frozen_at !== null)
357
375
  cancelCronsOnWake(id);
358
- // A wake-capable tier revives a dormant target so its inbox-watcher delivers
359
- // this; deferred never wakes — it rides the target's next natural cycle. The
360
- // APPENDED entry's tier decides, because a terminal target's deferred was
361
- // already raised to a wake-capable tier by the append.
376
+ // A wake-capable entry for an active or idle target is admitted by the
377
+ // lifecycle tick. Reopen delivery preserves this handler's synchronous wake
378
+ // authority; a terminal-lifecycle row also needs it because it has no later
379
+ // natural cycle to consume the entry.
362
380
  let revived = false;
363
381
  let frozen = false;
364
- if (entry.tier !== 'deferred' && !paused && !isBrokerLive(current)) {
382
+ const needsSynchronousWake = req.reopen || current.lifecycle === 'terminal' || !wantsExecution(current);
383
+ if (needsSynchronousWake && entry.tier !== 'deferred' && !paused && !isBrokerLive(current)) {
365
384
  try {
366
385
  // The entry is already durable, so a wake denied for capacity is only
367
386
  // delayed: the row freezes and the tick relaunches it — at which point
@@ -15,7 +15,8 @@ import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
15
15
  import { join } from 'node:path';
16
16
  import { getNode } from '../../../core/canvas/canvas.js';
17
17
  import { push, FinalizationError } from '../../../core/feed/feed.js';
18
- import { autoDropCleanManagedWorktree, branchIsPushedAtCurrentTip, hasOpenManagedWorktree, missingWorktreeRecoveryHint } from '../../../core/worktree.js';
18
+ import { branchIsPushedAtCurrentTip, hasOpenManagedWorktree, missingWorktreeRecoveryHint } from '../../../core/worktree.js';
19
+ import { autoDropCleanManagedWorktreeAsync } from '../../../core/worktree-mutation-async.js';
19
20
  import { reportsDir } from '../../../core/canvas/paths.js';
20
21
  import { notFound, usage } from '../../../core/errors.js';
21
22
  import { InputError } from '../../../core/io.js';
@@ -89,7 +90,7 @@ async function handlePush(ctx) {
89
90
  async function guardFinalWorktree(id) {
90
91
  if (!hasOpenManagedWorktree(getNode(id)))
91
92
  return undefined;
92
- const dropped = autoDropCleanManagedWorktree(id);
93
+ const dropped = await autoDropCleanManagedWorktreeAsync(id);
93
94
  if (dropped !== null)
94
95
  return dropped.worktree_path;
95
96
  if (await branchIsPushedAtCurrentTip(getNode(id).managed_worktree))
@@ -12,21 +12,22 @@
12
12
  // all of that onto `CrtrError` for every route that can raise it — so the CLI
13
13
  // leaf can reconstruct the original agent-facing contract instead of collapsing
14
14
  // to a generic status code.
15
- import { abandonManagedWorktree, closeManagedWorktree } from '../../../core/worktree.js';
15
+ import { closeManagedWorktreeAsync } from '../../../core/worktree-close.js';
16
+ import { abandonManagedWorktreeAsync } from '../../../core/worktree-mutation-async.js';
16
17
  import { listQuarantinedManagedWorktrees } from '../../../core/worktree-quarantine.js';
17
18
  import { usage } from '../../../core/errors.js';
18
- function handleWorktreeClose(ctx) {
19
+ async function handleWorktreeClose(ctx) {
19
20
  const id = ctx.params['id'];
20
- // No requireMeta: closeManagedWorktree already throws the precise
21
+ // No requireMeta: closeManagedWorktreeAsync already throws the precise
21
22
  // `unknown_node` / `no_managed_worktree` WorktreeError for a bad id.
22
- return { status: 200, body: closeManagedWorktree(id) };
23
+ return { status: 200, body: await closeManagedWorktreeAsync(id) };
23
24
  }
24
- function handleWorktreeAbandon(ctx) {
25
+ async function handleWorktreeAbandon(ctx) {
25
26
  const rawBy = ctx.body?.['by'];
26
27
  const by = typeof rawBy === 'string' ? rawBy.trim() : '';
27
28
  if (by === '')
28
29
  throw usage('abandon requires a non-empty invoking identity', { received: ctx.body, next: 'Pass the invoking node id or human identity.' });
29
- return { status: 200, body: abandonManagedWorktree(ctx.params['id'], by) };
30
+ return { status: 200, body: await abandonManagedWorktreeAsync(ctx.params['id'], by) };
30
31
  }
31
32
  async function handleQuarantinedWorktrees() {
32
33
  return { status: 200, body: await listQuarantinedManagedWorktrees() };
@@ -2,7 +2,7 @@ import { type NodeMeta } from '../core/canvas/index.js';
2
2
  import { type FleetEntry, type FleetExitStatus, type FleetRegistry } from '../core/runtime/fleet.js';
3
3
  import type { NodeOutcomeDetailV1, TerminalReason } from '../core/canvas/types.js';
4
4
  import type { HostHandle } from '../core/runtime/host.js';
5
- export declare function surfaceBootFailure(meta: NodeMeta): Promise<void>;
5
+ export declare function surfaceBootFailure(meta: NodeMeta, exitStatus?: FleetExitStatus): Promise<void>;
6
6
  /** Whether a dead relaunch should resume the saved session strictly, or retry
7
7
  * as a fresh cycle instead. A cycling attempt (cycle_pending) that dies before
8
8
  * session_start must retry AS a cycle: the fresh branch it was opening is not
@@ -34,16 +34,18 @@ import { brokerThresholdsForDaemon } from '../core/runtime/fleet.js';
34
34
  * When the daemon observes that death it errors LOUDLY up the spine: an
35
35
  * urgent push so the parent learns the child failed to launch instead of just
36
36
  * seeing a silent `dead`. */
37
- function bootFailureBody(meta) {
37
+ function bootFailureBody(meta, exitStatus) {
38
+ const status = exitStatus === undefined
39
+ ? 'The daemon did not observe an exit status.'
40
+ : `The observed exit status was code ${exitStatus.code ?? 'null'}, signal ${exitStatus.signal ?? 'null'}.`;
38
41
  return (`⚠ Spawn failed — \`${meta.name}\` (${meta.kind}) never started.\n\n` +
39
42
  `Its pi vehicle exited before the session came up (no pi_session_id was ever ` +
40
- `recorded), so the node produced no output. This is almost always a transient ` +
41
- `launch failure — e.g. resource pressure when several nodes boot at once — not ` +
42
- `a fault in the task itself.\n\n` +
43
- `If the work still needs doing, re-spawn it; if spawns keep dying, spawn fewer at a time.`);
43
+ `recorded), so the node produced no output. ${status}\n\n` +
44
+ `Inspect canonical event diagnostics with \`crtr sys logs --node ${meta.node_id}\`.\n\n` +
45
+ `If the work still needs doing, re-spawn it.`);
44
46
  }
45
- export async function surfaceBootFailure(meta) {
46
- await pushUrgent(meta.node_id, bootFailureBody(meta), { from: meta.node_id });
47
+ export async function surfaceBootFailure(meta, exitStatus) {
48
+ await pushUrgent(meta.node_id, bootFailureBody(meta, exitStatus), { from: meta.node_id });
47
49
  }
48
50
  async function surfaceCrashLoop(meta) {
49
51
  const transportPoison = readTransportPoisonVerdict(meta.node_id);
@@ -137,6 +139,8 @@ export class DaemonFleet {
137
139
  /** Brokers deliberately interrupted by profile pause. This belongs to the
138
140
  * exact child, so a resume cannot turn its later SIGTERM exit into a crash. */
139
141
  #profilePauseInterruptions = new Set();
142
+ /** Exit status held only until dead-row policy surfaces this broker's boot failure. */
143
+ #exitStatuses = new Map();
140
144
  #lastCapacityLogAt = Number.NEGATIVE_INFINITY;
141
145
  constructor(opts) {
142
146
  this.#epoch = opts.epoch;
@@ -155,6 +159,7 @@ export class DaemonFleet {
155
159
  throw new Error(`fleet: refusing to register ${nodeId} with no pid`);
156
160
  }
157
161
  this.#reservations.delete(nodeId);
162
+ this.#exitStatuses.delete(nodeId);
158
163
  this.#map.set(nodeId, { pid: handle.pid, handle });
159
164
  }
160
165
  has(nodeId) {
@@ -215,6 +220,7 @@ export class DaemonFleet {
215
220
  forget(nodeId) {
216
221
  this.#map.delete(nodeId);
217
222
  this.#reservations.delete(nodeId);
223
+ this.#exitStatuses.delete(nodeId);
218
224
  }
219
225
  onChildExit(nodeId, status) {
220
226
  // The real child exit is the liveness boundary. Its queued observation only
@@ -225,6 +231,10 @@ export class DaemonFleet {
225
231
  const uptimeMs = Number.isFinite(launchedMs) ? Math.max(0, this.#now() - launchedMs) : null;
226
232
  const consecutiveFailures = meta?.respawn_failures ?? 0;
227
233
  this.#map.delete(nodeId);
234
+ if (meta?.status === 'done' || meta?.status === 'canceled' || meta?.status === 'dead')
235
+ this.#exitStatuses.delete(nodeId);
236
+ else
237
+ this.#exitStatuses.set(nodeId, status);
228
238
  this.#enqueue(() => this.#emitExitObserved(nodeId, status, uptimeMs, consecutiveFailures));
229
239
  }
230
240
  deliverExit(nodeId, status) {
@@ -232,7 +242,14 @@ export class DaemonFleet {
232
242
  }
233
243
  /** The node-lifecycle tick's enactment entry. */
234
244
  async applyDeadRowPolicy(nodeId) {
235
- await this.#applyPolicy(nodeId);
245
+ const exitStatus = this.#exitStatuses.get(nodeId);
246
+ try {
247
+ await this.#applyPolicy(nodeId, exitStatus);
248
+ }
249
+ finally {
250
+ if (this.#exitStatuses.get(nodeId) === exitStatus)
251
+ this.#exitStatuses.delete(nodeId);
252
+ }
236
253
  }
237
254
  /** A revive refusal before its launch try/catch leaves the row active but
238
255
  * with no fleet entry. Convert that otherwise-silent strand into the same
@@ -267,10 +284,10 @@ export class DaemonFleet {
267
284
  },
268
285
  });
269
286
  }
270
- async #applyPolicy(nodeId) {
271
- return this.#decideAndEnact(nodeId);
287
+ async #applyPolicy(nodeId, exitStatus) {
288
+ return this.#decideAndEnact(nodeId, exitStatus);
272
289
  }
273
- async #decideAndEnact(nodeId) {
290
+ async #decideAndEnact(nodeId, exitStatus) {
274
291
  const meta = getNode(nodeId);
275
292
  if (meta !== null && isProfilePaused(meta.profile_id)) {
276
293
  clearRespawnPolicy(nodeId);
@@ -319,9 +336,10 @@ export class DaemonFleet {
319
336
  await this.#terminalize(nodeId, meta, {
320
337
  event: 'broker.boot.failed',
321
338
  reason: 'boot_failed',
322
- detail: { schema: 'crtr.node-outcome-detail/v1', message: bootFailureBody(meta) },
339
+ detail: { schema: 'crtr.node-outcome-detail/v1', message: bootFailureBody(meta, exitStatus) },
323
340
  wake: `Child crashed — ${fullName(meta)} (${nodeId}) exited before its session ever came up and is now dead. ` +
324
341
  `It stays dead until you revive it — \`crtr node lifecycle revive ${nodeId}\`.`,
342
+ surface: () => surfaceBootFailure(meta, exitStatus),
325
343
  });
326
344
  break;
327
345
  case 'respawn-fresh':
@@ -50,12 +50,14 @@ export interface SpawnDaemonResult {
50
50
  running?: true;
51
51
  /** Present with `running:true` when the Unix API socket is not serving. */
52
52
  serving?: false;
53
- /** Recovery instruction for a live but non-serving daemon. */
54
- next?: string;
53
+ /** The final `/healthz` probe failure when a live owner misses readiness. */
54
+ probe_error?: string;
55
55
  }
56
- /** True only when the mandatory Unix API socket answers `/healthz`. This is
57
- * deliberately separate from `isDaemonRunning()`: a pidfile remains useful for
58
- * identifying the process an operator must stop, but it is not readiness. */
56
+ /** Make one `/healthz` observation without client-side recovery. The caller
57
+ * owns the availability window, so this probe cannot extend it. */
58
+ export declare function probeDaemonServing(timeoutMs?: number): Promise<void>;
59
+ /** True only when one mandatory Unix API socket probe answers `/healthz`.
60
+ * A pidfile identifies an owner; it is never readiness. */
59
61
  export declare function isDaemonServing(): Promise<boolean>;
60
62
  export interface DaemonWaitDeps {
61
63
  readPidfile?: () => number | null;
@@ -69,6 +71,7 @@ export interface DaemonStartupDeps extends DaemonWaitDeps {
69
71
  signal: NodeJS.Signals | null;
70
72
  } | null;
71
73
  isDaemonServing?: () => Promise<boolean>;
74
+ probeDaemonServing?: (timeoutMs: number) => Promise<void>;
72
75
  }
73
76
  /** Thrown when the spawned daemon exited on `DAEMON_EXIT_STARTUP_BLOCKED` — a
74
77
  * standing condition (a blocked on-disk migration) that the very next spawn
@@ -19,7 +19,7 @@ import { clearDaemonStartupBlocked, daemonStartupBlocked } from './startup-block
19
19
  import { exclusiveLockOwnerPid } from '../core/exclusive-lock.js';
20
20
  import { onDiskMigrationLockPath } from '../core/canvas/paths.js';
21
21
  import { envNoDaemonAutostart } from '../shared/env.js';
22
- import { CrtrClient } from '../api/index.js';
22
+ import { CrtrClient, waitForDaemonAvailability } from '../api/index.js';
23
23
  // Daemon env sanitization
24
24
  /** Env keys that must NEVER reach the daemon process. Restarting crtrd is
25
25
  * overwhelmingly done from inside an agent node's own bash tool (`crtr sys
@@ -208,16 +208,20 @@ const DAEMON_VERIFY_POLL_MS = 10;
208
208
  /** A status/start readiness probe must not hang behind a daemon that accepts a
209
209
  * Unix connection but never answers it. */
210
210
  const DAEMON_HEALTH_PROBE_TIMEOUT_MS = 1_000;
211
- /** True only when the mandatory Unix API socket answers `/healthz`. This is
212
- * deliberately separate from `isDaemonRunning()`: a pidfile remains useful for
213
- * identifying the process an operator must stop, but it is not readiness. */
211
+ /** Make one `/healthz` observation without client-side recovery. The caller
212
+ * owns the availability window, so this probe cannot extend it. */
213
+ export async function probeDaemonServing(timeoutMs = DAEMON_HEALTH_PROBE_TIMEOUT_MS) {
214
+ const probeTimeoutMs = Math.min(timeoutMs, DAEMON_HEALTH_PROBE_TIMEOUT_MS);
215
+ const health = await CrtrClient.forLocalSocket({ autostart: false, timeoutMs: probeTimeoutMs }).probeHealthz(probeTimeoutMs);
216
+ if (!health.ok)
217
+ throw new Error('crtrd health check reported not ready');
218
+ }
219
+ /** True only when one mandatory Unix API socket probe answers `/healthz`.
220
+ * A pidfile identifies an owner; it is never readiness. */
214
221
  export async function isDaemonServing() {
215
222
  try {
216
- const health = await CrtrClient.forLocalSocket({
217
- autostart: false,
218
- timeoutMs: DAEMON_HEALTH_PROBE_TIMEOUT_MS,
219
- }).healthz();
220
- return health.ok;
223
+ await probeDaemonServing();
224
+ return true;
221
225
  }
222
226
  catch {
223
227
  return false;
@@ -255,32 +259,47 @@ export async function verifyDaemonStartup(pid, timeoutMs = DAEMON_STARTUP_WINDOW
255
259
  const read = deps.readPidfile ?? readPidfile;
256
260
  const alive = deps.isPidAlive ?? isPidAlive;
257
261
  const exited = deps.childExited;
258
- const serving = deps.isDaemonServing ?? isDaemonServing;
262
+ const serving = deps.isDaemonServing;
263
+ const probeServing = deps.probeDaemonServing ?? (async (remainingMs) => {
264
+ if (serving !== undefined) {
265
+ if (await serving())
266
+ return;
267
+ throw new Error('crtrd health check reported not ready');
268
+ }
269
+ await probeDaemonServing(remainingMs);
270
+ });
259
271
  const sleep = deps.sleepMs ?? sleepMs;
260
272
  const now = deps.now ?? Date.now;
261
- const deadline = now() + timeoutMs;
262
- while (now() <= deadline) {
263
- const owner = read();
264
- if (owner !== null && alive(owner) && await serving())
265
- return owner === pid ? null : owner;
266
- const exitState = exited?.();
267
- if (exitState !== null && exitState !== undefined) {
268
- if (exitState.signal !== null) {
269
- throw new Error(`daemon ${pid} exited before becoming ready (by ${exitState.signal})`);
270
- }
271
- if (exitState.code === DAEMON_EXIT_STARTUP_BLOCKED) {
272
- throw new DaemonStartupBlockedError(pid);
273
- }
274
- if (exitState.code !== 0) {
275
- throw new Error(`daemon ${pid} exited before becoming ready (with exit code ${exitState.code ?? '?'})`);
273
+ let owner = null;
274
+ let terminalError = null;
275
+ await waitForDaemonAvailability({
276
+ windowMs: timeoutMs,
277
+ pollIntervalMs: DAEMON_VERIFY_POLL_MS,
278
+ now,
279
+ sleep,
280
+ retry: (error) => error !== terminalError,
281
+ probe: async (remainingMs) => {
282
+ const candidate = read();
283
+ if (candidate === null || !alive(candidate)) {
284
+ const exitState = exited?.();
285
+ if (exitState !== null && exitState !== undefined) {
286
+ if (exitState.signal !== null)
287
+ terminalError = new Error(`daemon ${pid} exited before becoming ready (by ${exitState.signal})`);
288
+ else if (exitState.code === DAEMON_EXIT_STARTUP_BLOCKED)
289
+ terminalError = new DaemonStartupBlockedError(pid);
290
+ else if (exitState.code !== 0)
291
+ terminalError = new Error(`daemon ${pid} exited before becoming ready (with exit code ${exitState.code ?? '?'})`);
292
+ // A clean exit can have lost the ownership claim, so keep observing.
293
+ if (terminalError !== null)
294
+ throw terminalError;
295
+ }
296
+ throw new Error(`daemon ${pid} has no live pidfile owner`);
276
297
  }
277
- // Clean code-0 exit: a possible ownership-claim loser, not readiness.
278
- // Keep polling the same deadline for a different live pidfile owner; the
279
- // loop's timeout throw below fires if none appears.
280
- }
281
- await sleep(DAEMON_VERIFY_POLL_MS);
282
- }
283
- throw new Error(`daemon ${pid} did not become ready within ${timeoutMs}ms`);
298
+ await probeServing(remainingMs);
299
+ owner = candidate;
300
+ },
301
+ });
302
+ return owner === pid ? null : owner;
284
303
  }
285
304
  /** Wait until a signaled daemon has exited and relinquished its pidfile. This
286
305
  * makes a following start safe to claim the singleton rather than racing its
@@ -355,15 +374,19 @@ export async function stopDaemonProcess(pid, timeoutMs = DAEMON_SHUTDOWN_WINDOW_
355
374
  export async function spawnDaemon() {
356
375
  const recordedPid = readPidfile();
357
376
  if (recordedPid !== null && isPidAlive(recordedPid)) {
358
- if (await isDaemonServing())
377
+ try {
378
+ await probeDaemonServing();
359
379
  return { started: false, existing_pid: recordedPid };
360
- return {
361
- started: false,
362
- existing_pid: recordedPid,
363
- running: true,
364
- serving: false,
365
- next: 'Run `crtr sys daemon stop` to release the non-serving daemon before starting it again.',
366
- };
380
+ }
381
+ catch (error) {
382
+ return {
383
+ started: false,
384
+ existing_pid: recordedPid,
385
+ running: true,
386
+ serving: false,
387
+ probe_error: error.message,
388
+ };
389
+ }
367
390
  }
368
391
  // Ensure the canvas home directory exists so the daemon can write its pidfile.
369
392
  mkdirSync(crtrHome(), { recursive: true });