@north-light/crouter 0.3.303 → 0.3.304

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/dist/api/__tests__/integration/client.test.js +37 -36
  2. package/dist/api/client.d.ts +32 -55
  3. package/dist/api/client.js +102 -98
  4. package/dist/api/dto/messages.d.ts +6 -5
  5. package/dist/api/errors.d.ts +3 -0
  6. package/dist/api/errors.js +10 -0
  7. package/dist/api/index.d.ts +1 -1
  8. package/dist/api/index.js +1 -1
  9. package/dist/builtin-memory/internal/plugins.md +1 -1
  10. package/dist/clients/attach/viewer.js +532 -532
  11. package/dist/commands/__tests__/seam/daemon-status.test.d.ts +1 -0
  12. package/dist/commands/__tests__/seam/daemon-status.test.js +29 -0
  13. package/dist/commands/api-client.d.ts +3 -3
  14. package/dist/commands/api-client.js +3 -3
  15. package/dist/commands/memory/read.js +2 -1
  16. package/dist/commands/node/bash.js +6 -6
  17. package/dist/commands/node/message.js +3 -3
  18. package/dist/commands/sys/daemon.js +11 -13
  19. package/dist/core/__tests__/broker-stream-watchdog-floor.test.js +0 -2
  20. package/dist/core/__tests__/daemon-boot.test.js +1 -1
  21. package/dist/core/__tests__/helpers/harness.d.ts +2 -0
  22. package/dist/core/__tests__/helpers/harness.js +9 -0
  23. package/dist/core/__tests__/integration/worktree-land.test.js +50 -0
  24. package/dist/core/__tests__/integration/worktree-reap.test.js +184 -5
  25. package/dist/core/__tests__/parse-argv-stdin-secret.test.js +14 -0
  26. package/dist/core/__tests__/seam/broker-attach-stream.test.js +13 -0
  27. package/dist/core/__tests__/seam/broker-startup-diagnostics.test.d.ts +1 -0
  28. package/dist/core/__tests__/seam/broker-startup-diagnostics.test.js +82 -0
  29. package/dist/core/bash-jobs.d.ts +20 -8
  30. package/dist/core/bash-jobs.js +40 -21
  31. package/dist/core/canvas/types.d.ts +4 -2
  32. package/dist/core/command.js +1 -1
  33. package/dist/core/fault-classifier.d.ts +1 -1
  34. package/dist/core/runtime/broker/event-projection.d.ts +0 -3
  35. package/dist/core/runtime/broker/event-projection.js +2 -15
  36. package/dist/core/runtime/broker/fault-retry.js +8 -0
  37. package/dist/core/runtime/broker-persona-guidance.js +12 -0
  38. package/dist/core/runtime/broker.js +0 -5
  39. package/dist/core/runtime/fault.js +1 -1
  40. package/dist/core/runtime/host.js +10 -1
  41. package/dist/core/runtime/spawn.js +5 -6
  42. package/dist/core/worktree-close.d.ts +4 -0
  43. package/dist/core/worktree-close.js +225 -0
  44. package/dist/core/worktree-containment.d.ts +14 -0
  45. package/dist/core/worktree-containment.js +48 -0
  46. package/dist/core/worktree-mutation-async.d.ts +13 -0
  47. package/dist/core/worktree-mutation-async.js +281 -0
  48. package/dist/core/worktree-sweep.js +17 -79
  49. package/dist/core/worktree.d.ts +1 -0
  50. package/dist/core/worktree.js +1 -1
  51. package/dist/daemon/__tests__/integration/api-startup-readiness.test.js +10 -8
  52. package/dist/daemon/api/__tests__/seam/api-server.test.js +70 -1
  53. package/dist/daemon/api/handlers/bash-jobs.js +1 -1
  54. package/dist/daemon/api/handlers/messages.js +7 -5
  55. package/dist/daemon/api/handlers/reports.js +3 -2
  56. package/dist/daemon/api/handlers/worktree.js +7 -6
  57. package/dist/daemon/fleet.d.ts +1 -1
  58. package/dist/daemon/fleet.js +30 -12
  59. package/dist/daemon/manage.d.ts +8 -5
  60. package/dist/daemon/manage.js +63 -40
  61. package/dist/daemon/reconcilers/managed-worktree-sweep.js +12 -0
  62. package/dist/daemon/reconcilers/node-lifecycle/tick.d.ts +0 -6
  63. package/dist/daemon/reconcilers/node-lifecycle/tick.js +1 -13
  64. package/dist/daemon/reconcilers/node-lifecycle/wants-execution.d.ts +5 -0
  65. package/dist/daemon/reconcilers/node-lifecycle/wants-execution.js +11 -0
  66. package/dist/pi-extensions/__tests__/canvas-context-intro.test.js +83 -0
  67. package/dist/pi-extensions/__tests__/integration/canvas-bash-valve.test.d.ts +1 -0
  68. package/dist/pi-extensions/__tests__/integration/canvas-bash-valve.test.js +133 -0
  69. package/dist/pi-extensions/canvas-bash-valve.d.ts +2 -0
  70. package/dist/pi-extensions/canvas-bash-valve.js +73 -43
  71. package/dist/pi-extensions/canvas-inbox-watcher.js +0 -2
  72. package/package.json +1 -1
  73. package/runtime.lock.json +5 -5
@@ -21,6 +21,7 @@ import { resolve } from 'node:path';
21
21
  import { getNode, updateNode } from './canvas/index.js';
22
22
  import { withExclusiveDirectoryLockAsync } from './exclusive-lock.js';
23
23
  import { gitAsync } from './git.js';
24
+ import { proveBranchContained, upstreamTip } from './worktree-containment.js';
24
25
  import { parseStatusPorcelain, planLandIntoCheckout, refDeleteTransactionStdin } from './worktree-landing.js';
25
26
  import { WORKTREE_LOCK_WAIT_MS, checkoutPathForBranchInList, persistClosedWorktreeState, samePath, worktreeLockPathForCommonDir } from './worktree.js';
26
27
  /** Escalating recheck interval, in minutes, indexed by consecutive refusal. The
@@ -101,95 +102,32 @@ function registeredCheckoutPath(porcelain, path, branch) {
101
102
  }
102
103
  return null;
103
104
  }
104
- /** Refs that could prove delivery: the recorded local base, and the base's
105
- * upstream (a branch delivered by push rather than by a local land). */
106
- async function containmentCandidates(cwd, wt) {
107
- const candidates = [];
108
- const localBaseRef = `refs/heads/${wt.base_ref}`;
109
- const localBase = await revParseRef(cwd, localBaseRef);
110
- if (localBase !== null)
111
- candidates.push({ ref: localBaseRef, sha: localBase });
112
- const upstream = await gitAsync(['rev-parse', '--symbolic-full-name', '--verify', `${wt.base_ref}@{upstream}`], cwd);
113
- const upstreamRef = upstream.status === 0 && upstream.stdout.trim().startsWith('refs/')
114
- ? upstream.stdout.trim()
115
- : `refs/remotes/origin/${wt.base_ref}`;
116
- const upstreamSha = await revParseRef(cwd, upstreamRef);
117
- if (upstreamSha !== null)
118
- candidates.push({ ref: upstreamRef, sha: upstreamSha });
119
- return candidates;
120
- }
121
- /** Does removing this branch lose content the candidate does not already have?
122
- *
123
- * The invariant is about CONTENT, not history. A rebase rewrites every SHA, so
124
- * a fully-landed branch can look diverged; a squash-merge leaves no matching
125
- * commit at all. An ancestry-only proof would refuse exactly the cases this
126
- * sweep exists to clean up, so three layers run in increasing cost, and any one
127
- * of them is sufficient:
128
- *
129
- * 1. ancestry — the tip is already reachable from the candidate.
130
- * 2. patch identity — every commit not in the candidate by SHA has a
131
- * patch-equivalent commit there (`git cherry`). The rebase case.
132
- * 3. content no-op — merging the branch in would change nothing. The
133
- * squash-merge case, and most remaining "looks bad but isn't" states.
134
- *
135
- * Only when all three fail does the branch genuinely carry unlanded content.
136
- * Every layer fails CLOSED: a Git version without `merge-tree --write-tree`, an
137
- * unreadable object, any non-zero exit — none of them prove anything, so the
138
- * branch survives. */
139
- async function proveContained(cwd, wt, branchSha) {
140
- const candidates = await containmentCandidates(cwd, wt);
141
- if (candidates.length === 0)
142
- return null;
143
- for (const candidate of candidates) {
144
- if ((await gitAsync(['merge-base', '--is-ancestor', branchSha, candidate.sha], cwd)).status === 0) {
145
- return { containingRef: candidate.ref, containingSha: candidate.sha, layer: 'ancestry' };
146
- }
147
- }
148
- for (const candidate of candidates) {
149
- const cherry = await gitAsync(['cherry', candidate.sha, branchSha], cwd);
150
- if (cherry.status !== 0)
151
- continue;
152
- const lines = cherry.stdout.split('\n').map((l) => l.trim()).filter((l) => l !== '');
153
- if (lines.every((line) => line.startsWith('-'))) {
154
- return { containingRef: candidate.ref, containingSha: candidate.sha, layer: 'patch_identity' };
155
- }
156
- }
157
- for (const candidate of candidates) {
158
- const candidateTree = await revParseRef(cwd, `${candidate.sha}^{tree}`);
159
- if (candidateTree === null)
160
- continue;
161
- const merged = await gitAsync(['merge-tree', '--write-tree', candidate.sha, branchSha], cwd);
162
- if (merged.status !== 0)
163
- continue; // conflict, or a Git too old for this flag
164
- if (merged.stdout.split('\n')[0]?.trim() === candidateTree) {
165
- return { containingRef: candidate.ref, containingSha: candidate.sha, layer: 'content_noop' };
166
- }
167
- }
168
- return null;
169
- }
105
+ // Content containment is shared with explicit close. A delivered branch is
106
+ // never rebased onto a local base merely because that base later diverged.
170
107
  // Change detection
171
- /** Base tip, branch tip, and checkout state as they are right now. A pass that
172
- * reads the same three values as the refusal that preceded it can stop there:
173
- * nothing that decided the refusal has moved. An unreadable checkout records
174
- * `null`, which never compares equal, so uncertainty always re-proves. */
108
+ /** Base, upstream, and branch tips plus checkout state as they are right now.
109
+ * A pass that reads the same values as the refusal that preceded it can stop
110
+ * there: nothing that decided the refusal has moved. An unreadable checkout
111
+ * records `null`, which never compares equal, so uncertainty always re-proves. */
175
112
  async function observe(wt) {
176
113
  const present = existsSync(wt.path);
177
114
  const cwd = present ? wt.path : wt.repo_root;
178
115
  const base = await revParseRef(cwd, `refs/heads/${wt.base_ref}`);
116
+ const upstream = await upstreamTip(cwd, wt.base_ref);
179
117
  const branch = await revParseRef(cwd, `refs/heads/${wt.branch}`);
180
118
  if (!present)
181
- return { base, branch, checkout: 'absent' };
119
+ return { base, upstream: upstream?.sha ?? null, branch, checkout: 'absent' };
182
120
  const head = await gitAsync(['rev-parse', 'HEAD'], wt.path);
183
121
  const status = await gitAsync(['status', '--porcelain'], wt.path);
184
122
  const checkout = head.status !== 0 || status.status !== 0
185
123
  ? null
186
124
  : createHash('sha256').update(`${head.stdout.trim()}\n${status.stdout}`).digest('hex');
187
- return { base, branch, checkout };
125
+ return { base, upstream: upstream?.sha ?? null, branch, checkout };
188
126
  }
189
127
  function sameObservation(a, b) {
190
128
  if (a.checkout === null || b.checkout === null)
191
129
  return false;
192
- return a.base === b.base && a.branch === b.branch && a.checkout === b.checkout;
130
+ return a.base === b.base && a.upstream === b.upstream && a.branch === b.branch && a.checkout === b.checkout;
193
131
  }
194
132
  // Recording an unfinished pass
195
133
  /** Persist why this pass did not finish, with the fingerprint it decided
@@ -336,13 +274,13 @@ export async function reconcileManagedWorktree(nodeId, guard, now = Date.now())
336
274
  return { status: 'skipped', reason: 'already_reconciled' };
337
275
  const cwd = existsSync(wt.path) ? wt.path : wt.repo_root;
338
276
  if (!existsSync(cwd)) {
339
- return persistComplete(nodeId, wt, guard);
277
+ return finish(nodeId, wt, { status: 'refused', reason: 'repo_unavailable', detail: `neither the recorded checkout nor repository root exists, so ${wt.branch} cannot be proved absent` }, { base: null, upstream: null, branch: null, checkout: null }, now, guard);
340
278
  }
341
279
  const commonDir = await commonGitDirAsync(cwd);
342
280
  if (commonDir === null) {
343
- return finish(nodeId, wt, { status: 'failed', reason: 'git_dir_unreadable', detail: `could not resolve the git common dir from ${cwd}` }, { base: null, branch: null, checkout: null }, now, guard);
281
+ return finish(nodeId, wt, { status: 'failed', reason: 'git_dir_unreadable', detail: `could not resolve the git common dir from ${cwd}` }, { base: null, upstream: null, branch: null, checkout: null }, now, guard);
344
282
  }
345
- let observed = { base: null, branch: null, checkout: null };
283
+ let observed = { base: null, upstream: null, branch: null, checkout: null };
346
284
  let outcome;
347
285
  try {
348
286
  outcome = await withExclusiveDirectoryLockAsync(worktreeLockPathForCommonDir(commonDir), async () => {
@@ -412,7 +350,7 @@ async function concludeAbsentCheckout(nodeId, wt, observed, guard) {
412
350
  if (elsewhere !== null) {
413
351
  return { status: 'refused', reason: 'branch_checked_out_elsewhere', detail: `${wt.branch} is checked out at ${elsewhere}` };
414
352
  }
415
- const proof = await proveContained(admin, wt, observed.branch);
353
+ const proof = await proveBranchContained(admin, wt, observed.branch);
416
354
  if (proof === null) {
417
355
  return { status: 'refused', reason: 'unlanded_work', detail: `${wt.branch} carries content no base or upstream ref contains; its checkout is already gone, so the branch is the only copy` };
418
356
  }
@@ -477,7 +415,7 @@ async function reconcilePresentCheckout(nodeId, wt, guard) {
477
415
  // is preserved, never deleted, unlike the ordinary path below.
478
416
  return persistComplete(nodeId, wt, guard);
479
417
  }
480
- let proof = await proveContained(wt.path, wt, branchSha);
418
+ let proof = await proveBranchContained(wt.path, wt, branchSha);
481
419
  let stash;
482
420
  if (proof === null && wt.land_intent !== undefined) {
483
421
  const landed = await finishBlockedLand(wt, list.stdout, guard);
@@ -485,7 +423,7 @@ async function reconcilePresentCheckout(nodeId, wt, guard) {
485
423
  return landed.ineligible === true ? NODE_REVIVED : { status: 'refused', reason: 'land_blocked', detail: landed.detail };
486
424
  branchSha = landed.landedSha;
487
425
  stash = landed.stash;
488
- proof = await proveContained(wt.path, wt, branchSha);
426
+ proof = await proveBranchContained(wt.path, wt, branchSha);
489
427
  }
490
428
  if (proof === null) {
491
429
  return { status: 'refused', reason: 'unlanded_work', detail: `${wt.branch} carries commits its base does not contain, and this node never asked to land them` };
@@ -14,6 +14,7 @@ export declare function managedWorktreesRoot(): string;
14
14
  export declare function managedWorktreePath(nodeId: string): string;
15
15
  export declare function hasOpenManagedWorktree(node: NodeMeta | null | undefined): boolean;
16
16
  export declare function openManagedWorktreeForNode(nodeId: string): ManagedWorktree | null;
17
+ export declare function isManagedWorktreeBase(repoRoot: string): boolean;
17
18
  /** Shared parse for a `git worktree list --porcelain` blob already in hand —
18
19
  * callers that must not throw on a read failure (an absent-path proof that
19
20
  * needs to fail closed rather than error) call `gitSync` themselves and pass
@@ -51,7 +51,7 @@ function repoRootFor(cwd) {
51
51
  }
52
52
  return result.stdout.trim();
53
53
  }
54
- function isManagedWorktreeBase(repoRoot) {
54
+ export function isManagedWorktreeBase(repoRoot) {
55
55
  const path = relative(canonicalPath(managedWorktreesRoot()), canonicalPath(repoRoot));
56
56
  return path !== '' && path !== '..' && !path.startsWith(`..${sep}`) && !isAbsolute(path);
57
57
  }
@@ -49,15 +49,17 @@ test('a Unix socket bind failure rejects readiness and releases daemon ownership
49
49
  assert.equal(existsSync(pidfile), false, 'failed readiness removes the pidfile mirror');
50
50
  assert.equal(existsSync(socketPath), true, 'the pre-existing directory was never mistaken for a serving socket');
51
51
  assert.match(readFileSync(join(canvasHome, 'crtrd.log'), 'utf8'), /api\.server\.failed/, 'the bind failure is recorded for cold-start diagnostics');
52
+ rmSync(socketPath, { recursive: true, force: true });
52
53
  writeFileSync(pidfile, String(process.pid));
53
- assert.deepEqual(await spawnDaemon(), {
54
- started: false,
55
- existing_pid: process.pid,
56
- running: true,
57
- serving: false,
58
- next: 'Run `crtr sys daemon stop` to release the non-serving daemon before starting it again.',
59
- }, 'start distinguishes a live non-serving pidfile owner from a serving daemon');
60
- rmSync(pidfile, { force: true });
54
+ try {
55
+ const result = await spawnDaemon();
56
+ assert.deepEqual({ started: result.started, existing_pid: result.existing_pid, running: result.running, serving: result.serving }, { started: false, existing_pid: process.pid, running: true, serving: false }, 'a live owner with an unavailable socket remains in place');
57
+ assert.match(result.probe_error ?? '', /ENOENT|ECONNREFUSED/, 'the result retains the actual failed probe');
58
+ process.kill(process.pid, 0);
59
+ }
60
+ finally {
61
+ rmSync(pidfile, { force: true });
62
+ }
61
63
  const retryClaim = claimDaemonOwnership('post-api-bind-failure');
62
64
  assert.equal(retryClaim.claimed, true, 'failed readiness releases the authoritative ownership claim');
63
65
  if (retryClaim.claimed) {
@@ -26,7 +26,7 @@
26
26
  import { test, before, after } from 'node:test';
27
27
  import assert from 'node:assert/strict';
28
28
  import { spawnSync } from 'node:child_process';
29
- import { statSync, existsSync, readFileSync, mkdtempSync, rmSync, mkdirSync, writeFileSync } from 'node:fs';
29
+ import { chmodSync, statSync, existsSync, readFileSync, mkdtempSync, rmSync, mkdirSync, writeFileSync } from 'node:fs';
30
30
  import { tmpdir } from 'node:os';
31
31
  import { join, dirname } from 'node:path';
32
32
  import WebSocket from 'ws';
@@ -133,6 +133,45 @@ test('A2: POST /nodes boots a live broker; GET /nodes/{id} shows active + live p
133
133
  assert.ok(meta?.pi_pid != null && isPidAlive(meta.pi_pid), 'row records a live broker pid');
134
134
  assert.ok(existsSync(h.brokerSock(id)), 'view.sock exists for the live broker');
135
135
  });
136
+ function installBlockedStartHook(nodeId) {
137
+ const root = join(h.agentHome, '.crouter', 'plugins', `blocked-start-${nodeId}`);
138
+ const started = join(root, 'started');
139
+ const release = join(root, 'release');
140
+ mkdirSync(join(root, '.crouter-plugin'), { recursive: true });
141
+ writeFileSync(join(root, '.crouter-plugin', 'plugin.json'), JSON.stringify({
142
+ name: `blocked-start-${nodeId}`,
143
+ version: '0.1.0',
144
+ description: 'message acknowledgment fixture',
145
+ hooks: 'hooks.json',
146
+ hookExecutable: 'bin/hook.js',
147
+ }));
148
+ writeFileSync(join(root, 'hooks.json'), JSON.stringify({
149
+ schemaVersion: 2,
150
+ lifecycle: [{ event: 'node:start', phase: 'on', op: 'block', description: 'block target launch', effects: ['waits for test release'] }],
151
+ }));
152
+ const executable = join(root, 'bin', 'hook.js');
153
+ mkdirSync(join(root, 'bin'), { recursive: true });
154
+ writeFileSync(executable, `#!/usr/bin/env node
155
+ const { existsSync, writeFileSync } = require('node:fs');
156
+ let input = '';
157
+ process.stdin.on('data', (chunk) => { input += chunk; });
158
+ process.stdin.on('end', () => {
159
+ const request = JSON.parse(input);
160
+ if (request.node.id !== ${JSON.stringify(nodeId)}) {
161
+ process.stdout.write(JSON.stringify({ protocolVersion: 1, ok: true }));
162
+ return;
163
+ }
164
+ writeFileSync(${JSON.stringify(started)}, 'started');
165
+ const timer = setInterval(() => {
166
+ if (!existsSync(${JSON.stringify(release)})) return;
167
+ clearInterval(timer);
168
+ process.stdout.write(JSON.stringify({ protocolVersion: 1, ok: true }));
169
+ }, 10);
170
+ });
171
+ `);
172
+ chmodSync(executable, 0o755);
173
+ return { started, release };
174
+ }
136
175
  // A3 — POST .../messages appends inbox.jsonl; POST .../reports {tier:'final'}
137
176
  // writes the report, notifies subscribers, transitions the row
138
177
  test('A3: messages append inbox.jsonl; final report writes + notifies subscriber + transitions row', async () => {
@@ -153,6 +192,36 @@ test('A3: messages append inbox.jsonl; final report writes + notifies subscriber
153
192
  assert.equal(h.node(B)?.status, 'done', 'row status is done after final');
154
193
  await assert.rejects(() => client.pushReport(B, { tier: 'final', delivery_tier: 'deferred', body: 'invalid tiered final' }), (error) => error instanceof ApiError && error.code === 'usage' && error.message === 'delivery_tier is only valid for update reports', 'the API rejects a delivery tier on final before it can alter final delivery');
155
194
  });
195
+ test('A3b: a durable send acknowledges while the lifecycle tick is blocked launching its idle target', async () => {
196
+ const target = h.fabricateBrokerNode({ lifecycle: 'resident', status: 'idle', intent: 'idle-release' });
197
+ const hook = installBlockedStartHook(target);
198
+ const inboxFile = inboxPath(target);
199
+ const before = countLines(inboxFile);
200
+ const sending = client.sendMessage(target, { body: 'wake after durable acknowledgment' });
201
+ let tick;
202
+ try {
203
+ await h.waitFor(() => countLines(inboxFile) === before + 1, { label: 'durable inbox append' });
204
+ assert.equal(h.node(target)?.cycles ?? 0, 0, 'the send has not launched the target');
205
+ tick = h.tick();
206
+ await h.waitFor(() => existsSync(hook.started), { label: 'blocked lifecycle launch' });
207
+ const acknowledged = await Promise.race([
208
+ sending,
209
+ new Promise((_, reject) => setTimeout(() => reject(new Error('durable send waited for recipient launch')), 500)),
210
+ ]);
211
+ assert.equal(acknowledged.delivered, true, 'the API acknowledges the durable append');
212
+ assert.equal(acknowledged.revived, false, 'the response honestly reports no synchronous revive');
213
+ assert.equal(countLines(inboxFile), before + 1, 'exactly one inbox entry is durable while launch is blocked');
214
+ writeFileSync(hook.release, 'release');
215
+ await tick;
216
+ assert.equal(h.node(target)?.cycles, 1, 'the one queued entry drove lifecycle revival');
217
+ assert.equal(countLines(inboxFile), before + 1, 'revival did not append a second inbox entry');
218
+ }
219
+ finally {
220
+ if (!existsSync(hook.release))
221
+ writeFileSync(hook.release, 'release');
222
+ await tick;
223
+ }
224
+ });
156
225
  // A4 — WS attach reconstructs the same settled broker snapshot as a direct
157
226
  // ViewSocketClient; ?revive=0 on a dormant node → 409 without launching a broker
158
227
  test('A4a: WS attach welcome.snapshot is byte-identical to a direct ViewSocketClient snapshot', async () => {
@@ -59,7 +59,7 @@ async function handleStop(ctx) {
59
59
  throw new ApiError(404, 'job_not_found', `background job ${jobId} for ${nodeId} is no longer running`, { log_path: stopped.logPath });
60
60
  }
61
61
  if (stopped.kind === 'not-stoppable') {
62
- throw new ApiError(409, 'job_not_stoppable', `job ${jobId} recorded no process group`, { log_path: stopped.logPath });
62
+ throw new ApiError(409, 'job_not_stoppable', `job ${jobId} recorded no trusted process identity`, { log_path: stopped.logPath });
63
63
  }
64
64
  // Match `crtr node bash kill`: retire the job, then send an urgent durable
65
65
  // notice to the owning node. The terminal path sends that notice through the
@@ -16,6 +16,7 @@ import { appendSituationalContext, formatSituationalProse } from '../../../core/
16
16
  import { formatDataCard } from '../../../shared/generated-context.js';
17
17
  import { hasNoNaturalCycle } from '../../../core/runtime/revive-all.js';
18
18
  import { recordPendingParkActivity } from '../../park-pending.js';
19
+ import { wantsExecution } from '../../reconcilers/node-lifecycle/wants-execution.js';
19
20
  import { hasFreshGroundState } from '../../../core/runtime/revive.js';
20
21
  import { notFound, usage } from '../../../core/errors.js';
21
22
  import { InputError } from '../../../core/io.js';
@@ -355,13 +356,14 @@ async function handleMessage(ctx) {
355
356
  // owes a thaw; consume its deadline when that durable future wake arrives.
356
357
  if (entry.tier !== 'deferred' || meta.frozen_at !== null)
357
358
  cancelCronsOnWake(id);
358
- // A wake-capable tier revives a dormant target so its inbox-watcher delivers
359
- // this; deferred never wakes — it rides the target's next natural cycle. The
360
- // APPENDED entry's tier decides, because a terminal target's deferred was
361
- // already raised to a wake-capable tier by the append.
359
+ // A wake-capable entry for an active or idle target is admitted by the
360
+ // lifecycle tick. Reopen delivery preserves this handler's synchronous wake
361
+ // authority; a terminal-lifecycle row also needs it because it has no later
362
+ // natural cycle to consume the entry.
362
363
  let revived = false;
363
364
  let frozen = false;
364
- if (entry.tier !== 'deferred' && !paused && !isBrokerLive(current)) {
365
+ const needsSynchronousWake = req.reopen || current.lifecycle === 'terminal' || !wantsExecution(current);
366
+ if (needsSynchronousWake && entry.tier !== 'deferred' && !paused && !isBrokerLive(current)) {
365
367
  try {
366
368
  // The entry is already durable, so a wake denied for capacity is only
367
369
  // delayed: the row freezes and the tick relaunches it — at which point
@@ -15,7 +15,8 @@ import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
15
15
  import { join } from 'node:path';
16
16
  import { getNode } from '../../../core/canvas/canvas.js';
17
17
  import { push, FinalizationError } from '../../../core/feed/feed.js';
18
- import { autoDropCleanManagedWorktree, branchIsPushedAtCurrentTip, hasOpenManagedWorktree, missingWorktreeRecoveryHint } from '../../../core/worktree.js';
18
+ import { branchIsPushedAtCurrentTip, hasOpenManagedWorktree, missingWorktreeRecoveryHint } from '../../../core/worktree.js';
19
+ import { autoDropCleanManagedWorktreeAsync } from '../../../core/worktree-mutation-async.js';
19
20
  import { reportsDir } from '../../../core/canvas/paths.js';
20
21
  import { notFound, usage } from '../../../core/errors.js';
21
22
  import { InputError } from '../../../core/io.js';
@@ -89,7 +90,7 @@ async function handlePush(ctx) {
89
90
  async function guardFinalWorktree(id) {
90
91
  if (!hasOpenManagedWorktree(getNode(id)))
91
92
  return undefined;
92
- const dropped = autoDropCleanManagedWorktree(id);
93
+ const dropped = await autoDropCleanManagedWorktreeAsync(id);
93
94
  if (dropped !== null)
94
95
  return dropped.worktree_path;
95
96
  if (await branchIsPushedAtCurrentTip(getNode(id).managed_worktree))
@@ -12,21 +12,22 @@
12
12
  // all of that onto `CrtrError` for every route that can raise it — so the CLI
13
13
  // leaf can reconstruct the original agent-facing contract instead of collapsing
14
14
  // to a generic status code.
15
- import { abandonManagedWorktree, closeManagedWorktree } from '../../../core/worktree.js';
15
+ import { closeManagedWorktreeAsync } from '../../../core/worktree-close.js';
16
+ import { abandonManagedWorktreeAsync } from '../../../core/worktree-mutation-async.js';
16
17
  import { listQuarantinedManagedWorktrees } from '../../../core/worktree-quarantine.js';
17
18
  import { usage } from '../../../core/errors.js';
18
- function handleWorktreeClose(ctx) {
19
+ async function handleWorktreeClose(ctx) {
19
20
  const id = ctx.params['id'];
20
- // No requireMeta: closeManagedWorktree already throws the precise
21
+ // No requireMeta: closeManagedWorktreeAsync already throws the precise
21
22
  // `unknown_node` / `no_managed_worktree` WorktreeError for a bad id.
22
- return { status: 200, body: closeManagedWorktree(id) };
23
+ return { status: 200, body: await closeManagedWorktreeAsync(id) };
23
24
  }
24
- function handleWorktreeAbandon(ctx) {
25
+ async function handleWorktreeAbandon(ctx) {
25
26
  const rawBy = ctx.body?.['by'];
26
27
  const by = typeof rawBy === 'string' ? rawBy.trim() : '';
27
28
  if (by === '')
28
29
  throw usage('abandon requires a non-empty invoking identity', { received: ctx.body, next: 'Pass the invoking node id or human identity.' });
29
- return { status: 200, body: abandonManagedWorktree(ctx.params['id'], by) };
30
+ return { status: 200, body: await abandonManagedWorktreeAsync(ctx.params['id'], by) };
30
31
  }
31
32
  async function handleQuarantinedWorktrees() {
32
33
  return { status: 200, body: await listQuarantinedManagedWorktrees() };
@@ -2,7 +2,7 @@ import { type NodeMeta } from '../core/canvas/index.js';
2
2
  import { type FleetEntry, type FleetExitStatus, type FleetRegistry } from '../core/runtime/fleet.js';
3
3
  import type { NodeOutcomeDetailV1, TerminalReason } from '../core/canvas/types.js';
4
4
  import type { HostHandle } from '../core/runtime/host.js';
5
- export declare function surfaceBootFailure(meta: NodeMeta): Promise<void>;
5
+ export declare function surfaceBootFailure(meta: NodeMeta, exitStatus?: FleetExitStatus): Promise<void>;
6
6
  /** Whether a dead relaunch should resume the saved session strictly, or retry
7
7
  * as a fresh cycle instead. A cycling attempt (cycle_pending) that dies before
8
8
  * session_start must retry AS a cycle: the fresh branch it was opening is not
@@ -34,16 +34,18 @@ import { brokerThresholdsForDaemon } from '../core/runtime/fleet.js';
34
34
  * When the daemon observes that death it errors LOUDLY up the spine: an
35
35
  * urgent push so the parent learns the child failed to launch instead of just
36
36
  * seeing a silent `dead`. */
37
- function bootFailureBody(meta) {
37
+ function bootFailureBody(meta, exitStatus) {
38
+ const status = exitStatus === undefined
39
+ ? 'The daemon did not observe an exit status.'
40
+ : `The observed exit status was code ${exitStatus.code ?? 'null'}, signal ${exitStatus.signal ?? 'null'}.`;
38
41
  return (`⚠ Spawn failed — \`${meta.name}\` (${meta.kind}) never started.\n\n` +
39
42
  `Its pi vehicle exited before the session came up (no pi_session_id was ever ` +
40
- `recorded), so the node produced no output. This is almost always a transient ` +
41
- `launch failure — e.g. resource pressure when several nodes boot at once — not ` +
42
- `a fault in the task itself.\n\n` +
43
- `If the work still needs doing, re-spawn it; if spawns keep dying, spawn fewer at a time.`);
43
+ `recorded), so the node produced no output. ${status}\n\n` +
44
+ `Inspect canonical event diagnostics with \`crtr sys logs --node ${meta.node_id}\`.\n\n` +
45
+ `If the work still needs doing, re-spawn it.`);
44
46
  }
45
- export async function surfaceBootFailure(meta) {
46
- await pushUrgent(meta.node_id, bootFailureBody(meta), { from: meta.node_id });
47
+ export async function surfaceBootFailure(meta, exitStatus) {
48
+ await pushUrgent(meta.node_id, bootFailureBody(meta, exitStatus), { from: meta.node_id });
47
49
  }
48
50
  async function surfaceCrashLoop(meta) {
49
51
  const transportPoison = readTransportPoisonVerdict(meta.node_id);
@@ -137,6 +139,8 @@ export class DaemonFleet {
137
139
  /** Brokers deliberately interrupted by profile pause. This belongs to the
138
140
  * exact child, so a resume cannot turn its later SIGTERM exit into a crash. */
139
141
  #profilePauseInterruptions = new Set();
142
+ /** Exit status held only until dead-row policy surfaces this broker's boot failure. */
143
+ #exitStatuses = new Map();
140
144
  #lastCapacityLogAt = Number.NEGATIVE_INFINITY;
141
145
  constructor(opts) {
142
146
  this.#epoch = opts.epoch;
@@ -155,6 +159,7 @@ export class DaemonFleet {
155
159
  throw new Error(`fleet: refusing to register ${nodeId} with no pid`);
156
160
  }
157
161
  this.#reservations.delete(nodeId);
162
+ this.#exitStatuses.delete(nodeId);
158
163
  this.#map.set(nodeId, { pid: handle.pid, handle });
159
164
  }
160
165
  has(nodeId) {
@@ -215,6 +220,7 @@ export class DaemonFleet {
215
220
  forget(nodeId) {
216
221
  this.#map.delete(nodeId);
217
222
  this.#reservations.delete(nodeId);
223
+ this.#exitStatuses.delete(nodeId);
218
224
  }
219
225
  onChildExit(nodeId, status) {
220
226
  // The real child exit is the liveness boundary. Its queued observation only
@@ -225,6 +231,10 @@ export class DaemonFleet {
225
231
  const uptimeMs = Number.isFinite(launchedMs) ? Math.max(0, this.#now() - launchedMs) : null;
226
232
  const consecutiveFailures = meta?.respawn_failures ?? 0;
227
233
  this.#map.delete(nodeId);
234
+ if (meta?.status === 'done' || meta?.status === 'canceled' || meta?.status === 'dead')
235
+ this.#exitStatuses.delete(nodeId);
236
+ else
237
+ this.#exitStatuses.set(nodeId, status);
228
238
  this.#enqueue(() => this.#emitExitObserved(nodeId, status, uptimeMs, consecutiveFailures));
229
239
  }
230
240
  deliverExit(nodeId, status) {
@@ -232,7 +242,14 @@ export class DaemonFleet {
232
242
  }
233
243
  /** The node-lifecycle tick's enactment entry. */
234
244
  async applyDeadRowPolicy(nodeId) {
235
- await this.#applyPolicy(nodeId);
245
+ const exitStatus = this.#exitStatuses.get(nodeId);
246
+ try {
247
+ await this.#applyPolicy(nodeId, exitStatus);
248
+ }
249
+ finally {
250
+ if (this.#exitStatuses.get(nodeId) === exitStatus)
251
+ this.#exitStatuses.delete(nodeId);
252
+ }
236
253
  }
237
254
  /** A revive refusal before its launch try/catch leaves the row active but
238
255
  * with no fleet entry. Convert that otherwise-silent strand into the same
@@ -267,10 +284,10 @@ export class DaemonFleet {
267
284
  },
268
285
  });
269
286
  }
270
- async #applyPolicy(nodeId) {
271
- return this.#decideAndEnact(nodeId);
287
+ async #applyPolicy(nodeId, exitStatus) {
288
+ return this.#decideAndEnact(nodeId, exitStatus);
272
289
  }
273
- async #decideAndEnact(nodeId) {
290
+ async #decideAndEnact(nodeId, exitStatus) {
274
291
  const meta = getNode(nodeId);
275
292
  if (meta !== null && isProfilePaused(meta.profile_id)) {
276
293
  clearRespawnPolicy(nodeId);
@@ -319,9 +336,10 @@ export class DaemonFleet {
319
336
  await this.#terminalize(nodeId, meta, {
320
337
  event: 'broker.boot.failed',
321
338
  reason: 'boot_failed',
322
- detail: { schema: 'crtr.node-outcome-detail/v1', message: bootFailureBody(meta) },
339
+ detail: { schema: 'crtr.node-outcome-detail/v1', message: bootFailureBody(meta, exitStatus) },
323
340
  wake: `Child crashed — ${fullName(meta)} (${nodeId}) exited before its session ever came up and is now dead. ` +
324
341
  `It stays dead until you revive it — \`crtr node lifecycle revive ${nodeId}\`.`,
342
+ surface: () => surfaceBootFailure(meta, exitStatus),
325
343
  });
326
344
  break;
327
345
  case 'respawn-fresh':
@@ -50,12 +50,14 @@ export interface SpawnDaemonResult {
50
50
  running?: true;
51
51
  /** Present with `running:true` when the Unix API socket is not serving. */
52
52
  serving?: false;
53
- /** Recovery instruction for a live but non-serving daemon. */
54
- next?: string;
53
+ /** The final `/healthz` probe failure when a live owner misses readiness. */
54
+ probe_error?: string;
55
55
  }
56
- /** True only when the mandatory Unix API socket answers `/healthz`. This is
57
- * deliberately separate from `isDaemonRunning()`: a pidfile remains useful for
58
- * identifying the process an operator must stop, but it is not readiness. */
56
+ /** Make one `/healthz` observation without client-side recovery. The caller
57
+ * owns the availability window, so this probe cannot extend it. */
58
+ export declare function probeDaemonServing(timeoutMs?: number): Promise<void>;
59
+ /** True only when one mandatory Unix API socket probe answers `/healthz`.
60
+ * A pidfile identifies an owner; it is never readiness. */
59
61
  export declare function isDaemonServing(): Promise<boolean>;
60
62
  export interface DaemonWaitDeps {
61
63
  readPidfile?: () => number | null;
@@ -69,6 +71,7 @@ export interface DaemonStartupDeps extends DaemonWaitDeps {
69
71
  signal: NodeJS.Signals | null;
70
72
  } | null;
71
73
  isDaemonServing?: () => Promise<boolean>;
74
+ probeDaemonServing?: (timeoutMs: number) => Promise<void>;
72
75
  }
73
76
  /** Thrown when the spawned daemon exited on `DAEMON_EXIT_STARTUP_BLOCKED` — a
74
77
  * standing condition (a blocked on-disk migration) that the very next spawn