@north-light/crouter 0.3.303 → 0.3.304
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/__tests__/integration/client.test.js +37 -36
- package/dist/api/client.d.ts +32 -55
- package/dist/api/client.js +102 -98
- package/dist/api/dto/messages.d.ts +6 -5
- package/dist/api/errors.d.ts +3 -0
- package/dist/api/errors.js +10 -0
- package/dist/api/index.d.ts +1 -1
- package/dist/api/index.js +1 -1
- package/dist/builtin-memory/internal/plugins.md +1 -1
- package/dist/clients/attach/viewer.js +532 -532
- package/dist/commands/__tests__/seam/daemon-status.test.d.ts +1 -0
- package/dist/commands/__tests__/seam/daemon-status.test.js +29 -0
- package/dist/commands/api-client.d.ts +3 -3
- package/dist/commands/api-client.js +3 -3
- package/dist/commands/memory/read.js +2 -1
- package/dist/commands/node/bash.js +6 -6
- package/dist/commands/node/message.js +3 -3
- package/dist/commands/sys/daemon.js +11 -13
- package/dist/core/__tests__/broker-stream-watchdog-floor.test.js +0 -2
- package/dist/core/__tests__/daemon-boot.test.js +1 -1
- package/dist/core/__tests__/helpers/harness.d.ts +2 -0
- package/dist/core/__tests__/helpers/harness.js +9 -0
- package/dist/core/__tests__/integration/worktree-land.test.js +50 -0
- package/dist/core/__tests__/integration/worktree-reap.test.js +184 -5
- package/dist/core/__tests__/parse-argv-stdin-secret.test.js +14 -0
- package/dist/core/__tests__/seam/broker-attach-stream.test.js +13 -0
- package/dist/core/__tests__/seam/broker-startup-diagnostics.test.d.ts +1 -0
- package/dist/core/__tests__/seam/broker-startup-diagnostics.test.js +82 -0
- package/dist/core/bash-jobs.d.ts +20 -8
- package/dist/core/bash-jobs.js +40 -21
- package/dist/core/canvas/types.d.ts +4 -2
- package/dist/core/command.js +1 -1
- package/dist/core/fault-classifier.d.ts +1 -1
- package/dist/core/runtime/broker/event-projection.d.ts +0 -3
- package/dist/core/runtime/broker/event-projection.js +2 -15
- package/dist/core/runtime/broker/fault-retry.js +8 -0
- package/dist/core/runtime/broker-persona-guidance.js +12 -0
- package/dist/core/runtime/broker.js +0 -5
- package/dist/core/runtime/fault.js +1 -1
- package/dist/core/runtime/host.js +10 -1
- package/dist/core/runtime/spawn.js +5 -6
- package/dist/core/worktree-close.d.ts +4 -0
- package/dist/core/worktree-close.js +225 -0
- package/dist/core/worktree-containment.d.ts +14 -0
- package/dist/core/worktree-containment.js +48 -0
- package/dist/core/worktree-mutation-async.d.ts +13 -0
- package/dist/core/worktree-mutation-async.js +281 -0
- package/dist/core/worktree-sweep.js +17 -79
- package/dist/core/worktree.d.ts +1 -0
- package/dist/core/worktree.js +1 -1
- package/dist/daemon/__tests__/integration/api-startup-readiness.test.js +10 -8
- package/dist/daemon/api/__tests__/seam/api-server.test.js +70 -1
- package/dist/daemon/api/handlers/bash-jobs.js +1 -1
- package/dist/daemon/api/handlers/messages.js +7 -5
- package/dist/daemon/api/handlers/reports.js +3 -2
- package/dist/daemon/api/handlers/worktree.js +7 -6
- package/dist/daemon/fleet.d.ts +1 -1
- package/dist/daemon/fleet.js +30 -12
- package/dist/daemon/manage.d.ts +8 -5
- package/dist/daemon/manage.js +63 -40
- package/dist/daemon/reconcilers/managed-worktree-sweep.js +12 -0
- package/dist/daemon/reconcilers/node-lifecycle/tick.d.ts +0 -6
- package/dist/daemon/reconcilers/node-lifecycle/tick.js +1 -13
- package/dist/daemon/reconcilers/node-lifecycle/wants-execution.d.ts +5 -0
- package/dist/daemon/reconcilers/node-lifecycle/wants-execution.js +11 -0
- package/dist/pi-extensions/__tests__/canvas-context-intro.test.js +83 -0
- package/dist/pi-extensions/__tests__/integration/canvas-bash-valve.test.d.ts +1 -0
- package/dist/pi-extensions/__tests__/integration/canvas-bash-valve.test.js +133 -0
- package/dist/pi-extensions/canvas-bash-valve.d.ts +2 -0
- package/dist/pi-extensions/canvas-bash-valve.js +73 -43
- package/dist/pi-extensions/canvas-inbox-watcher.js +0 -2
- package/package.json +1 -1
- package/runtime.lock.json +5 -5
|
@@ -21,6 +21,7 @@ import { resolve } from 'node:path';
|
|
|
21
21
|
import { getNode, updateNode } from './canvas/index.js';
|
|
22
22
|
import { withExclusiveDirectoryLockAsync } from './exclusive-lock.js';
|
|
23
23
|
import { gitAsync } from './git.js';
|
|
24
|
+
import { proveBranchContained, upstreamTip } from './worktree-containment.js';
|
|
24
25
|
import { parseStatusPorcelain, planLandIntoCheckout, refDeleteTransactionStdin } from './worktree-landing.js';
|
|
25
26
|
import { WORKTREE_LOCK_WAIT_MS, checkoutPathForBranchInList, persistClosedWorktreeState, samePath, worktreeLockPathForCommonDir } from './worktree.js';
|
|
26
27
|
/** Escalating recheck interval, in minutes, indexed by consecutive refusal. The
|
|
@@ -101,95 +102,32 @@ function registeredCheckoutPath(porcelain, path, branch) {
|
|
|
101
102
|
}
|
|
102
103
|
return null;
|
|
103
104
|
}
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
async function containmentCandidates(cwd, wt) {
|
|
107
|
-
const candidates = [];
|
|
108
|
-
const localBaseRef = `refs/heads/${wt.base_ref}`;
|
|
109
|
-
const localBase = await revParseRef(cwd, localBaseRef);
|
|
110
|
-
if (localBase !== null)
|
|
111
|
-
candidates.push({ ref: localBaseRef, sha: localBase });
|
|
112
|
-
const upstream = await gitAsync(['rev-parse', '--symbolic-full-name', '--verify', `${wt.base_ref}@{upstream}`], cwd);
|
|
113
|
-
const upstreamRef = upstream.status === 0 && upstream.stdout.trim().startsWith('refs/')
|
|
114
|
-
? upstream.stdout.trim()
|
|
115
|
-
: `refs/remotes/origin/${wt.base_ref}`;
|
|
116
|
-
const upstreamSha = await revParseRef(cwd, upstreamRef);
|
|
117
|
-
if (upstreamSha !== null)
|
|
118
|
-
candidates.push({ ref: upstreamRef, sha: upstreamSha });
|
|
119
|
-
return candidates;
|
|
120
|
-
}
|
|
121
|
-
/** Does removing this branch lose content the candidate does not already have?
|
|
122
|
-
*
|
|
123
|
-
* The invariant is about CONTENT, not history. A rebase rewrites every SHA, so
|
|
124
|
-
* a fully-landed branch can look diverged; a squash-merge leaves no matching
|
|
125
|
-
* commit at all. An ancestry-only proof would refuse exactly the cases this
|
|
126
|
-
* sweep exists to clean up, so three layers run in increasing cost, and any one
|
|
127
|
-
* of them is sufficient:
|
|
128
|
-
*
|
|
129
|
-
* 1. ancestry — the tip is already reachable from the candidate.
|
|
130
|
-
* 2. patch identity — every commit not in the candidate by SHA has a
|
|
131
|
-
* patch-equivalent commit there (`git cherry`). The rebase case.
|
|
132
|
-
* 3. content no-op — merging the branch in would change nothing. The
|
|
133
|
-
* squash-merge case, and most remaining "looks bad but isn't" states.
|
|
134
|
-
*
|
|
135
|
-
* Only when all three fail does the branch genuinely carry unlanded content.
|
|
136
|
-
* Every layer fails CLOSED: a Git version without `merge-tree --write-tree`, an
|
|
137
|
-
* unreadable object, any non-zero exit — none of them prove anything, so the
|
|
138
|
-
* branch survives. */
|
|
139
|
-
async function proveContained(cwd, wt, branchSha) {
|
|
140
|
-
const candidates = await containmentCandidates(cwd, wt);
|
|
141
|
-
if (candidates.length === 0)
|
|
142
|
-
return null;
|
|
143
|
-
for (const candidate of candidates) {
|
|
144
|
-
if ((await gitAsync(['merge-base', '--is-ancestor', branchSha, candidate.sha], cwd)).status === 0) {
|
|
145
|
-
return { containingRef: candidate.ref, containingSha: candidate.sha, layer: 'ancestry' };
|
|
146
|
-
}
|
|
147
|
-
}
|
|
148
|
-
for (const candidate of candidates) {
|
|
149
|
-
const cherry = await gitAsync(['cherry', candidate.sha, branchSha], cwd);
|
|
150
|
-
if (cherry.status !== 0)
|
|
151
|
-
continue;
|
|
152
|
-
const lines = cherry.stdout.split('\n').map((l) => l.trim()).filter((l) => l !== '');
|
|
153
|
-
if (lines.every((line) => line.startsWith('-'))) {
|
|
154
|
-
return { containingRef: candidate.ref, containingSha: candidate.sha, layer: 'patch_identity' };
|
|
155
|
-
}
|
|
156
|
-
}
|
|
157
|
-
for (const candidate of candidates) {
|
|
158
|
-
const candidateTree = await revParseRef(cwd, `${candidate.sha}^{tree}`);
|
|
159
|
-
if (candidateTree === null)
|
|
160
|
-
continue;
|
|
161
|
-
const merged = await gitAsync(['merge-tree', '--write-tree', candidate.sha, branchSha], cwd);
|
|
162
|
-
if (merged.status !== 0)
|
|
163
|
-
continue; // conflict, or a Git too old for this flag
|
|
164
|
-
if (merged.stdout.split('\n')[0]?.trim() === candidateTree) {
|
|
165
|
-
return { containingRef: candidate.ref, containingSha: candidate.sha, layer: 'content_noop' };
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
return null;
|
|
169
|
-
}
|
|
105
|
+
// Content containment is shared with explicit close. A delivered branch is
|
|
106
|
+
// never rebased onto a local base merely because that base later diverged.
|
|
170
107
|
// Change detection
|
|
171
|
-
/** Base
|
|
172
|
-
* reads the same
|
|
173
|
-
* nothing that decided the refusal has moved. An unreadable checkout
|
|
174
|
-
* `null`, which never compares equal, so uncertainty always re-proves. */
|
|
108
|
+
/** Base, upstream, and branch tips plus checkout state as they are right now.
|
|
109
|
+
* A pass that reads the same values as the refusal that preceded it can stop
|
|
110
|
+
* there: nothing that decided the refusal has moved. An unreadable checkout
|
|
111
|
+
* records `null`, which never compares equal, so uncertainty always re-proves. */
|
|
175
112
|
async function observe(wt) {
|
|
176
113
|
const present = existsSync(wt.path);
|
|
177
114
|
const cwd = present ? wt.path : wt.repo_root;
|
|
178
115
|
const base = await revParseRef(cwd, `refs/heads/${wt.base_ref}`);
|
|
116
|
+
const upstream = await upstreamTip(cwd, wt.base_ref);
|
|
179
117
|
const branch = await revParseRef(cwd, `refs/heads/${wt.branch}`);
|
|
180
118
|
if (!present)
|
|
181
|
-
return { base, branch, checkout: 'absent' };
|
|
119
|
+
return { base, upstream: upstream?.sha ?? null, branch, checkout: 'absent' };
|
|
182
120
|
const head = await gitAsync(['rev-parse', 'HEAD'], wt.path);
|
|
183
121
|
const status = await gitAsync(['status', '--porcelain'], wt.path);
|
|
184
122
|
const checkout = head.status !== 0 || status.status !== 0
|
|
185
123
|
? null
|
|
186
124
|
: createHash('sha256').update(`${head.stdout.trim()}\n${status.stdout}`).digest('hex');
|
|
187
|
-
return { base, branch, checkout };
|
|
125
|
+
return { base, upstream: upstream?.sha ?? null, branch, checkout };
|
|
188
126
|
}
|
|
189
127
|
function sameObservation(a, b) {
|
|
190
128
|
if (a.checkout === null || b.checkout === null)
|
|
191
129
|
return false;
|
|
192
|
-
return a.base === b.base && a.branch === b.branch && a.checkout === b.checkout;
|
|
130
|
+
return a.base === b.base && a.upstream === b.upstream && a.branch === b.branch && a.checkout === b.checkout;
|
|
193
131
|
}
|
|
194
132
|
// Recording an unfinished pass
|
|
195
133
|
/** Persist why this pass did not finish, with the fingerprint it decided
|
|
@@ -336,13 +274,13 @@ export async function reconcileManagedWorktree(nodeId, guard, now = Date.now())
|
|
|
336
274
|
return { status: 'skipped', reason: 'already_reconciled' };
|
|
337
275
|
const cwd = existsSync(wt.path) ? wt.path : wt.repo_root;
|
|
338
276
|
if (!existsSync(cwd)) {
|
|
339
|
-
return
|
|
277
|
+
return finish(nodeId, wt, { status: 'refused', reason: 'repo_unavailable', detail: `neither the recorded checkout nor repository root exists, so ${wt.branch} cannot be proved absent` }, { base: null, upstream: null, branch: null, checkout: null }, now, guard);
|
|
340
278
|
}
|
|
341
279
|
const commonDir = await commonGitDirAsync(cwd);
|
|
342
280
|
if (commonDir === null) {
|
|
343
|
-
return finish(nodeId, wt, { status: 'failed', reason: 'git_dir_unreadable', detail: `could not resolve the git common dir from ${cwd}` }, { base: null, branch: null, checkout: null }, now, guard);
|
|
281
|
+
return finish(nodeId, wt, { status: 'failed', reason: 'git_dir_unreadable', detail: `could not resolve the git common dir from ${cwd}` }, { base: null, upstream: null, branch: null, checkout: null }, now, guard);
|
|
344
282
|
}
|
|
345
|
-
let observed = { base: null, branch: null, checkout: null };
|
|
283
|
+
let observed = { base: null, upstream: null, branch: null, checkout: null };
|
|
346
284
|
let outcome;
|
|
347
285
|
try {
|
|
348
286
|
outcome = await withExclusiveDirectoryLockAsync(worktreeLockPathForCommonDir(commonDir), async () => {
|
|
@@ -412,7 +350,7 @@ async function concludeAbsentCheckout(nodeId, wt, observed, guard) {
|
|
|
412
350
|
if (elsewhere !== null) {
|
|
413
351
|
return { status: 'refused', reason: 'branch_checked_out_elsewhere', detail: `${wt.branch} is checked out at ${elsewhere}` };
|
|
414
352
|
}
|
|
415
|
-
const proof = await
|
|
353
|
+
const proof = await proveBranchContained(admin, wt, observed.branch);
|
|
416
354
|
if (proof === null) {
|
|
417
355
|
return { status: 'refused', reason: 'unlanded_work', detail: `${wt.branch} carries content no base or upstream ref contains; its checkout is already gone, so the branch is the only copy` };
|
|
418
356
|
}
|
|
@@ -477,7 +415,7 @@ async function reconcilePresentCheckout(nodeId, wt, guard) {
|
|
|
477
415
|
// is preserved, never deleted, unlike the ordinary path below.
|
|
478
416
|
return persistComplete(nodeId, wt, guard);
|
|
479
417
|
}
|
|
480
|
-
let proof = await
|
|
418
|
+
let proof = await proveBranchContained(wt.path, wt, branchSha);
|
|
481
419
|
let stash;
|
|
482
420
|
if (proof === null && wt.land_intent !== undefined) {
|
|
483
421
|
const landed = await finishBlockedLand(wt, list.stdout, guard);
|
|
@@ -485,7 +423,7 @@ async function reconcilePresentCheckout(nodeId, wt, guard) {
|
|
|
485
423
|
return landed.ineligible === true ? NODE_REVIVED : { status: 'refused', reason: 'land_blocked', detail: landed.detail };
|
|
486
424
|
branchSha = landed.landedSha;
|
|
487
425
|
stash = landed.stash;
|
|
488
|
-
proof = await
|
|
426
|
+
proof = await proveBranchContained(wt.path, wt, branchSha);
|
|
489
427
|
}
|
|
490
428
|
if (proof === null) {
|
|
491
429
|
return { status: 'refused', reason: 'unlanded_work', detail: `${wt.branch} carries commits its base does not contain, and this node never asked to land them` };
|
package/dist/core/worktree.d.ts
CHANGED
|
@@ -14,6 +14,7 @@ export declare function managedWorktreesRoot(): string;
|
|
|
14
14
|
export declare function managedWorktreePath(nodeId: string): string;
|
|
15
15
|
export declare function hasOpenManagedWorktree(node: NodeMeta | null | undefined): boolean;
|
|
16
16
|
export declare function openManagedWorktreeForNode(nodeId: string): ManagedWorktree | null;
|
|
17
|
+
export declare function isManagedWorktreeBase(repoRoot: string): boolean;
|
|
17
18
|
/** Shared parse for a `git worktree list --porcelain` blob already in hand —
|
|
18
19
|
* callers that must not throw on a read failure (an absent-path proof that
|
|
19
20
|
* needs to fail closed rather than error) call `gitSync` themselves and pass
|
package/dist/core/worktree.js
CHANGED
|
@@ -51,7 +51,7 @@ function repoRootFor(cwd) {
|
|
|
51
51
|
}
|
|
52
52
|
return result.stdout.trim();
|
|
53
53
|
}
|
|
54
|
-
function isManagedWorktreeBase(repoRoot) {
|
|
54
|
+
export function isManagedWorktreeBase(repoRoot) {
|
|
55
55
|
const path = relative(canonicalPath(managedWorktreesRoot()), canonicalPath(repoRoot));
|
|
56
56
|
return path !== '' && path !== '..' && !path.startsWith(`..${sep}`) && !isAbsolute(path);
|
|
57
57
|
}
|
|
@@ -49,15 +49,17 @@ test('a Unix socket bind failure rejects readiness and releases daemon ownership
|
|
|
49
49
|
assert.equal(existsSync(pidfile), false, 'failed readiness removes the pidfile mirror');
|
|
50
50
|
assert.equal(existsSync(socketPath), true, 'the pre-existing directory was never mistaken for a serving socket');
|
|
51
51
|
assert.match(readFileSync(join(canvasHome, 'crtrd.log'), 'utf8'), /api\.server\.failed/, 'the bind failure is recorded for cold-start diagnostics');
|
|
52
|
+
rmSync(socketPath, { recursive: true, force: true });
|
|
52
53
|
writeFileSync(pidfile, String(process.pid));
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
existing_pid: process.pid,
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
54
|
+
try {
|
|
55
|
+
const result = await spawnDaemon();
|
|
56
|
+
assert.deepEqual({ started: result.started, existing_pid: result.existing_pid, running: result.running, serving: result.serving }, { started: false, existing_pid: process.pid, running: true, serving: false }, 'a live owner with an unavailable socket remains in place');
|
|
57
|
+
assert.match(result.probe_error ?? '', /ENOENT|ECONNREFUSED/, 'the result retains the actual failed probe');
|
|
58
|
+
process.kill(process.pid, 0);
|
|
59
|
+
}
|
|
60
|
+
finally {
|
|
61
|
+
rmSync(pidfile, { force: true });
|
|
62
|
+
}
|
|
61
63
|
const retryClaim = claimDaemonOwnership('post-api-bind-failure');
|
|
62
64
|
assert.equal(retryClaim.claimed, true, 'failed readiness releases the authoritative ownership claim');
|
|
63
65
|
if (retryClaim.claimed) {
|
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
import { test, before, after } from 'node:test';
|
|
27
27
|
import assert from 'node:assert/strict';
|
|
28
28
|
import { spawnSync } from 'node:child_process';
|
|
29
|
-
import { statSync, existsSync, readFileSync, mkdtempSync, rmSync, mkdirSync, writeFileSync } from 'node:fs';
|
|
29
|
+
import { chmodSync, statSync, existsSync, readFileSync, mkdtempSync, rmSync, mkdirSync, writeFileSync } from 'node:fs';
|
|
30
30
|
import { tmpdir } from 'node:os';
|
|
31
31
|
import { join, dirname } from 'node:path';
|
|
32
32
|
import WebSocket from 'ws';
|
|
@@ -133,6 +133,45 @@ test('A2: POST /nodes boots a live broker; GET /nodes/{id} shows active + live p
|
|
|
133
133
|
assert.ok(meta?.pi_pid != null && isPidAlive(meta.pi_pid), 'row records a live broker pid');
|
|
134
134
|
assert.ok(existsSync(h.brokerSock(id)), 'view.sock exists for the live broker');
|
|
135
135
|
});
|
|
136
|
+
function installBlockedStartHook(nodeId) {
|
|
137
|
+
const root = join(h.agentHome, '.crouter', 'plugins', `blocked-start-${nodeId}`);
|
|
138
|
+
const started = join(root, 'started');
|
|
139
|
+
const release = join(root, 'release');
|
|
140
|
+
mkdirSync(join(root, '.crouter-plugin'), { recursive: true });
|
|
141
|
+
writeFileSync(join(root, '.crouter-plugin', 'plugin.json'), JSON.stringify({
|
|
142
|
+
name: `blocked-start-${nodeId}`,
|
|
143
|
+
version: '0.1.0',
|
|
144
|
+
description: 'message acknowledgment fixture',
|
|
145
|
+
hooks: 'hooks.json',
|
|
146
|
+
hookExecutable: 'bin/hook.js',
|
|
147
|
+
}));
|
|
148
|
+
writeFileSync(join(root, 'hooks.json'), JSON.stringify({
|
|
149
|
+
schemaVersion: 2,
|
|
150
|
+
lifecycle: [{ event: 'node:start', phase: 'on', op: 'block', description: 'block target launch', effects: ['waits for test release'] }],
|
|
151
|
+
}));
|
|
152
|
+
const executable = join(root, 'bin', 'hook.js');
|
|
153
|
+
mkdirSync(join(root, 'bin'), { recursive: true });
|
|
154
|
+
writeFileSync(executable, `#!/usr/bin/env node
|
|
155
|
+
const { existsSync, writeFileSync } = require('node:fs');
|
|
156
|
+
let input = '';
|
|
157
|
+
process.stdin.on('data', (chunk) => { input += chunk; });
|
|
158
|
+
process.stdin.on('end', () => {
|
|
159
|
+
const request = JSON.parse(input);
|
|
160
|
+
if (request.node.id !== ${JSON.stringify(nodeId)}) {
|
|
161
|
+
process.stdout.write(JSON.stringify({ protocolVersion: 1, ok: true }));
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
writeFileSync(${JSON.stringify(started)}, 'started');
|
|
165
|
+
const timer = setInterval(() => {
|
|
166
|
+
if (!existsSync(${JSON.stringify(release)})) return;
|
|
167
|
+
clearInterval(timer);
|
|
168
|
+
process.stdout.write(JSON.stringify({ protocolVersion: 1, ok: true }));
|
|
169
|
+
}, 10);
|
|
170
|
+
});
|
|
171
|
+
`);
|
|
172
|
+
chmodSync(executable, 0o755);
|
|
173
|
+
return { started, release };
|
|
174
|
+
}
|
|
136
175
|
// A3 — POST .../messages appends inbox.jsonl; POST .../reports {tier:'final'}
|
|
137
176
|
// writes the report, notifies subscribers, transitions the row
|
|
138
177
|
test('A3: messages append inbox.jsonl; final report writes + notifies subscriber + transitions row', async () => {
|
|
@@ -153,6 +192,36 @@ test('A3: messages append inbox.jsonl; final report writes + notifies subscriber
|
|
|
153
192
|
assert.equal(h.node(B)?.status, 'done', 'row status is done after final');
|
|
154
193
|
await assert.rejects(() => client.pushReport(B, { tier: 'final', delivery_tier: 'deferred', body: 'invalid tiered final' }), (error) => error instanceof ApiError && error.code === 'usage' && error.message === 'delivery_tier is only valid for update reports', 'the API rejects a delivery tier on final before it can alter final delivery');
|
|
155
194
|
});
|
|
195
|
+
test('A3b: a durable send acknowledges while the lifecycle tick is blocked launching its idle target', async () => {
|
|
196
|
+
const target = h.fabricateBrokerNode({ lifecycle: 'resident', status: 'idle', intent: 'idle-release' });
|
|
197
|
+
const hook = installBlockedStartHook(target);
|
|
198
|
+
const inboxFile = inboxPath(target);
|
|
199
|
+
const before = countLines(inboxFile);
|
|
200
|
+
const sending = client.sendMessage(target, { body: 'wake after durable acknowledgment' });
|
|
201
|
+
let tick;
|
|
202
|
+
try {
|
|
203
|
+
await h.waitFor(() => countLines(inboxFile) === before + 1, { label: 'durable inbox append' });
|
|
204
|
+
assert.equal(h.node(target)?.cycles ?? 0, 0, 'the send has not launched the target');
|
|
205
|
+
tick = h.tick();
|
|
206
|
+
await h.waitFor(() => existsSync(hook.started), { label: 'blocked lifecycle launch' });
|
|
207
|
+
const acknowledged = await Promise.race([
|
|
208
|
+
sending,
|
|
209
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error('durable send waited for recipient launch')), 500)),
|
|
210
|
+
]);
|
|
211
|
+
assert.equal(acknowledged.delivered, true, 'the API acknowledges the durable append');
|
|
212
|
+
assert.equal(acknowledged.revived, false, 'the response honestly reports no synchronous revive');
|
|
213
|
+
assert.equal(countLines(inboxFile), before + 1, 'exactly one inbox entry is durable while launch is blocked');
|
|
214
|
+
writeFileSync(hook.release, 'release');
|
|
215
|
+
await tick;
|
|
216
|
+
assert.equal(h.node(target)?.cycles, 1, 'the one queued entry drove lifecycle revival');
|
|
217
|
+
assert.equal(countLines(inboxFile), before + 1, 'revival did not append a second inbox entry');
|
|
218
|
+
}
|
|
219
|
+
finally {
|
|
220
|
+
if (!existsSync(hook.release))
|
|
221
|
+
writeFileSync(hook.release, 'release');
|
|
222
|
+
await tick;
|
|
223
|
+
}
|
|
224
|
+
});
|
|
156
225
|
// A4 — WS attach reconstructs the same settled broker snapshot as a direct
|
|
157
226
|
// ViewSocketClient; ?revive=0 on a dormant node → 409 without launching a broker
|
|
158
227
|
test('A4a: WS attach welcome.snapshot is byte-identical to a direct ViewSocketClient snapshot', async () => {
|
|
@@ -59,7 +59,7 @@ async function handleStop(ctx) {
|
|
|
59
59
|
throw new ApiError(404, 'job_not_found', `background job ${jobId} for ${nodeId} is no longer running`, { log_path: stopped.logPath });
|
|
60
60
|
}
|
|
61
61
|
if (stopped.kind === 'not-stoppable') {
|
|
62
|
-
throw new ApiError(409, 'job_not_stoppable', `job ${jobId} recorded no process
|
|
62
|
+
throw new ApiError(409, 'job_not_stoppable', `job ${jobId} recorded no trusted process identity`, { log_path: stopped.logPath });
|
|
63
63
|
}
|
|
64
64
|
// Match `crtr node bash kill`: retire the job, then send an urgent durable
|
|
65
65
|
// notice to the owning node. The terminal path sends that notice through the
|
|
@@ -16,6 +16,7 @@ import { appendSituationalContext, formatSituationalProse } from '../../../core/
|
|
|
16
16
|
import { formatDataCard } from '../../../shared/generated-context.js';
|
|
17
17
|
import { hasNoNaturalCycle } from '../../../core/runtime/revive-all.js';
|
|
18
18
|
import { recordPendingParkActivity } from '../../park-pending.js';
|
|
19
|
+
import { wantsExecution } from '../../reconcilers/node-lifecycle/wants-execution.js';
|
|
19
20
|
import { hasFreshGroundState } from '../../../core/runtime/revive.js';
|
|
20
21
|
import { notFound, usage } from '../../../core/errors.js';
|
|
21
22
|
import { InputError } from '../../../core/io.js';
|
|
@@ -355,13 +356,14 @@ async function handleMessage(ctx) {
|
|
|
355
356
|
// owes a thaw; consume its deadline when that durable future wake arrives.
|
|
356
357
|
if (entry.tier !== 'deferred' || meta.frozen_at !== null)
|
|
357
358
|
cancelCronsOnWake(id);
|
|
358
|
-
// A wake-capable
|
|
359
|
-
//
|
|
360
|
-
//
|
|
361
|
-
//
|
|
359
|
+
// A wake-capable entry for an active or idle target is admitted by the
|
|
360
|
+
// lifecycle tick. Reopen delivery preserves this handler's synchronous wake
|
|
361
|
+
// authority; a terminal-lifecycle row also needs it because it has no later
|
|
362
|
+
// natural cycle to consume the entry.
|
|
362
363
|
let revived = false;
|
|
363
364
|
let frozen = false;
|
|
364
|
-
|
|
365
|
+
const needsSynchronousWake = req.reopen || current.lifecycle === 'terminal' || !wantsExecution(current);
|
|
366
|
+
if (needsSynchronousWake && entry.tier !== 'deferred' && !paused && !isBrokerLive(current)) {
|
|
365
367
|
try {
|
|
366
368
|
// The entry is already durable, so a wake denied for capacity is only
|
|
367
369
|
// delayed: the row freezes and the tick relaunches it — at which point
|
|
@@ -15,7 +15,8 @@ import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
|
|
|
15
15
|
import { join } from 'node:path';
|
|
16
16
|
import { getNode } from '../../../core/canvas/canvas.js';
|
|
17
17
|
import { push, FinalizationError } from '../../../core/feed/feed.js';
|
|
18
|
-
import {
|
|
18
|
+
import { branchIsPushedAtCurrentTip, hasOpenManagedWorktree, missingWorktreeRecoveryHint } from '../../../core/worktree.js';
|
|
19
|
+
import { autoDropCleanManagedWorktreeAsync } from '../../../core/worktree-mutation-async.js';
|
|
19
20
|
import { reportsDir } from '../../../core/canvas/paths.js';
|
|
20
21
|
import { notFound, usage } from '../../../core/errors.js';
|
|
21
22
|
import { InputError } from '../../../core/io.js';
|
|
@@ -89,7 +90,7 @@ async function handlePush(ctx) {
|
|
|
89
90
|
async function guardFinalWorktree(id) {
|
|
90
91
|
if (!hasOpenManagedWorktree(getNode(id)))
|
|
91
92
|
return undefined;
|
|
92
|
-
const dropped =
|
|
93
|
+
const dropped = await autoDropCleanManagedWorktreeAsync(id);
|
|
93
94
|
if (dropped !== null)
|
|
94
95
|
return dropped.worktree_path;
|
|
95
96
|
if (await branchIsPushedAtCurrentTip(getNode(id).managed_worktree))
|
|
@@ -12,21 +12,22 @@
|
|
|
12
12
|
// all of that onto `CrtrError` for every route that can raise it — so the CLI
|
|
13
13
|
// leaf can reconstruct the original agent-facing contract instead of collapsing
|
|
14
14
|
// to a generic status code.
|
|
15
|
-
import {
|
|
15
|
+
import { closeManagedWorktreeAsync } from '../../../core/worktree-close.js';
|
|
16
|
+
import { abandonManagedWorktreeAsync } from '../../../core/worktree-mutation-async.js';
|
|
16
17
|
import { listQuarantinedManagedWorktrees } from '../../../core/worktree-quarantine.js';
|
|
17
18
|
import { usage } from '../../../core/errors.js';
|
|
18
|
-
function handleWorktreeClose(ctx) {
|
|
19
|
+
async function handleWorktreeClose(ctx) {
|
|
19
20
|
const id = ctx.params['id'];
|
|
20
|
-
// No requireMeta:
|
|
21
|
+
// No requireMeta: closeManagedWorktreeAsync already throws the precise
|
|
21
22
|
// `unknown_node` / `no_managed_worktree` WorktreeError for a bad id.
|
|
22
|
-
return { status: 200, body:
|
|
23
|
+
return { status: 200, body: await closeManagedWorktreeAsync(id) };
|
|
23
24
|
}
|
|
24
|
-
function handleWorktreeAbandon(ctx) {
|
|
25
|
+
async function handleWorktreeAbandon(ctx) {
|
|
25
26
|
const rawBy = ctx.body?.['by'];
|
|
26
27
|
const by = typeof rawBy === 'string' ? rawBy.trim() : '';
|
|
27
28
|
if (by === '')
|
|
28
29
|
throw usage('abandon requires a non-empty invoking identity', { received: ctx.body, next: 'Pass the invoking node id or human identity.' });
|
|
29
|
-
return { status: 200, body:
|
|
30
|
+
return { status: 200, body: await abandonManagedWorktreeAsync(ctx.params['id'], by) };
|
|
30
31
|
}
|
|
31
32
|
async function handleQuarantinedWorktrees() {
|
|
32
33
|
return { status: 200, body: await listQuarantinedManagedWorktrees() };
|
package/dist/daemon/fleet.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { type NodeMeta } from '../core/canvas/index.js';
|
|
|
2
2
|
import { type FleetEntry, type FleetExitStatus, type FleetRegistry } from '../core/runtime/fleet.js';
|
|
3
3
|
import type { NodeOutcomeDetailV1, TerminalReason } from '../core/canvas/types.js';
|
|
4
4
|
import type { HostHandle } from '../core/runtime/host.js';
|
|
5
|
-
export declare function surfaceBootFailure(meta: NodeMeta): Promise<void>;
|
|
5
|
+
export declare function surfaceBootFailure(meta: NodeMeta, exitStatus?: FleetExitStatus): Promise<void>;
|
|
6
6
|
/** Whether a dead relaunch should resume the saved session strictly, or retry
|
|
7
7
|
* as a fresh cycle instead. A cycling attempt (cycle_pending) that dies before
|
|
8
8
|
* session_start must retry AS a cycle: the fresh branch it was opening is not
|
package/dist/daemon/fleet.js
CHANGED
|
@@ -34,16 +34,18 @@ import { brokerThresholdsForDaemon } from '../core/runtime/fleet.js';
|
|
|
34
34
|
* When the daemon observes that death it errors LOUDLY up the spine: an
|
|
35
35
|
* urgent push so the parent learns the child failed to launch instead of just
|
|
36
36
|
* seeing a silent `dead`. */
|
|
37
|
-
function bootFailureBody(meta) {
|
|
37
|
+
function bootFailureBody(meta, exitStatus) {
|
|
38
|
+
const status = exitStatus === undefined
|
|
39
|
+
? 'The daemon did not observe an exit status.'
|
|
40
|
+
: `The observed exit status was code ${exitStatus.code ?? 'null'}, signal ${exitStatus.signal ?? 'null'}.`;
|
|
38
41
|
return (`⚠ Spawn failed — \`${meta.name}\` (${meta.kind}) never started.\n\n` +
|
|
39
42
|
`Its pi vehicle exited before the session came up (no pi_session_id was ever ` +
|
|
40
|
-
`recorded), so the node produced no output.
|
|
41
|
-
`
|
|
42
|
-
`
|
|
43
|
-
`If the work still needs doing, re-spawn it; if spawns keep dying, spawn fewer at a time.`);
|
|
43
|
+
`recorded), so the node produced no output. ${status}\n\n` +
|
|
44
|
+
`Inspect canonical event diagnostics with \`crtr sys logs --node ${meta.node_id}\`.\n\n` +
|
|
45
|
+
`If the work still needs doing, re-spawn it.`);
|
|
44
46
|
}
|
|
45
|
-
export async function surfaceBootFailure(meta) {
|
|
46
|
-
await pushUrgent(meta.node_id, bootFailureBody(meta), { from: meta.node_id });
|
|
47
|
+
export async function surfaceBootFailure(meta, exitStatus) {
|
|
48
|
+
await pushUrgent(meta.node_id, bootFailureBody(meta, exitStatus), { from: meta.node_id });
|
|
47
49
|
}
|
|
48
50
|
async function surfaceCrashLoop(meta) {
|
|
49
51
|
const transportPoison = readTransportPoisonVerdict(meta.node_id);
|
|
@@ -137,6 +139,8 @@ export class DaemonFleet {
|
|
|
137
139
|
/** Brokers deliberately interrupted by profile pause. This belongs to the
|
|
138
140
|
* exact child, so a resume cannot turn its later SIGTERM exit into a crash. */
|
|
139
141
|
#profilePauseInterruptions = new Set();
|
|
142
|
+
/** Exit status held only until dead-row policy surfaces this broker's boot failure. */
|
|
143
|
+
#exitStatuses = new Map();
|
|
140
144
|
#lastCapacityLogAt = Number.NEGATIVE_INFINITY;
|
|
141
145
|
constructor(opts) {
|
|
142
146
|
this.#epoch = opts.epoch;
|
|
@@ -155,6 +159,7 @@ export class DaemonFleet {
|
|
|
155
159
|
throw new Error(`fleet: refusing to register ${nodeId} with no pid`);
|
|
156
160
|
}
|
|
157
161
|
this.#reservations.delete(nodeId);
|
|
162
|
+
this.#exitStatuses.delete(nodeId);
|
|
158
163
|
this.#map.set(nodeId, { pid: handle.pid, handle });
|
|
159
164
|
}
|
|
160
165
|
has(nodeId) {
|
|
@@ -215,6 +220,7 @@ export class DaemonFleet {
|
|
|
215
220
|
forget(nodeId) {
|
|
216
221
|
this.#map.delete(nodeId);
|
|
217
222
|
this.#reservations.delete(nodeId);
|
|
223
|
+
this.#exitStatuses.delete(nodeId);
|
|
218
224
|
}
|
|
219
225
|
onChildExit(nodeId, status) {
|
|
220
226
|
// The real child exit is the liveness boundary. Its queued observation only
|
|
@@ -225,6 +231,10 @@ export class DaemonFleet {
|
|
|
225
231
|
const uptimeMs = Number.isFinite(launchedMs) ? Math.max(0, this.#now() - launchedMs) : null;
|
|
226
232
|
const consecutiveFailures = meta?.respawn_failures ?? 0;
|
|
227
233
|
this.#map.delete(nodeId);
|
|
234
|
+
if (meta?.status === 'done' || meta?.status === 'canceled' || meta?.status === 'dead')
|
|
235
|
+
this.#exitStatuses.delete(nodeId);
|
|
236
|
+
else
|
|
237
|
+
this.#exitStatuses.set(nodeId, status);
|
|
228
238
|
this.#enqueue(() => this.#emitExitObserved(nodeId, status, uptimeMs, consecutiveFailures));
|
|
229
239
|
}
|
|
230
240
|
deliverExit(nodeId, status) {
|
|
@@ -232,7 +242,14 @@ export class DaemonFleet {
|
|
|
232
242
|
}
|
|
233
243
|
/** The node-lifecycle tick's enactment entry. */
|
|
234
244
|
async applyDeadRowPolicy(nodeId) {
|
|
235
|
-
|
|
245
|
+
const exitStatus = this.#exitStatuses.get(nodeId);
|
|
246
|
+
try {
|
|
247
|
+
await this.#applyPolicy(nodeId, exitStatus);
|
|
248
|
+
}
|
|
249
|
+
finally {
|
|
250
|
+
if (this.#exitStatuses.get(nodeId) === exitStatus)
|
|
251
|
+
this.#exitStatuses.delete(nodeId);
|
|
252
|
+
}
|
|
236
253
|
}
|
|
237
254
|
/** A revive refusal before its launch try/catch leaves the row active but
|
|
238
255
|
* with no fleet entry. Convert that otherwise-silent strand into the same
|
|
@@ -267,10 +284,10 @@ export class DaemonFleet {
|
|
|
267
284
|
},
|
|
268
285
|
});
|
|
269
286
|
}
|
|
270
|
-
async #applyPolicy(nodeId) {
|
|
271
|
-
return this.#decideAndEnact(nodeId);
|
|
287
|
+
async #applyPolicy(nodeId, exitStatus) {
|
|
288
|
+
return this.#decideAndEnact(nodeId, exitStatus);
|
|
272
289
|
}
|
|
273
|
-
async #decideAndEnact(nodeId) {
|
|
290
|
+
async #decideAndEnact(nodeId, exitStatus) {
|
|
274
291
|
const meta = getNode(nodeId);
|
|
275
292
|
if (meta !== null && isProfilePaused(meta.profile_id)) {
|
|
276
293
|
clearRespawnPolicy(nodeId);
|
|
@@ -319,9 +336,10 @@ export class DaemonFleet {
|
|
|
319
336
|
await this.#terminalize(nodeId, meta, {
|
|
320
337
|
event: 'broker.boot.failed',
|
|
321
338
|
reason: 'boot_failed',
|
|
322
|
-
detail: { schema: 'crtr.node-outcome-detail/v1', message: bootFailureBody(meta) },
|
|
339
|
+
detail: { schema: 'crtr.node-outcome-detail/v1', message: bootFailureBody(meta, exitStatus) },
|
|
323
340
|
wake: `Child crashed — ${fullName(meta)} (${nodeId}) exited before its session ever came up and is now dead. ` +
|
|
324
341
|
`It stays dead until you revive it — \`crtr node lifecycle revive ${nodeId}\`.`,
|
|
342
|
+
surface: () => surfaceBootFailure(meta, exitStatus),
|
|
325
343
|
});
|
|
326
344
|
break;
|
|
327
345
|
case 'respawn-fresh':
|
package/dist/daemon/manage.d.ts
CHANGED
|
@@ -50,12 +50,14 @@ export interface SpawnDaemonResult {
|
|
|
50
50
|
running?: true;
|
|
51
51
|
/** Present with `running:true` when the Unix API socket is not serving. */
|
|
52
52
|
serving?: false;
|
|
53
|
-
/**
|
|
54
|
-
|
|
53
|
+
/** The final `/healthz` probe failure when a live owner misses readiness. */
|
|
54
|
+
probe_error?: string;
|
|
55
55
|
}
|
|
56
|
-
/**
|
|
57
|
-
*
|
|
58
|
-
|
|
56
|
+
/** Make one `/healthz` observation without client-side recovery. The caller
|
|
57
|
+
* owns the availability window, so this probe cannot extend it. */
|
|
58
|
+
export declare function probeDaemonServing(timeoutMs?: number): Promise<void>;
|
|
59
|
+
/** True only when one mandatory Unix API socket probe answers `/healthz`.
|
|
60
|
+
* A pidfile identifies an owner; it is never readiness. */
|
|
59
61
|
export declare function isDaemonServing(): Promise<boolean>;
|
|
60
62
|
export interface DaemonWaitDeps {
|
|
61
63
|
readPidfile?: () => number | null;
|
|
@@ -69,6 +71,7 @@ export interface DaemonStartupDeps extends DaemonWaitDeps {
|
|
|
69
71
|
signal: NodeJS.Signals | null;
|
|
70
72
|
} | null;
|
|
71
73
|
isDaemonServing?: () => Promise<boolean>;
|
|
74
|
+
probeDaemonServing?: (timeoutMs: number) => Promise<void>;
|
|
72
75
|
}
|
|
73
76
|
/** Thrown when the spawned daemon exited on `DAEMON_EXIT_STARTUP_BLOCKED` — a
|
|
74
77
|
* standing condition (a blocked on-disk migration) that the very next spawn
|