@north-light/crouter 0.3.303 → 0.3.304
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/__tests__/integration/client.test.js +37 -36
- package/dist/api/client.d.ts +32 -55
- package/dist/api/client.js +102 -98
- package/dist/api/dto/messages.d.ts +6 -5
- package/dist/api/errors.d.ts +3 -0
- package/dist/api/errors.js +10 -0
- package/dist/api/index.d.ts +1 -1
- package/dist/api/index.js +1 -1
- package/dist/builtin-memory/internal/plugins.md +1 -1
- package/dist/clients/attach/viewer.js +532 -532
- package/dist/commands/__tests__/seam/daemon-status.test.d.ts +1 -0
- package/dist/commands/__tests__/seam/daemon-status.test.js +29 -0
- package/dist/commands/api-client.d.ts +3 -3
- package/dist/commands/api-client.js +3 -3
- package/dist/commands/memory/read.js +2 -1
- package/dist/commands/node/bash.js +6 -6
- package/dist/commands/node/message.js +3 -3
- package/dist/commands/sys/daemon.js +11 -13
- package/dist/core/__tests__/broker-stream-watchdog-floor.test.js +0 -2
- package/dist/core/__tests__/daemon-boot.test.js +1 -1
- package/dist/core/__tests__/helpers/harness.d.ts +2 -0
- package/dist/core/__tests__/helpers/harness.js +9 -0
- package/dist/core/__tests__/integration/worktree-land.test.js +50 -0
- package/dist/core/__tests__/integration/worktree-reap.test.js +184 -5
- package/dist/core/__tests__/parse-argv-stdin-secret.test.js +14 -0
- package/dist/core/__tests__/seam/broker-attach-stream.test.js +13 -0
- package/dist/core/__tests__/seam/broker-startup-diagnostics.test.d.ts +1 -0
- package/dist/core/__tests__/seam/broker-startup-diagnostics.test.js +82 -0
- package/dist/core/bash-jobs.d.ts +20 -8
- package/dist/core/bash-jobs.js +40 -21
- package/dist/core/canvas/types.d.ts +4 -2
- package/dist/core/command.js +1 -1
- package/dist/core/fault-classifier.d.ts +1 -1
- package/dist/core/runtime/broker/event-projection.d.ts +0 -3
- package/dist/core/runtime/broker/event-projection.js +2 -15
- package/dist/core/runtime/broker/fault-retry.js +8 -0
- package/dist/core/runtime/broker-persona-guidance.js +12 -0
- package/dist/core/runtime/broker.js +0 -5
- package/dist/core/runtime/fault.js +1 -1
- package/dist/core/runtime/host.js +10 -1
- package/dist/core/runtime/spawn.js +5 -6
- package/dist/core/worktree-close.d.ts +4 -0
- package/dist/core/worktree-close.js +225 -0
- package/dist/core/worktree-containment.d.ts +14 -0
- package/dist/core/worktree-containment.js +48 -0
- package/dist/core/worktree-mutation-async.d.ts +13 -0
- package/dist/core/worktree-mutation-async.js +281 -0
- package/dist/core/worktree-sweep.js +17 -79
- package/dist/core/worktree.d.ts +1 -0
- package/dist/core/worktree.js +1 -1
- package/dist/daemon/__tests__/integration/api-startup-readiness.test.js +10 -8
- package/dist/daemon/api/__tests__/seam/api-server.test.js +70 -1
- package/dist/daemon/api/handlers/bash-jobs.js +1 -1
- package/dist/daemon/api/handlers/messages.js +7 -5
- package/dist/daemon/api/handlers/reports.js +3 -2
- package/dist/daemon/api/handlers/worktree.js +7 -6
- package/dist/daemon/fleet.d.ts +1 -1
- package/dist/daemon/fleet.js +30 -12
- package/dist/daemon/manage.d.ts +8 -5
- package/dist/daemon/manage.js +63 -40
- package/dist/daemon/reconcilers/managed-worktree-sweep.js +12 -0
- package/dist/daemon/reconcilers/node-lifecycle/tick.d.ts +0 -6
- package/dist/daemon/reconcilers/node-lifecycle/tick.js +1 -13
- package/dist/daemon/reconcilers/node-lifecycle/wants-execution.d.ts +5 -0
- package/dist/daemon/reconcilers/node-lifecycle/wants-execution.js +11 -0
- package/dist/pi-extensions/__tests__/canvas-context-intro.test.js +83 -0
- package/dist/pi-extensions/__tests__/integration/canvas-bash-valve.test.d.ts +1 -0
- package/dist/pi-extensions/__tests__/integration/canvas-bash-valve.test.js +133 -0
- package/dist/pi-extensions/canvas-bash-valve.d.ts +2 -0
- package/dist/pi-extensions/canvas-bash-valve.js +73 -43
- package/dist/pi-extensions/canvas-inbox-watcher.js +0 -2
- package/package.json +1 -1
- package/runtime.lock.json +5 -5
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
// Run with: npm run build && node --test --test-timeout=180000 dist/core/__tests__/seam/broker-startup-diagnostics.test.js
|
|
2
|
+
//
|
|
3
|
+
// A refused broker launch, a broker fatal emitted before session_start, and a
|
|
4
|
+
// SIGKILL all surface through canonical diagnostics. broker.log remains raw
|
|
5
|
+
// stdout/stderr residue, so a canonical fatal need not write anything there.
|
|
6
|
+
import { after, before, test } from 'node:test';
|
|
7
|
+
import assert from 'node:assert/strict';
|
|
8
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
9
|
+
import { spawnSync } from 'node:child_process';
|
|
10
|
+
import { join } from 'node:path';
|
|
11
|
+
import { subscribe } from '../../canvas/canvas.js';
|
|
12
|
+
import { resolveRef } from '../../canvas/history.js';
|
|
13
|
+
import { readLogSnapshot } from '../../events/read.js';
|
|
14
|
+
import { spawnChild } from '../../runtime/spawn.js';
|
|
15
|
+
import { FAIL_BEFORE_SESSION_START } from '../fixtures/fake-engine.js';
|
|
16
|
+
import { createHeadlessHarness } from '../helpers/harness.js';
|
|
17
|
+
let h;
|
|
18
|
+
let root;
|
|
19
|
+
function eventsFor(nodeId) {
|
|
20
|
+
return readLogSnapshot({ nodeId }, false).items.flatMap((item) => item.kind === 'event' ? [item.envelope] : []);
|
|
21
|
+
}
|
|
22
|
+
function reportBody(entry) {
|
|
23
|
+
assert.ok(entry.ref, 'the boot-failure notification has a report reference');
|
|
24
|
+
const report = resolveRef(entry.ref);
|
|
25
|
+
assert.ok(report, 'the boot-failure notification report resolves');
|
|
26
|
+
return report.body;
|
|
27
|
+
}
|
|
28
|
+
before(async () => {
|
|
29
|
+
h = await createHeadlessHarness({ sessionPrefix: 'crtr-broker-startup-diagnostics' });
|
|
30
|
+
root = h.spawnRoot('broker startup diagnostics seam root');
|
|
31
|
+
});
|
|
32
|
+
after(async () => {
|
|
33
|
+
if (h !== undefined)
|
|
34
|
+
await h.dispose();
|
|
35
|
+
});
|
|
36
|
+
test('broker startup failures name canonical diagnostics, preserve launch errors, and report the observed exit status', { timeout: 60_000 }, async () => {
|
|
37
|
+
const refusedId = 'launch-refused';
|
|
38
|
+
const missingCwd = join(h.home, 'missing-cwd');
|
|
39
|
+
await assert.rejects(spawnChild({
|
|
40
|
+
kind: 'general',
|
|
41
|
+
cwd: missingCwd,
|
|
42
|
+
prompt: 'this launch must be refused by the operating system',
|
|
43
|
+
parent: root,
|
|
44
|
+
nodeId: refusedId,
|
|
45
|
+
}), (error) => {
|
|
46
|
+
const details = error.details;
|
|
47
|
+
assert.match(String(details?.next), new RegExp(`crtr sys logs --node ${refusedId}`));
|
|
48
|
+
return true;
|
|
49
|
+
});
|
|
50
|
+
const launchFailure = await h.waitFor(() => eventsFor(refusedId).find((event) => event.event === 'broker.launch.failed') ?? null, { label: 'canonical launch refusal' });
|
|
51
|
+
assert.match(launchFailure.error?.message ?? '', /ENOENT/, 'the canonical event preserves the operating-system launch error');
|
|
52
|
+
const refusedLog = join(h.home, 'nodes', refusedId, 'job', 'broker.log');
|
|
53
|
+
assert.equal(existsSync(refusedLog), true, 'the raw broker log exists for the refused launch');
|
|
54
|
+
assert.equal(readFileSync(refusedLog, 'utf8'), '', 'a launch refusal writes its diagnostic canonically, not as raw broker output');
|
|
55
|
+
const fatal = await spawnChild({
|
|
56
|
+
kind: 'general',
|
|
57
|
+
cwd: h.node(root).cwd,
|
|
58
|
+
prompt: `canonical fatal ${FAIL_BEFORE_SESSION_START}`,
|
|
59
|
+
parent: root,
|
|
60
|
+
});
|
|
61
|
+
const fatalId = fatal.node.node_id;
|
|
62
|
+
await h.awaitFleetExit(fatalId);
|
|
63
|
+
await h.tick();
|
|
64
|
+
const fatalEvent = eventsFor(fatalId).find((event) => event.event === 'broker.runtime.fatal');
|
|
65
|
+
assert.ok(fatalEvent, 'a pre-session broker fatal is available in canonical node diagnostics');
|
|
66
|
+
assert.match(fatalEvent.error?.message ?? '', /simulated pre-session_start boot failure/);
|
|
67
|
+
const fatalLog = join(h.home, 'nodes', fatalId, 'job', 'broker.log');
|
|
68
|
+
assert.equal(readFileSync(fatalLog, 'utf8'), '', 'a successfully emitted canonical fatal leaves raw broker.log empty');
|
|
69
|
+
const fatalNotice = h.inbox(root).find((entry) => entry.from === fatalId && entry.tier === 'urgent');
|
|
70
|
+
assert.ok(fatalNotice, 'the parent receives the fatal broker boot-failure notification');
|
|
71
|
+
assert.match(reportBody(fatalNotice), /code 1, signal null/);
|
|
72
|
+
assert.match(reportBody(fatalNotice), new RegExp(`crtr sys logs --node ${fatalId}`));
|
|
73
|
+
const exited = spawnSync(process.execPath, ['-e', ''], { stdio: 'ignore' });
|
|
74
|
+
assert.ok(exited.pid !== undefined, 'the fabricated boot failure has a recorded, dead pid');
|
|
75
|
+
const killedId = h.fabricateBrokerNode({ id: 'killed-before-session', parent: root, pi_pid: exited.pid });
|
|
76
|
+
subscribe(root, killedId, true);
|
|
77
|
+
await h.deliverExit(killedId, { code: null, signal: 'SIGKILL' });
|
|
78
|
+
await h.tick();
|
|
79
|
+
const killNotice = h.inbox(root).find((entry) => entry.from === killedId && entry.tier === 'urgent');
|
|
80
|
+
assert.ok(killNotice, 'the parent receives the SIGKILL boot-failure notification');
|
|
81
|
+
assert.match(reportBody(killNotice), /code null, signal SIGKILL/);
|
|
82
|
+
});
|
package/dist/core/bash-jobs.d.ts
CHANGED
|
@@ -12,6 +12,9 @@ export interface BashJobPaths {
|
|
|
12
12
|
* Inspector's cancel) — without it on disk, only the agent's own handoff
|
|
13
13
|
* notice knows the group to signal. */
|
|
14
14
|
jobPgid: string;
|
|
15
|
+
/** Launch-time identity of the detached supervisor, used to refuse a PID that
|
|
16
|
+
* has since been reused by an unrelated process. */
|
|
17
|
+
jobPgidIdentity: string;
|
|
15
18
|
/** Human-readable label supplied with the bash call, when any. */
|
|
16
19
|
jobPurpose: string;
|
|
17
20
|
/** Epoch-ms cancel deadline for an AUTO-backgrounded job. Present only when
|
|
@@ -33,6 +36,9 @@ export declare function bashJobPaths(contextDir: string, jobId: string): BashJob
|
|
|
33
36
|
/** The job's supervisor process group, or undefined for a job started before
|
|
34
37
|
* pgid was persisted (or a partially-written dir). */
|
|
35
38
|
export declare function readJobPgid(contextDir: string, jobId: string): number | undefined;
|
|
39
|
+
/** Launch-time identity of the supervisor. Missing for jobs created before
|
|
40
|
+
* process-identity persistence, which are deliberately not safe to signal. */
|
|
41
|
+
export declare function readJobPgidIdentity(contextDir: string, jobId: string): string | undefined;
|
|
36
42
|
/** The optional human-readable job label. Jobs created before labels existed
|
|
37
43
|
* have no file and deliberately project null. */
|
|
38
44
|
export declare function readJobPurpose(contextDir: string, jobId: string): string | null;
|
|
@@ -73,6 +79,12 @@ export declare function clearJobDeadline(paths: BashJobPaths): void;
|
|
|
73
79
|
* means the group exists but cannot be signaled by this process; it is still
|
|
74
80
|
* live for the status surface. */
|
|
75
81
|
export declare function isBashJobProcessGroupAlive(pgid: number): boolean;
|
|
82
|
+
/** Best-effort cancellation for a detached bash supervisor. It snapshots the
|
|
83
|
+
* observable descendant tree while the supervisor is still alive, signals its
|
|
84
|
+
* leaves before their parents, then uses the shared identity-guarded KILL
|
|
85
|
+
* escalation. A descendant that detached and reparented before this snapshot
|
|
86
|
+
* is outside this guarantee. */
|
|
87
|
+
export declare function cancelBashJobProcessTree(pgid: number, expectedIdentity: string): Promise<boolean>;
|
|
76
88
|
export type StopBackgroundBashJobResult = {
|
|
77
89
|
kind: 'stopped';
|
|
78
90
|
signaled: boolean;
|
|
@@ -87,15 +99,15 @@ export type StopBackgroundBashJobResult = {
|
|
|
87
99
|
kind: 'not-stoppable';
|
|
88
100
|
logPath: string;
|
|
89
101
|
};
|
|
90
|
-
/** Stop one live background job using the same
|
|
91
|
-
*
|
|
92
|
-
*
|
|
102
|
+
/** Stop one live background job using the same best-effort tree cancellation
|
|
103
|
+
* semantics as inline abort and timeout. The caller owns the user-facing
|
|
104
|
+
* notification; this primitive owns only the job control-plane mutation.
|
|
93
105
|
*
|
|
94
|
-
*
|
|
95
|
-
*
|
|
96
|
-
*
|
|
97
|
-
*
|
|
98
|
-
*
|
|
106
|
+
* Only a job in the live-background state the roster projects (job.bg present,
|
|
107
|
+
* job.exit absent) may be signaled. Job directories persist after the
|
|
108
|
+
* supervisor exits, so a retired, still-foreground, or already-stopped job
|
|
109
|
+
* would otherwise have its recorded pgid signaled long after the OS could have
|
|
110
|
+
* reused that id for an unrelated process group. */
|
|
99
111
|
export declare function stopBackgroundBashJob(contextDir: string, jobId: string): Promise<StopBackgroundBashJobResult>;
|
|
100
112
|
/** Last `count` lines of a job log, cheaply: read at most the trailing 64 KiB
|
|
101
113
|
* rather than the whole file, which for a long-running job can be enormous. */
|
package/dist/core/bash-jobs.js
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
import { randomBytes } from 'node:crypto';
|
|
6
6
|
import { closeSync, existsSync, mkdirSync, openSync, readdirSync, readFileSync, readSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
|
|
7
7
|
import { join } from 'node:path';
|
|
8
|
+
import { captureTeardownSnapshot, isAnyPidAlive, killProcessTreePids } from './canvas/pid.js';
|
|
8
9
|
const MAX_BASH_JOB_PURPOSE_BYTES = 280;
|
|
9
10
|
/** A purpose is display text, never shell input. Keep the file-backed control
|
|
10
11
|
* plane to one bounded, visible line so old or hand-written job directories
|
|
@@ -41,6 +42,7 @@ export function bashJobPaths(contextDir, jobId) {
|
|
|
41
42
|
jobBg: join(dir, 'job.bg'),
|
|
42
43
|
jobDone: join(dir, 'job.done'),
|
|
43
44
|
jobPgid: join(dir, 'job.pgid'),
|
|
45
|
+
jobPgidIdentity: join(dir, 'job.pgid-identity'),
|
|
44
46
|
jobPurpose: join(dir, 'purpose'),
|
|
45
47
|
jobDeadline: join(dir, 'job.deadline'),
|
|
46
48
|
};
|
|
@@ -56,6 +58,17 @@ export function readJobPgid(contextDir, jobId) {
|
|
|
56
58
|
return undefined;
|
|
57
59
|
}
|
|
58
60
|
}
|
|
61
|
+
/** Launch-time identity of the supervisor. Missing for jobs created before
|
|
62
|
+
* process-identity persistence, which are deliberately not safe to signal. */
|
|
63
|
+
export function readJobPgidIdentity(contextDir, jobId) {
|
|
64
|
+
try {
|
|
65
|
+
const identity = readFileSync(bashJobPaths(contextDir, jobId).jobPgidIdentity, 'utf8').trim();
|
|
66
|
+
return identity === '' ? undefined : identity;
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
return undefined;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
59
72
|
/** The optional human-readable job label. Jobs created before labels existed
|
|
60
73
|
* have no file and deliberately project null. */
|
|
61
74
|
export function readJobPurpose(contextDir, jobId) {
|
|
@@ -143,15 +156,31 @@ export function isBashJobProcessGroupAlive(pgid) {
|
|
|
143
156
|
return err.code === 'EPERM';
|
|
144
157
|
}
|
|
145
158
|
}
|
|
146
|
-
/**
|
|
147
|
-
*
|
|
148
|
-
*
|
|
159
|
+
/** Best-effort cancellation for a detached bash supervisor. It snapshots the
|
|
160
|
+
* observable descendant tree while the supervisor is still alive, signals its
|
|
161
|
+
* leaves before their parents, then uses the shared identity-guarded KILL
|
|
162
|
+
* escalation. A descendant that detached and reparented before this snapshot
|
|
163
|
+
* is outside this guarantee. */
|
|
164
|
+
export async function cancelBashJobProcessTree(pgid, expectedIdentity) {
|
|
165
|
+
const snapshot = captureTeardownSnapshot(pgid, expectedIdentity);
|
|
166
|
+
if (snapshot.reused || snapshot.tree.length === 0)
|
|
167
|
+
return false;
|
|
168
|
+
const tree = [...snapshot.tree].reverse();
|
|
169
|
+
const signaled = isAnyPidAlive(tree);
|
|
170
|
+
killProcessTreePids(tree, 'SIGTERM', snapshot.identities);
|
|
171
|
+
await new Promise((resolve) => setTimeout(resolve, 400));
|
|
172
|
+
killProcessTreePids(tree, 'SIGKILL', snapshot.identities);
|
|
173
|
+
return signaled;
|
|
174
|
+
}
|
|
175
|
+
/** Stop one live background job using the same best-effort tree cancellation
|
|
176
|
+
* semantics as inline abort and timeout. The caller owns the user-facing
|
|
177
|
+
* notification; this primitive owns only the job control-plane mutation.
|
|
149
178
|
*
|
|
150
|
-
*
|
|
151
|
-
*
|
|
152
|
-
*
|
|
153
|
-
*
|
|
154
|
-
*
|
|
179
|
+
* Only a job in the live-background state the roster projects (job.bg present,
|
|
180
|
+
* job.exit absent) may be signaled. Job directories persist after the
|
|
181
|
+
* supervisor exits, so a retired, still-foreground, or already-stopped job
|
|
182
|
+
* would otherwise have its recorded pgid signaled long after the OS could have
|
|
183
|
+
* reused that id for an unrelated process group. */
|
|
155
184
|
export async function stopBackgroundBashJob(contextDir, jobId) {
|
|
156
185
|
const paths = bashJobPaths(contextDir, jobId);
|
|
157
186
|
if (!existsSync(paths.dir))
|
|
@@ -159,7 +188,8 @@ export async function stopBackgroundBashJob(contextDir, jobId) {
|
|
|
159
188
|
if (!existsSync(paths.jobBg) || existsSync(paths.jobExit))
|
|
160
189
|
return { kind: 'retired', logPath: paths.jobLog };
|
|
161
190
|
const pgid = readJobPgid(contextDir, jobId);
|
|
162
|
-
|
|
191
|
+
const pgidIdentity = readJobPgidIdentity(contextDir, jobId);
|
|
192
|
+
if (pgid === undefined || pgidIdentity === undefined)
|
|
163
193
|
return { kind: 'not-stoppable', logPath: paths.jobLog };
|
|
164
194
|
// Claim the stop by creating job.exit exclusively, before signaling. The
|
|
165
195
|
// loser of two concurrent stops — and a stop that races the supervisor's own
|
|
@@ -179,18 +209,7 @@ export async function stopBackgroundBashJob(contextDir, jobId) {
|
|
|
179
209
|
return { kind: 'retired', logPath: paths.jobLog };
|
|
180
210
|
throw err;
|
|
181
211
|
}
|
|
182
|
-
|
|
183
|
-
try {
|
|
184
|
-
process.kill(-pgid, 'SIGTERM');
|
|
185
|
-
}
|
|
186
|
-
catch {
|
|
187
|
-
signaled = false;
|
|
188
|
-
}
|
|
189
|
-
await new Promise((resolve) => setTimeout(resolve, 400));
|
|
190
|
-
try {
|
|
191
|
-
process.kill(-pgid, 'SIGKILL');
|
|
192
|
-
}
|
|
193
|
-
catch { /* already gone */ }
|
|
212
|
+
const signaled = await cancelBashJobProcessTree(pgid, pgidIdentity);
|
|
194
213
|
return { kind: 'stopped', signaled, pgid, logPath: paths.jobLog };
|
|
195
214
|
}
|
|
196
215
|
/** Last `count` lines of a job log, cheaply: read at most the trailing 64 KiB
|
|
@@ -76,11 +76,13 @@ export interface LaunchSpec {
|
|
|
76
76
|
env: Record<string, string>;
|
|
77
77
|
}
|
|
78
78
|
/** The repository fingerprint a refusal was decided against. An examination
|
|
79
|
-
* that reads the same
|
|
80
|
-
*
|
|
79
|
+
* that reads the same values can skip the whole proof cascade: nothing about
|
|
80
|
+
* the refusal can have changed. */
|
|
81
81
|
export interface ManagedWorktreeObservation {
|
|
82
82
|
/** Tip of `refs/heads/<base_ref>`, or null when it no longer resolves. */
|
|
83
83
|
base: string | null;
|
|
84
|
+
/** Tip of the base branch's upstream (or `origin/<base_ref>` fallback). */
|
|
85
|
+
upstream: string | null;
|
|
84
86
|
/** Tip of `refs/heads/<branch>`, or null when the branch is gone. */
|
|
85
87
|
branch: string | null;
|
|
86
88
|
/** `absent`, or a hash of the checkout's HEAD plus `git status --porcelain`. */
|
package/dist/core/command.js
CHANGED
|
@@ -403,7 +403,7 @@ export async function parseArgv(params, tokens, options) {
|
|
|
403
403
|
continue;
|
|
404
404
|
}
|
|
405
405
|
const rawVal = inlineValue !== undefined ? inlineValue : tokens[++i];
|
|
406
|
-
if (rawVal === undefined || rawVal.startsWith('--')) {
|
|
406
|
+
if (rawVal === undefined || (inlineValue === undefined && rawVal.startsWith('--'))) {
|
|
407
407
|
const focusedNext = flagDef.focusedHelp !== undefined && (options?.leafPath?.length ?? 0) > 0
|
|
408
408
|
? `Run \`crtr ${options.leafPath.join(' ')} --${flagName} -h\` and read the focused value contract before retrying.`
|
|
409
409
|
: undefined;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ErrorClass, OperationId } from './events/types.js';
|
|
2
|
-
export type FaultLink = 'pi→provider' | 'viewer↔broker' | 'relay↔broker' | 'viewer↔crtrd' | 'daemon→node' | 'crtr→pi';
|
|
2
|
+
export type FaultLink = 'pi→provider' | 'viewer↔broker' | 'relay↔broker' | 'viewer↔crtrd' | 'broker↔crtrd' | 'daemon→node' | 'crtr→pi';
|
|
3
3
|
export type FaultKind = 'rate-limit' | 'overloaded' | 'connection' | 'auth' | 'protocol' | 'context-overflow' | 'other' | 'wedged' | 'model-not-found';
|
|
4
4
|
export type FaultRetryDisposition = 'auto' | 'manual' | 'fatal';
|
|
5
5
|
export type FaultRetryOwner = 'sdk' | 'daemon' | 'client';
|
|
@@ -11,8 +11,6 @@ type EventProjectionDeps = {
|
|
|
11
11
|
notifyTurnAccepted: () => void;
|
|
12
12
|
emitStartupMilestone: (event: string, model: string | null) => void;
|
|
13
13
|
formatModelSpec: (model: BrokerSession['model'], thinkingLevel?: string) => string | null;
|
|
14
|
-
refreshIntent: () => Promise<boolean>;
|
|
15
|
-
onRefreshIntentError: (error: unknown) => void;
|
|
16
14
|
};
|
|
17
15
|
/** Projects one live engine event stream to broker viewers and owns its relay timers. */
|
|
18
16
|
export declare class EventProjection {
|
|
@@ -40,7 +38,6 @@ export declare class EventProjection {
|
|
|
40
38
|
private flushPendingUpdate;
|
|
41
39
|
private invalidatePendingBroadcasts;
|
|
42
40
|
private broadcast;
|
|
43
|
-
private broadcastMessageEnd;
|
|
44
41
|
private setPendingBroadcast;
|
|
45
42
|
}
|
|
46
43
|
export {};
|
|
@@ -128,8 +128,8 @@ export class EventProjection {
|
|
|
128
128
|
return;
|
|
129
129
|
}
|
|
130
130
|
this.flushPendingUpdate();
|
|
131
|
-
if (type === 'message_end') {
|
|
132
|
-
this.
|
|
131
|
+
if (type === 'message_end' && generation.stagedRefreshAbort) {
|
|
132
|
+
this.broadcast({ ...event, message: withoutYieldAbort(event.message) });
|
|
133
133
|
return;
|
|
134
134
|
}
|
|
135
135
|
this.broadcast(event);
|
|
@@ -183,19 +183,6 @@ export class EventProjection {
|
|
|
183
183
|
this.deps.registry.broadcast(frame);
|
|
184
184
|
}));
|
|
185
185
|
}
|
|
186
|
-
broadcastMessageEnd(event) {
|
|
187
|
-
const preceding = this.pendingBroadcast ?? Promise.resolve();
|
|
188
|
-
const generation = this.broadcastGeneration;
|
|
189
|
-
this.setPendingBroadcast(preceding.then(async () => {
|
|
190
|
-
const refresh = await this.deps.refreshIntent();
|
|
191
|
-
if (this.broadcastGeneration !== generation)
|
|
192
|
-
return;
|
|
193
|
-
this.deps.registry.broadcast(refresh ? { ...event, message: withoutYieldAbort(event.message) } : event);
|
|
194
|
-
}).catch((error) => {
|
|
195
|
-
if (this.broadcastGeneration === generation)
|
|
196
|
-
this.deps.onRefreshIntentError(error);
|
|
197
|
-
}));
|
|
198
|
-
}
|
|
199
186
|
setPendingBroadcast(work) {
|
|
200
187
|
this.pendingBroadcast = work;
|
|
201
188
|
void work.then(() => {
|
|
@@ -71,6 +71,14 @@ export class FaultRetry {
|
|
|
71
71
|
// admitted retry therefore carries from its durable episode, while an
|
|
72
72
|
// initial provider failure still reads from the ordinary marker.
|
|
73
73
|
const prior = readFault(this.deps.nodeId);
|
|
74
|
+
// A persona gate runs before Pi contacts a provider. It records a typed
|
|
75
|
+
// local API transport failure, then Pi represents that thrown error as a
|
|
76
|
+
// generic provider-looking agent_end. Keep its actual boundary instead of
|
|
77
|
+
// turning it into a fatal pi→provider `other` fault.
|
|
78
|
+
if (prior?.link === 'broker↔crtrd' && prior.kind === 'connection') {
|
|
79
|
+
clearProviderRetryEpisode(this.deps.nodeId);
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
74
82
|
const durableEpisode = readProviderRetryEpisode(this.deps.nodeId);
|
|
75
83
|
const episodeFault = this.isActiveAutoFault(prior) && prior.link === 'pi→provider'
|
|
76
84
|
? prior
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { contextDir } from '../canvas/paths.js';
|
|
2
|
+
import { isDaemonTransportApiError } from '../../api/errors.js';
|
|
2
3
|
import { emitEvent } from '../events/emit.js';
|
|
4
|
+
import { clearFault, recordFault } from './fault.js';
|
|
3
5
|
import { brokerExtensionState, commitBrokerPersonaAck } from './broker/daemon-ops.js';
|
|
4
6
|
import { readRoadmap, roadmapPath } from './roadmap.js';
|
|
5
7
|
import { buildSubPersonaMenu, renderConfigChangeDelta } from '../substrate/render.js';
|
|
@@ -137,9 +139,19 @@ export function installPersonaTransitionGate(nodeId, session) {
|
|
|
137
139
|
for (const context of new Set([active, messages, transformed]))
|
|
138
140
|
context.push(message);
|
|
139
141
|
});
|
|
142
|
+
clearFault(nodeId, { link: 'broker↔crtrd' });
|
|
140
143
|
return transformed;
|
|
141
144
|
}
|
|
142
145
|
catch (error) {
|
|
146
|
+
if (isDaemonTransportApiError(error)) {
|
|
147
|
+
recordFault(nodeId, {
|
|
148
|
+
link: 'broker↔crtrd',
|
|
149
|
+
op: 'persona transition',
|
|
150
|
+
kind: 'connection',
|
|
151
|
+
retry: { disposition: 'manual' },
|
|
152
|
+
message: error.message,
|
|
153
|
+
});
|
|
154
|
+
}
|
|
143
155
|
emitEvent({
|
|
144
156
|
event: 'broker.daemon.operation.failed',
|
|
145
157
|
level: 'error',
|
|
@@ -375,11 +375,6 @@ export async function runBroker(nodeId, startupAt = process.hrtime.bigint(), sta
|
|
|
375
375
|
notifyTurnAccepted: () => notifyTurnAccepted(),
|
|
376
376
|
emitStartupMilestone,
|
|
377
377
|
formatModelSpec,
|
|
378
|
-
refreshIntent: async () => (await brokerExtensionState(nodeId)).node.intent === 'refresh',
|
|
379
|
-
onRefreshIntentError: (error) => {
|
|
380
|
-
emitEvent({ level: 'error', event: 'broker.daemon_state.read_failed', error });
|
|
381
|
-
disposeAndExit('daemon-state-read-failed', 1);
|
|
382
|
-
},
|
|
383
378
|
});
|
|
384
379
|
const buildSnapshot = (client) => {
|
|
385
380
|
const liveSession = rebind.session();
|
|
@@ -31,7 +31,7 @@ function faultPriority(fault) {
|
|
|
31
31
|
return 2;
|
|
32
32
|
return 1;
|
|
33
33
|
}
|
|
34
|
-
const faultLinks = ['pi→provider', 'viewer↔broker', 'relay↔broker', 'viewer↔crtrd', 'daemon→node', 'crtr→pi'];
|
|
34
|
+
const faultLinks = ['pi→provider', 'viewer↔broker', 'relay↔broker', 'viewer↔crtrd', 'broker↔crtrd', 'daemon→node', 'crtr→pi'];
|
|
35
35
|
const faultKinds = ['rate-limit', 'overloaded', 'connection', 'auth', 'protocol', 'context-overflow', 'other', 'wedged', 'model-not-found'];
|
|
36
36
|
function parseFault(raw) {
|
|
37
37
|
try {
|
|
@@ -319,7 +319,16 @@ export const headlessBrokerHost = {
|
|
|
319
319
|
});
|
|
320
320
|
finish({ code, signal });
|
|
321
321
|
});
|
|
322
|
-
spawned.once('error', () =>
|
|
322
|
+
spawned.once('error', (error) => {
|
|
323
|
+
emitEvent({
|
|
324
|
+
level: 'error',
|
|
325
|
+
event: 'broker.launch.failed',
|
|
326
|
+
node_id: nodeId,
|
|
327
|
+
error,
|
|
328
|
+
fields: { uptime_ms: Date.now() - launchedAt },
|
|
329
|
+
});
|
|
330
|
+
finish({ code: null, signal: null });
|
|
331
|
+
});
|
|
323
332
|
});
|
|
324
333
|
spawned.unref();
|
|
325
334
|
const handle = { pid: spawned.pid ?? null, exited };
|
|
@@ -15,7 +15,7 @@ import { spawnNode, currentNodeContext, rootOfSpine, newNodeId, preflightNodeId
|
|
|
15
15
|
import { resolveProfileOperand } from '../profiles/manifest.js';
|
|
16
16
|
import { selectProfileForCwd, selectProfileForCwdReadOnly } from '../profiles/select.js';
|
|
17
17
|
import { buildLaunchSpecAsync, buildPiArgv } from './launch.js';
|
|
18
|
-
import {
|
|
18
|
+
import { createManagedWorktreeAsync, rollbackManagedWorktreeAsync } from '../worktree-mutation-async.js';
|
|
19
19
|
import { usage, brokerLaunchFailed } from '../errors.js';
|
|
20
20
|
import { writeGoal } from './kickoff.js';
|
|
21
21
|
import { appendSituationalContext, formatSituationalProse } from './situational-context.js';
|
|
@@ -25,7 +25,6 @@ import { encodeKickoffOrigin, KICKOFF_ORIGIN_ENV } from './stamp/protocol.js';
|
|
|
25
25
|
import { canonicalSessionFile, contextDir, findNodeBySessionFile, getNode, fullName, recordPid, setFrozen } from '../canvas/index.js';
|
|
26
26
|
import { boundFleet, brokerThresholdsForDaemon } from './fleet.js';
|
|
27
27
|
import { emitEvent } from '../events/emit.js';
|
|
28
|
-
import { jobDir } from '../canvas/paths.js';
|
|
29
28
|
import { openViewerWindow, focusOf, windowOfPane, } from './placement.js';
|
|
30
29
|
import { waitForBrokerViewSocket } from './placement-tmux.js';
|
|
31
30
|
import { transition } from './lifecycle.js';
|
|
@@ -259,7 +258,7 @@ export async function spawnChildPrepared(opts, beforeBrokerLaunch) {
|
|
|
259
258
|
let spawnCwd = opts.cwd;
|
|
260
259
|
try {
|
|
261
260
|
if (wantsWorktree) {
|
|
262
|
-
managedWorktree =
|
|
261
|
+
managedWorktree = await createManagedWorktreeAsync(opts.worktreeCwd ?? opts.cwd, nodeId, opts.worktreeBase);
|
|
263
262
|
spawnCwd = managedWorktree.path;
|
|
264
263
|
}
|
|
265
264
|
// Spine: a managed child reports up to its spawner (has a manager); an
|
|
@@ -409,7 +408,7 @@ export async function spawnChildPrepared(opts, beforeBrokerLaunch) {
|
|
|
409
408
|
if (placed.pid === null) {
|
|
410
409
|
const error = new Error(`broker host returned no pid for ${meta.node_id}`);
|
|
411
410
|
transition(meta.node_id, 'crash', { reason: 'launch_failed', outcome: launchFailureOutcome(error) });
|
|
412
|
-
throw brokerLaunchFailed(`failed to launch the broker engine for ${meta.node_id} (${meta.name}) — the node was not started.`, `Inspect
|
|
411
|
+
throw brokerLaunchFailed(`failed to launch the broker engine for ${meta.node_id} (${meta.name}) — the node was not started.`, `Inspect canonical event diagnostics with \`crtr sys logs --node ${meta.node_id}\`, then verify the pi engine can start in this environment.`);
|
|
413
412
|
}
|
|
414
413
|
recordPid(meta.node_id, placed.pid);
|
|
415
414
|
// A root created through `node new --root` must accept viewers before the
|
|
@@ -425,7 +424,7 @@ export async function spawnChildPrepared(opts, beforeBrokerLaunch) {
|
|
|
425
424
|
? `the root broker ${meta.node_id} could not start — ${why}`
|
|
426
425
|
: `the root broker ${meta.node_id} never bound its view socket — it was not started.`, why !== null
|
|
427
426
|
? 'Resolve the cause named above, then retry.'
|
|
428
|
-
: `The broker engine exited before binding its view socket. Inspect
|
|
427
|
+
: `The broker engine exited before binding its view socket. Inspect canonical event diagnostics with \`crtr sys logs --node ${meta.node_id}\`, then retry.`);
|
|
429
428
|
}
|
|
430
429
|
// A --root opens NO viewer from here. spawnChild runs daemon-side only, and
|
|
431
430
|
// the daemon is paneless by construction (manage.ts strips TMUX/TMUX_PANE
|
|
@@ -484,7 +483,7 @@ export async function spawnChildPrepared(opts, beforeBrokerLaunch) {
|
|
|
484
483
|
// worktree — the node stays alive (crashed, if the launch itself failed)
|
|
485
484
|
// with its worktree intact rather than pinned to a deleted path.
|
|
486
485
|
if (managedWorktree !== undefined && nodeId !== undefined && getNode(nodeId) === null) {
|
|
487
|
-
|
|
486
|
+
await rollbackManagedWorktreeAsync(managedWorktree);
|
|
488
487
|
}
|
|
489
488
|
throw err;
|
|
490
489
|
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { type CloseManagedWorktreeResult } from './worktree.js';
|
|
2
|
+
/** Async daemon executor for explicit close. The lock wait and every Git
|
|
3
|
+
* subprocess yield to the daemon event loop. */
|
|
4
|
+
export declare function closeManagedWorktreeAsync(nodeId: string): Promise<CloseManagedWorktreeResult>;
|