@north-light/crouter 0.3.302 → 0.3.304
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/__tests__/integration/client.test.js +37 -36
- package/dist/api/client.d.ts +32 -55
- package/dist/api/client.js +102 -98
- package/dist/api/dto/messages.d.ts +6 -5
- package/dist/api/errors.d.ts +3 -0
- package/dist/api/errors.js +10 -0
- package/dist/api/index.d.ts +1 -1
- package/dist/api/index.js +1 -1
- package/dist/builtin-memory/internal/plugins.md +1 -1
- package/dist/clients/attach/viewer.js +532 -532
- package/dist/commands/__tests__/seam/daemon-status.test.d.ts +1 -0
- package/dist/commands/__tests__/seam/daemon-status.test.js +29 -0
- package/dist/commands/api-client.d.ts +3 -3
- package/dist/commands/api-client.js +3 -3
- package/dist/commands/memory/read.js +2 -1
- package/dist/commands/node/bash.js +6 -6
- package/dist/commands/node/message.js +3 -3
- package/dist/commands/sys/daemon.js +11 -13
- package/dist/core/__tests__/bash-guard.test.d.ts +1 -0
- package/dist/core/__tests__/bash-guard.test.js +190 -0
- package/dist/core/__tests__/broker-stream-watchdog-floor.test.js +0 -2
- package/dist/core/__tests__/daemon-boot.test.js +1 -1
- package/dist/core/__tests__/fixtures/fake-engine.d.ts +6 -0
- package/dist/core/__tests__/fixtures/fake-engine.js +51 -11
- package/dist/core/__tests__/helpers/harness.d.ts +2 -0
- package/dist/core/__tests__/helpers/harness.js +9 -0
- package/dist/core/__tests__/integration/worktree-land.test.js +50 -0
- package/dist/core/__tests__/integration/worktree-reap.test.js +184 -5
- package/dist/core/__tests__/parse-argv-stdin-secret.test.js +14 -0
- package/dist/core/__tests__/seam/broker-attach-stream.test.js +13 -0
- package/dist/core/__tests__/seam/broker-provider-retry.test.js +57 -0
- package/dist/core/__tests__/seam/broker-startup-diagnostics.test.d.ts +1 -0
- package/dist/core/__tests__/seam/broker-startup-diagnostics.test.js +82 -0
- package/dist/core/bash-guard.d.ts +6 -0
- package/dist/core/bash-guard.js +393 -0
- package/dist/core/bash-jobs.d.ts +20 -8
- package/dist/core/bash-jobs.js +40 -21
- package/dist/core/canvas/types.d.ts +4 -2
- package/dist/core/command.js +1 -1
- package/dist/core/fault-classifier.d.ts +1 -1
- package/dist/core/runtime/broker/event-projection.d.ts +0 -3
- package/dist/core/runtime/broker/event-projection.js +2 -15
- package/dist/core/runtime/broker/fault-retry.d.ts +4 -0
- package/dist/core/runtime/broker/fault-retry.js +74 -6
- package/dist/core/runtime/broker-persona-guidance.js +12 -0
- package/dist/core/runtime/broker.js +0 -5
- package/dist/core/runtime/fault.js +1 -1
- package/dist/core/runtime/host.js +10 -1
- package/dist/core/runtime/spawn.js +5 -6
- package/dist/core/shell-segments.d.ts +42 -0
- package/dist/core/shell-segments.js +169 -0
- package/dist/core/substrate/surface-match.d.ts +0 -9
- package/dist/core/substrate/surface-match.js +4 -160
- package/dist/core/worktree-close.d.ts +4 -0
- package/dist/core/worktree-close.js +225 -0
- package/dist/core/worktree-containment.d.ts +14 -0
- package/dist/core/worktree-containment.js +48 -0
- package/dist/core/worktree-mutation-async.d.ts +13 -0
- package/dist/core/worktree-mutation-async.js +281 -0
- package/dist/core/worktree-sweep.js +17 -79
- package/dist/core/worktree.d.ts +1 -0
- package/dist/core/worktree.js +1 -1
- package/dist/daemon/__tests__/integration/api-startup-readiness.test.js +10 -8
- package/dist/daemon/api/__tests__/seam/api-server.test.js +70 -1
- package/dist/daemon/api/handlers/bash-jobs.js +1 -1
- package/dist/daemon/api/handlers/messages.js +7 -5
- package/dist/daemon/api/handlers/reports.js +3 -2
- package/dist/daemon/api/handlers/worktree.js +7 -6
- package/dist/daemon/fleet.d.ts +1 -1
- package/dist/daemon/fleet.js +30 -12
- package/dist/daemon/manage.d.ts +8 -5
- package/dist/daemon/manage.js +63 -40
- package/dist/daemon/reconcilers/managed-worktree-sweep.js +12 -0
- package/dist/daemon/reconcilers/node-lifecycle/tick.d.ts +0 -6
- package/dist/daemon/reconcilers/node-lifecycle/tick.js +1 -13
- package/dist/daemon/reconcilers/node-lifecycle/wants-execution.d.ts +5 -0
- package/dist/daemon/reconcilers/node-lifecycle/wants-execution.js +11 -0
- package/dist/pi-extensions/__tests__/canvas-context-intro.test.js +83 -0
- package/dist/pi-extensions/__tests__/integration/canvas-bash-valve.test.d.ts +1 -0
- package/dist/pi-extensions/__tests__/integration/canvas-bash-valve.test.js +133 -0
- package/dist/pi-extensions/canvas-bash-valve.d.ts +2 -3
- package/dist/pi-extensions/canvas-bash-valve.js +75 -77
- package/dist/pi-extensions/canvas-inbox-watcher.js +0 -2
- package/package.json +1 -1
- package/runtime.lock.json +5 -5
|
@@ -15,7 +15,8 @@ import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
|
|
|
15
15
|
import { join } from 'node:path';
|
|
16
16
|
import { getNode } from '../../../core/canvas/canvas.js';
|
|
17
17
|
import { push, FinalizationError } from '../../../core/feed/feed.js';
|
|
18
|
-
import {
|
|
18
|
+
import { branchIsPushedAtCurrentTip, hasOpenManagedWorktree, missingWorktreeRecoveryHint } from '../../../core/worktree.js';
|
|
19
|
+
import { autoDropCleanManagedWorktreeAsync } from '../../../core/worktree-mutation-async.js';
|
|
19
20
|
import { reportsDir } from '../../../core/canvas/paths.js';
|
|
20
21
|
import { notFound, usage } from '../../../core/errors.js';
|
|
21
22
|
import { InputError } from '../../../core/io.js';
|
|
@@ -89,7 +90,7 @@ async function handlePush(ctx) {
|
|
|
89
90
|
async function guardFinalWorktree(id) {
|
|
90
91
|
if (!hasOpenManagedWorktree(getNode(id)))
|
|
91
92
|
return undefined;
|
|
92
|
-
const dropped =
|
|
93
|
+
const dropped = await autoDropCleanManagedWorktreeAsync(id);
|
|
93
94
|
if (dropped !== null)
|
|
94
95
|
return dropped.worktree_path;
|
|
95
96
|
if (await branchIsPushedAtCurrentTip(getNode(id).managed_worktree))
|
|
@@ -12,21 +12,22 @@
|
|
|
12
12
|
// all of that onto `CrtrError` for every route that can raise it — so the CLI
|
|
13
13
|
// leaf can reconstruct the original agent-facing contract instead of collapsing
|
|
14
14
|
// to a generic status code.
|
|
15
|
-
import {
|
|
15
|
+
import { closeManagedWorktreeAsync } from '../../../core/worktree-close.js';
|
|
16
|
+
import { abandonManagedWorktreeAsync } from '../../../core/worktree-mutation-async.js';
|
|
16
17
|
import { listQuarantinedManagedWorktrees } from '../../../core/worktree-quarantine.js';
|
|
17
18
|
import { usage } from '../../../core/errors.js';
|
|
18
|
-
function handleWorktreeClose(ctx) {
|
|
19
|
+
async function handleWorktreeClose(ctx) {
|
|
19
20
|
const id = ctx.params['id'];
|
|
20
|
-
// No requireMeta:
|
|
21
|
+
// No requireMeta: closeManagedWorktreeAsync already throws the precise
|
|
21
22
|
// `unknown_node` / `no_managed_worktree` WorktreeError for a bad id.
|
|
22
|
-
return { status: 200, body:
|
|
23
|
+
return { status: 200, body: await closeManagedWorktreeAsync(id) };
|
|
23
24
|
}
|
|
24
|
-
function handleWorktreeAbandon(ctx) {
|
|
25
|
+
async function handleWorktreeAbandon(ctx) {
|
|
25
26
|
const rawBy = ctx.body?.['by'];
|
|
26
27
|
const by = typeof rawBy === 'string' ? rawBy.trim() : '';
|
|
27
28
|
if (by === '')
|
|
28
29
|
throw usage('abandon requires a non-empty invoking identity', { received: ctx.body, next: 'Pass the invoking node id or human identity.' });
|
|
29
|
-
return { status: 200, body:
|
|
30
|
+
return { status: 200, body: await abandonManagedWorktreeAsync(ctx.params['id'], by) };
|
|
30
31
|
}
|
|
31
32
|
async function handleQuarantinedWorktrees() {
|
|
32
33
|
return { status: 200, body: await listQuarantinedManagedWorktrees() };
|
package/dist/daemon/fleet.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { type NodeMeta } from '../core/canvas/index.js';
|
|
|
2
2
|
import { type FleetEntry, type FleetExitStatus, type FleetRegistry } from '../core/runtime/fleet.js';
|
|
3
3
|
import type { NodeOutcomeDetailV1, TerminalReason } from '../core/canvas/types.js';
|
|
4
4
|
import type { HostHandle } from '../core/runtime/host.js';
|
|
5
|
-
export declare function surfaceBootFailure(meta: NodeMeta): Promise<void>;
|
|
5
|
+
export declare function surfaceBootFailure(meta: NodeMeta, exitStatus?: FleetExitStatus): Promise<void>;
|
|
6
6
|
/** Whether a dead relaunch should resume the saved session strictly, or retry
|
|
7
7
|
* as a fresh cycle instead. A cycling attempt (cycle_pending) that dies before
|
|
8
8
|
* session_start must retry AS a cycle: the fresh branch it was opening is not
|
package/dist/daemon/fleet.js
CHANGED
|
@@ -34,16 +34,18 @@ import { brokerThresholdsForDaemon } from '../core/runtime/fleet.js';
|
|
|
34
34
|
* When the daemon observes that death it errors LOUDLY up the spine: an
|
|
35
35
|
* urgent push so the parent learns the child failed to launch instead of just
|
|
36
36
|
* seeing a silent `dead`. */
|
|
37
|
-
function bootFailureBody(meta) {
|
|
37
|
+
function bootFailureBody(meta, exitStatus) {
|
|
38
|
+
const status = exitStatus === undefined
|
|
39
|
+
? 'The daemon did not observe an exit status.'
|
|
40
|
+
: `The observed exit status was code ${exitStatus.code ?? 'null'}, signal ${exitStatus.signal ?? 'null'}.`;
|
|
38
41
|
return (`⚠ Spawn failed — \`${meta.name}\` (${meta.kind}) never started.\n\n` +
|
|
39
42
|
`Its pi vehicle exited before the session came up (no pi_session_id was ever ` +
|
|
40
|
-
`recorded), so the node produced no output.
|
|
41
|
-
`
|
|
42
|
-
`
|
|
43
|
-
`If the work still needs doing, re-spawn it; if spawns keep dying, spawn fewer at a time.`);
|
|
43
|
+
`recorded), so the node produced no output. ${status}\n\n` +
|
|
44
|
+
`Inspect canonical event diagnostics with \`crtr sys logs --node ${meta.node_id}\`.\n\n` +
|
|
45
|
+
`If the work still needs doing, re-spawn it.`);
|
|
44
46
|
}
|
|
45
|
-
export async function surfaceBootFailure(meta) {
|
|
46
|
-
await pushUrgent(meta.node_id, bootFailureBody(meta), { from: meta.node_id });
|
|
47
|
+
export async function surfaceBootFailure(meta, exitStatus) {
|
|
48
|
+
await pushUrgent(meta.node_id, bootFailureBody(meta, exitStatus), { from: meta.node_id });
|
|
47
49
|
}
|
|
48
50
|
async function surfaceCrashLoop(meta) {
|
|
49
51
|
const transportPoison = readTransportPoisonVerdict(meta.node_id);
|
|
@@ -137,6 +139,8 @@ export class DaemonFleet {
|
|
|
137
139
|
/** Brokers deliberately interrupted by profile pause. This belongs to the
|
|
138
140
|
* exact child, so a resume cannot turn its later SIGTERM exit into a crash. */
|
|
139
141
|
#profilePauseInterruptions = new Set();
|
|
142
|
+
/** Exit status held only until dead-row policy surfaces this broker's boot failure. */
|
|
143
|
+
#exitStatuses = new Map();
|
|
140
144
|
#lastCapacityLogAt = Number.NEGATIVE_INFINITY;
|
|
141
145
|
constructor(opts) {
|
|
142
146
|
this.#epoch = opts.epoch;
|
|
@@ -155,6 +159,7 @@ export class DaemonFleet {
|
|
|
155
159
|
throw new Error(`fleet: refusing to register ${nodeId} with no pid`);
|
|
156
160
|
}
|
|
157
161
|
this.#reservations.delete(nodeId);
|
|
162
|
+
this.#exitStatuses.delete(nodeId);
|
|
158
163
|
this.#map.set(nodeId, { pid: handle.pid, handle });
|
|
159
164
|
}
|
|
160
165
|
has(nodeId) {
|
|
@@ -215,6 +220,7 @@ export class DaemonFleet {
|
|
|
215
220
|
forget(nodeId) {
|
|
216
221
|
this.#map.delete(nodeId);
|
|
217
222
|
this.#reservations.delete(nodeId);
|
|
223
|
+
this.#exitStatuses.delete(nodeId);
|
|
218
224
|
}
|
|
219
225
|
onChildExit(nodeId, status) {
|
|
220
226
|
// The real child exit is the liveness boundary. Its queued observation only
|
|
@@ -225,6 +231,10 @@ export class DaemonFleet {
|
|
|
225
231
|
const uptimeMs = Number.isFinite(launchedMs) ? Math.max(0, this.#now() - launchedMs) : null;
|
|
226
232
|
const consecutiveFailures = meta?.respawn_failures ?? 0;
|
|
227
233
|
this.#map.delete(nodeId);
|
|
234
|
+
if (meta?.status === 'done' || meta?.status === 'canceled' || meta?.status === 'dead')
|
|
235
|
+
this.#exitStatuses.delete(nodeId);
|
|
236
|
+
else
|
|
237
|
+
this.#exitStatuses.set(nodeId, status);
|
|
228
238
|
this.#enqueue(() => this.#emitExitObserved(nodeId, status, uptimeMs, consecutiveFailures));
|
|
229
239
|
}
|
|
230
240
|
deliverExit(nodeId, status) {
|
|
@@ -232,7 +242,14 @@ export class DaemonFleet {
|
|
|
232
242
|
}
|
|
233
243
|
/** The node-lifecycle tick's enactment entry. */
|
|
234
244
|
async applyDeadRowPolicy(nodeId) {
|
|
235
|
-
|
|
245
|
+
const exitStatus = this.#exitStatuses.get(nodeId);
|
|
246
|
+
try {
|
|
247
|
+
await this.#applyPolicy(nodeId, exitStatus);
|
|
248
|
+
}
|
|
249
|
+
finally {
|
|
250
|
+
if (this.#exitStatuses.get(nodeId) === exitStatus)
|
|
251
|
+
this.#exitStatuses.delete(nodeId);
|
|
252
|
+
}
|
|
236
253
|
}
|
|
237
254
|
/** A revive refusal before its launch try/catch leaves the row active but
|
|
238
255
|
* with no fleet entry. Convert that otherwise-silent strand into the same
|
|
@@ -267,10 +284,10 @@ export class DaemonFleet {
|
|
|
267
284
|
},
|
|
268
285
|
});
|
|
269
286
|
}
|
|
270
|
-
async #applyPolicy(nodeId) {
|
|
271
|
-
return this.#decideAndEnact(nodeId);
|
|
287
|
+
async #applyPolicy(nodeId, exitStatus) {
|
|
288
|
+
return this.#decideAndEnact(nodeId, exitStatus);
|
|
272
289
|
}
|
|
273
|
-
async #decideAndEnact(nodeId) {
|
|
290
|
+
async #decideAndEnact(nodeId, exitStatus) {
|
|
274
291
|
const meta = getNode(nodeId);
|
|
275
292
|
if (meta !== null && isProfilePaused(meta.profile_id)) {
|
|
276
293
|
clearRespawnPolicy(nodeId);
|
|
@@ -319,9 +336,10 @@ export class DaemonFleet {
|
|
|
319
336
|
await this.#terminalize(nodeId, meta, {
|
|
320
337
|
event: 'broker.boot.failed',
|
|
321
338
|
reason: 'boot_failed',
|
|
322
|
-
detail: { schema: 'crtr.node-outcome-detail/v1', message: bootFailureBody(meta) },
|
|
339
|
+
detail: { schema: 'crtr.node-outcome-detail/v1', message: bootFailureBody(meta, exitStatus) },
|
|
323
340
|
wake: `Child crashed — ${fullName(meta)} (${nodeId}) exited before its session ever came up and is now dead. ` +
|
|
324
341
|
`It stays dead until you revive it — \`crtr node lifecycle revive ${nodeId}\`.`,
|
|
342
|
+
surface: () => surfaceBootFailure(meta, exitStatus),
|
|
325
343
|
});
|
|
326
344
|
break;
|
|
327
345
|
case 'respawn-fresh':
|
package/dist/daemon/manage.d.ts
CHANGED
|
@@ -50,12 +50,14 @@ export interface SpawnDaemonResult {
|
|
|
50
50
|
running?: true;
|
|
51
51
|
/** Present with `running:true` when the Unix API socket is not serving. */
|
|
52
52
|
serving?: false;
|
|
53
|
-
/**
|
|
54
|
-
|
|
53
|
+
/** The final `/healthz` probe failure when a live owner misses readiness. */
|
|
54
|
+
probe_error?: string;
|
|
55
55
|
}
|
|
56
|
-
/**
|
|
57
|
-
*
|
|
58
|
-
|
|
56
|
+
/** Make one `/healthz` observation without client-side recovery. The caller
|
|
57
|
+
* owns the availability window, so this probe cannot extend it. */
|
|
58
|
+
export declare function probeDaemonServing(timeoutMs?: number): Promise<void>;
|
|
59
|
+
/** True only when one mandatory Unix API socket probe answers `/healthz`.
|
|
60
|
+
* A pidfile identifies an owner; it is never readiness. */
|
|
59
61
|
export declare function isDaemonServing(): Promise<boolean>;
|
|
60
62
|
export interface DaemonWaitDeps {
|
|
61
63
|
readPidfile?: () => number | null;
|
|
@@ -69,6 +71,7 @@ export interface DaemonStartupDeps extends DaemonWaitDeps {
|
|
|
69
71
|
signal: NodeJS.Signals | null;
|
|
70
72
|
} | null;
|
|
71
73
|
isDaemonServing?: () => Promise<boolean>;
|
|
74
|
+
probeDaemonServing?: (timeoutMs: number) => Promise<void>;
|
|
72
75
|
}
|
|
73
76
|
/** Thrown when the spawned daemon exited on `DAEMON_EXIT_STARTUP_BLOCKED` — a
|
|
74
77
|
* standing condition (a blocked on-disk migration) that the very next spawn
|
package/dist/daemon/manage.js
CHANGED
|
@@ -19,7 +19,7 @@ import { clearDaemonStartupBlocked, daemonStartupBlocked } from './startup-block
|
|
|
19
19
|
import { exclusiveLockOwnerPid } from '../core/exclusive-lock.js';
|
|
20
20
|
import { onDiskMigrationLockPath } from '../core/canvas/paths.js';
|
|
21
21
|
import { envNoDaemonAutostart } from '../shared/env.js';
|
|
22
|
-
import { CrtrClient } from '../api/index.js';
|
|
22
|
+
import { CrtrClient, waitForDaemonAvailability } from '../api/index.js';
|
|
23
23
|
// Daemon env sanitization
|
|
24
24
|
/** Env keys that must NEVER reach the daemon process. Restarting crtrd is
|
|
25
25
|
* overwhelmingly done from inside an agent node's own bash tool (`crtr sys
|
|
@@ -208,16 +208,20 @@ const DAEMON_VERIFY_POLL_MS = 10;
|
|
|
208
208
|
/** A status/start readiness probe must not hang behind a daemon that accepts a
|
|
209
209
|
* Unix connection but never answers it. */
|
|
210
210
|
const DAEMON_HEALTH_PROBE_TIMEOUT_MS = 1_000;
|
|
211
|
-
/**
|
|
212
|
-
*
|
|
213
|
-
|
|
211
|
+
/** Make one `/healthz` observation without client-side recovery. The caller
|
|
212
|
+
* owns the availability window, so this probe cannot extend it. */
|
|
213
|
+
export async function probeDaemonServing(timeoutMs = DAEMON_HEALTH_PROBE_TIMEOUT_MS) {
|
|
214
|
+
const probeTimeoutMs = Math.min(timeoutMs, DAEMON_HEALTH_PROBE_TIMEOUT_MS);
|
|
215
|
+
const health = await CrtrClient.forLocalSocket({ autostart: false, timeoutMs: probeTimeoutMs }).probeHealthz(probeTimeoutMs);
|
|
216
|
+
if (!health.ok)
|
|
217
|
+
throw new Error('crtrd health check reported not ready');
|
|
218
|
+
}
|
|
219
|
+
/** True only when one mandatory Unix API socket probe answers `/healthz`.
|
|
220
|
+
* A pidfile identifies an owner; it is never readiness. */
|
|
214
221
|
export async function isDaemonServing() {
|
|
215
222
|
try {
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
timeoutMs: DAEMON_HEALTH_PROBE_TIMEOUT_MS,
|
|
219
|
-
}).healthz();
|
|
220
|
-
return health.ok;
|
|
223
|
+
await probeDaemonServing();
|
|
224
|
+
return true;
|
|
221
225
|
}
|
|
222
226
|
catch {
|
|
223
227
|
return false;
|
|
@@ -255,32 +259,47 @@ export async function verifyDaemonStartup(pid, timeoutMs = DAEMON_STARTUP_WINDOW
|
|
|
255
259
|
const read = deps.readPidfile ?? readPidfile;
|
|
256
260
|
const alive = deps.isPidAlive ?? isPidAlive;
|
|
257
261
|
const exited = deps.childExited;
|
|
258
|
-
const serving = deps.isDaemonServing
|
|
262
|
+
const serving = deps.isDaemonServing;
|
|
263
|
+
const probeServing = deps.probeDaemonServing ?? (async (remainingMs) => {
|
|
264
|
+
if (serving !== undefined) {
|
|
265
|
+
if (await serving())
|
|
266
|
+
return;
|
|
267
|
+
throw new Error('crtrd health check reported not ready');
|
|
268
|
+
}
|
|
269
|
+
await probeDaemonServing(remainingMs);
|
|
270
|
+
});
|
|
259
271
|
const sleep = deps.sleepMs ?? sleepMs;
|
|
260
272
|
const now = deps.now ?? Date.now;
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
if (
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
273
|
+
let owner = null;
|
|
274
|
+
let terminalError = null;
|
|
275
|
+
await waitForDaemonAvailability({
|
|
276
|
+
windowMs: timeoutMs,
|
|
277
|
+
pollIntervalMs: DAEMON_VERIFY_POLL_MS,
|
|
278
|
+
now,
|
|
279
|
+
sleep,
|
|
280
|
+
retry: (error) => error !== terminalError,
|
|
281
|
+
probe: async (remainingMs) => {
|
|
282
|
+
const candidate = read();
|
|
283
|
+
if (candidate === null || !alive(candidate)) {
|
|
284
|
+
const exitState = exited?.();
|
|
285
|
+
if (exitState !== null && exitState !== undefined) {
|
|
286
|
+
if (exitState.signal !== null)
|
|
287
|
+
terminalError = new Error(`daemon ${pid} exited before becoming ready (by ${exitState.signal})`);
|
|
288
|
+
else if (exitState.code === DAEMON_EXIT_STARTUP_BLOCKED)
|
|
289
|
+
terminalError = new DaemonStartupBlockedError(pid);
|
|
290
|
+
else if (exitState.code !== 0)
|
|
291
|
+
terminalError = new Error(`daemon ${pid} exited before becoming ready (with exit code ${exitState.code ?? '?'})`);
|
|
292
|
+
// A clean exit can have lost the ownership claim, so keep observing.
|
|
293
|
+
if (terminalError !== null)
|
|
294
|
+
throw terminalError;
|
|
295
|
+
}
|
|
296
|
+
throw new Error(`daemon ${pid} has no live pidfile owner`);
|
|
276
297
|
}
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
}
|
|
283
|
-
throw new Error(`daemon ${pid} did not become ready within ${timeoutMs}ms`);
|
|
298
|
+
await probeServing(remainingMs);
|
|
299
|
+
owner = candidate;
|
|
300
|
+
},
|
|
301
|
+
});
|
|
302
|
+
return owner === pid ? null : owner;
|
|
284
303
|
}
|
|
285
304
|
/** Wait until a signaled daemon has exited and relinquished its pidfile. This
|
|
286
305
|
* makes a following start safe to claim the singleton rather than racing its
|
|
@@ -355,15 +374,19 @@ export async function stopDaemonProcess(pid, timeoutMs = DAEMON_SHUTDOWN_WINDOW_
|
|
|
355
374
|
export async function spawnDaemon() {
|
|
356
375
|
const recordedPid = readPidfile();
|
|
357
376
|
if (recordedPid !== null && isPidAlive(recordedPid)) {
|
|
358
|
-
|
|
377
|
+
try {
|
|
378
|
+
await probeDaemonServing();
|
|
359
379
|
return { started: false, existing_pid: recordedPid };
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
380
|
+
}
|
|
381
|
+
catch (error) {
|
|
382
|
+
return {
|
|
383
|
+
started: false,
|
|
384
|
+
existing_pid: recordedPid,
|
|
385
|
+
running: true,
|
|
386
|
+
serving: false,
|
|
387
|
+
probe_error: error.message,
|
|
388
|
+
};
|
|
389
|
+
}
|
|
367
390
|
}
|
|
368
391
|
// Ensure the canvas home directory exists so the daemon can write its pidfile.
|
|
369
392
|
mkdirSync(crtrHome(), { recursive: true });
|
|
@@ -30,6 +30,12 @@ const TERMINAL_STATUSES = ['done', 'dead', 'canceled'];
|
|
|
30
30
|
function isTerminal(row) {
|
|
31
31
|
return TERMINAL_STATUSES.includes(row.status);
|
|
32
32
|
}
|
|
33
|
+
/** A closed record already has an explicit close disposition. An open record
|
|
34
|
+
* reaches the sweep only when its owner published a final result; a crash or
|
|
35
|
+
* cancellation leaves its managed checkout available for explicit recovery. */
|
|
36
|
+
function hasSweepDisposition(row, wt) {
|
|
37
|
+
return wt?.state === 'closed' || (wt?.state === 'open' && row.status === 'done' && row.final_report !== null);
|
|
38
|
+
}
|
|
33
39
|
/** Is this node's engine definitively gone?
|
|
34
40
|
*
|
|
35
41
|
* Terminal status alone is not enough: `push final` marks a node `done` while
|
|
@@ -49,6 +55,8 @@ function engineIsGone(row, fleet, snapshot) {
|
|
|
49
55
|
return false;
|
|
50
56
|
if (!isTerminal(row))
|
|
51
57
|
return false;
|
|
58
|
+
if (!hasSweepDisposition(row, getNode(row.node_id)?.managed_worktree))
|
|
59
|
+
return false;
|
|
52
60
|
return recordedPidLiveness(row.pi_pid, row.pi_pid_identity, snapshot) === 'dead';
|
|
53
61
|
}
|
|
54
62
|
/** The eligibility recheck the reconciliation runs immediately before EVERY Git
|
|
@@ -70,6 +78,8 @@ function eligibilityGuard(row, fleet) {
|
|
|
70
78
|
const current = getRow(nodeId);
|
|
71
79
|
if (current === null || !isTerminal(current) || current.frozen_at !== null)
|
|
72
80
|
return false;
|
|
81
|
+
if (!hasSweepDisposition(current, getNode(nodeId)?.managed_worktree))
|
|
82
|
+
return false;
|
|
73
83
|
return current.pi_pid === pid && current.pi_pid_identity === identity;
|
|
74
84
|
},
|
|
75
85
|
};
|
|
@@ -120,6 +130,8 @@ export class ManagedWorktreeSweepReconciler {
|
|
|
120
130
|
}
|
|
121
131
|
if (!isReconcilable(wt) || wt == null)
|
|
122
132
|
continue;
|
|
133
|
+
if (!hasSweepDisposition(row, wt))
|
|
134
|
+
continue;
|
|
123
135
|
if (!isDueForSweep(wt, now))
|
|
124
136
|
continue;
|
|
125
137
|
due.push(row.node_id);
|
|
@@ -7,12 +7,6 @@ export interface NodeLifecycleContext {
|
|
|
7
7
|
rows: readonly NodeRow[];
|
|
8
8
|
fleet: FleetRegistry;
|
|
9
9
|
}
|
|
10
|
-
/** Whether this row should have a live engine right now. An active row always
|
|
11
|
-
* should — that status IS the claim that it is executing. An idle row was
|
|
12
|
-
* deliberately released and holds no engine on purpose: only mail it has not
|
|
13
|
-
* seen brings one back, because a live obligation keeps a node WAKEABLE rather
|
|
14
|
-
* than awake, and relaunching on one would re-release it every tick. */
|
|
15
|
-
export declare function wantsExecution(row: NodeRow): boolean;
|
|
16
10
|
export declare class NodeLifecycleReconciler {
|
|
17
11
|
private readonly freezeLane;
|
|
18
12
|
run(now: number, ctx: NodeLifecycleContext): Promise<FrozenCounts>;
|
|
@@ -12,7 +12,6 @@ import { isSafeNodeId } from '../../../core/canvas/paths.js';
|
|
|
12
12
|
import { recordedPidLiveness } from '../../../core/canvas/pid.js';
|
|
13
13
|
import { emitEvent } from '../../../core/events/emit.js';
|
|
14
14
|
import { operationIdContext } from '../../../core/events/operation-id.js';
|
|
15
|
-
import { hasUnseenWakeMail } from '../../../core/feed/inbox.js';
|
|
16
15
|
import { hasCleanAbort, isBusy } from '../../../core/runtime/busy.js';
|
|
17
16
|
import { isProfilePaused } from '../../../core/profiles/manifest.js';
|
|
18
17
|
import { classifyDeadNode } from '../../fleet.js';
|
|
@@ -20,21 +19,10 @@ import { applyParkEvent, hasPendingWake, parkEvent } from '../live-obligation.js
|
|
|
20
19
|
import { FreezeLaneReconciler } from './freeze-lane.js';
|
|
21
20
|
import { isInBackoff } from './respawn-policy.js';
|
|
22
21
|
import { handleTerminatingRow } from './terminating.js';
|
|
22
|
+
import { wantsExecution } from './wants-execution.js';
|
|
23
23
|
function isTerminal(row) {
|
|
24
24
|
return row.status === 'done' || row.status === 'canceled' || row.status === 'dead';
|
|
25
25
|
}
|
|
26
|
-
/** Whether this row should have a live engine right now. An active row always
|
|
27
|
-
* should — that status IS the claim that it is executing. An idle row was
|
|
28
|
-
* deliberately released and holds no engine on purpose: only mail it has not
|
|
29
|
-
* seen brings one back, because a live obligation keeps a node WAKEABLE rather
|
|
30
|
-
* than awake, and relaunching on one would re-release it every tick. */
|
|
31
|
-
export function wantsExecution(row) {
|
|
32
|
-
if (row.status === 'active')
|
|
33
|
-
return true;
|
|
34
|
-
if (row.status !== 'idle')
|
|
35
|
-
return false;
|
|
36
|
-
return hasUnseenWakeMail(row.node_id);
|
|
37
|
-
}
|
|
38
26
|
export class NodeLifecycleReconciler {
|
|
39
27
|
freezeLane = new FreezeLaneReconciler();
|
|
40
28
|
async run(now, ctx) {
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { NodeRow } from '../../../core/canvas/index.js';
|
|
2
|
+
/** Whether the lifecycle tick must ensure this row has an engine. The durable
|
|
3
|
+
* message handler calls this after appending its entry: that append makes the
|
|
4
|
+
* idle branch's unseen-wake-mail check true for a wake-capable entry. */
|
|
5
|
+
export declare function wantsExecution(row: Pick<NodeRow, 'node_id' | 'status'>): boolean;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { hasUnseenWakeMail } from '../../../core/feed/inbox.js';
|
|
2
|
+
/** Whether the lifecycle tick must ensure this row has an engine. The durable
|
|
3
|
+
* message handler calls this after appending its entry: that append makes the
|
|
4
|
+
* idle branch's unseen-wake-mail check true for a wake-capable entry. */
|
|
5
|
+
export function wantsExecution(row) {
|
|
6
|
+
if (row.status === 'active')
|
|
7
|
+
return true;
|
|
8
|
+
if (row.status !== 'idle')
|
|
9
|
+
return false;
|
|
10
|
+
return hasUnseenWakeMail(row.node_id);
|
|
11
|
+
}
|
|
@@ -5,12 +5,16 @@
|
|
|
5
5
|
import { test, before, after, beforeEach, afterEach } from 'node:test';
|
|
6
6
|
import assert from 'node:assert/strict';
|
|
7
7
|
import { existsSync, mkdtempSync, mkdirSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
|
|
8
|
+
import { createServer } from 'node:http';
|
|
8
9
|
import { tmpdir } from 'node:os';
|
|
9
10
|
import { join } from 'node:path';
|
|
10
11
|
import { registerCanvasContextIntro, CONTEXT_INTRO_CUSTOM_TYPE, } from '../canvas-context-intro.js';
|
|
11
12
|
import { registerCanvasDocSubstrate } from '../canvas-doc-substrate.js';
|
|
12
13
|
import { buildContextBearings as buildContextIntro } from '../../core/runtime/bearings.js';
|
|
13
14
|
import { installPersonaTransitionGate } from '../../core/runtime/broker-persona-guidance.js';
|
|
15
|
+
import { FaultRetry } from '../../core/runtime/broker/fault-retry.js';
|
|
16
|
+
import { readFault } from '../../core/runtime/fault.js';
|
|
17
|
+
import { ApiError } from '../../api/errors.js';
|
|
14
18
|
import { createNode, deleteNode, getNode, updateNode } from '../../core/canvas/canvas.js';
|
|
15
19
|
import { apiSocketPath } from '../../core/canvas/paths.js';
|
|
16
20
|
import { createApiServer } from '../../daemon/api/server.js';
|
|
@@ -32,6 +36,23 @@ async function waitForSocket(path) {
|
|
|
32
36
|
}
|
|
33
37
|
throw new Error(`api socket never bound at ${path}`);
|
|
34
38
|
}
|
|
39
|
+
async function listenUnavailableApi(path) {
|
|
40
|
+
const unavailable = createServer((_request, response) => {
|
|
41
|
+
response.writeHead(503, { 'content-type': 'application/json' });
|
|
42
|
+
response.end(JSON.stringify({ error: { code: 'daemon_unavailable', message: 'local daemon connection refused' } }));
|
|
43
|
+
});
|
|
44
|
+
await new Promise((resolve, reject) => {
|
|
45
|
+
unavailable.once('error', reject);
|
|
46
|
+
unavailable.listen(path, () => {
|
|
47
|
+
unavailable.off('error', reject);
|
|
48
|
+
resolve();
|
|
49
|
+
});
|
|
50
|
+
});
|
|
51
|
+
return unavailable;
|
|
52
|
+
}
|
|
53
|
+
function closeHttpServer(server) {
|
|
54
|
+
return new Promise((resolve, reject) => server.close((error) => error === undefined ? resolve() : reject(error)));
|
|
55
|
+
}
|
|
35
56
|
function node(id, over = {}) {
|
|
36
57
|
return {
|
|
37
58
|
node_id: id,
|
|
@@ -322,6 +343,68 @@ test('child /new commits its baseline before ack and the provider gate delivers
|
|
|
322
343
|
process.env['CRTR_PROFILE_ID'] = previousProfile;
|
|
323
344
|
}
|
|
324
345
|
});
|
|
346
|
+
test('a typed local API error remains a retryable broker-to-daemon fault after settlement', async () => {
|
|
347
|
+
createNode(node('persona-transport', {
|
|
348
|
+
persona_ack: { kind: 'review', mode: 'base', lifecycle: 'terminal' },
|
|
349
|
+
}));
|
|
350
|
+
await server.close();
|
|
351
|
+
let unavailable;
|
|
352
|
+
try {
|
|
353
|
+
unavailable = await listenUnavailableApi(apiSocketPath());
|
|
354
|
+
const messages = [];
|
|
355
|
+
const agent = {
|
|
356
|
+
state: { messages },
|
|
357
|
+
transformContext: async (context) => context,
|
|
358
|
+
};
|
|
359
|
+
installPersonaTransitionGate('persona-transport', {
|
|
360
|
+
agent,
|
|
361
|
+
sessionManager: {
|
|
362
|
+
appendCustomMessageEntry: () => undefined,
|
|
363
|
+
},
|
|
364
|
+
});
|
|
365
|
+
await assert.rejects(() => agent.transformContext([{ role: 'user', content: 'continue' }]), (error) => error instanceof ApiError && error.status === 503 && error.code === 'daemon_unavailable');
|
|
366
|
+
assert.deepEqual(readFault('persona-transport'), {
|
|
367
|
+
link: 'broker↔crtrd',
|
|
368
|
+
op: 'persona transition',
|
|
369
|
+
kind: 'connection',
|
|
370
|
+
retry: { disposition: 'manual' },
|
|
371
|
+
message: 'local daemon connection refused',
|
|
372
|
+
since: readFault('persona-transport')?.since,
|
|
373
|
+
operation_id: readFault('persona-transport')?.operation_id,
|
|
374
|
+
}, 'the typed daemon transport error is recorded before Pi turns it into a generic agent error');
|
|
375
|
+
const session = {
|
|
376
|
+
sessionManager: { getLeafId: () => 'persona-transport-leaf' },
|
|
377
|
+
model: undefined,
|
|
378
|
+
};
|
|
379
|
+
const generation = {
|
|
380
|
+
id: 1,
|
|
381
|
+
session,
|
|
382
|
+
fallback: { blocked: false },
|
|
383
|
+
stagedProviderAgentEnd: {
|
|
384
|
+
messages: [{ role: 'assistant', stopReason: 'error', errorMessage: 'local daemon connection refused' }],
|
|
385
|
+
},
|
|
386
|
+
stagedWatchdogAbort: false,
|
|
387
|
+
stagedRefreshAbort: false,
|
|
388
|
+
stagedOverflowFailure: null,
|
|
389
|
+
executingTools: new Set(),
|
|
390
|
+
};
|
|
391
|
+
new FaultRetry({
|
|
392
|
+
nodeId: 'persona-transport',
|
|
393
|
+
currentSession: () => session,
|
|
394
|
+
installedGeneration: () => generation,
|
|
395
|
+
}).publishSettledProviderFault(generation);
|
|
396
|
+
const fault = readFault('persona-transport');
|
|
397
|
+
assert.equal(fault?.link, 'broker↔crtrd');
|
|
398
|
+
assert.equal(fault?.kind, 'connection');
|
|
399
|
+
assert.equal(fault?.retry.disposition, 'manual');
|
|
400
|
+
}
|
|
401
|
+
finally {
|
|
402
|
+
if (unavailable !== undefined)
|
|
403
|
+
await closeHttpServer(unavailable);
|
|
404
|
+
server = createApiServer();
|
|
405
|
+
await waitForSocket(apiSocketPath());
|
|
406
|
+
}
|
|
407
|
+
});
|
|
325
408
|
test('resume idempotency uses the exact details stamp independent of block text', async () => {
|
|
326
409
|
createNode(node('stampD'));
|
|
327
410
|
// The exact node-id stamp is authoritative even when prose carries no id.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|