@north-light/crouter 0.3.251 → 0.3.253
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/dto/config.d.ts +2 -0
- package/dist/builtin-memory/02-turn-lifecycle/00-ending-a-turn.md +1 -5
- package/dist/builtin-memory/04-base-worker-exploring.md +13 -0
- package/dist/builtin-memory/04-base-worker.md +1 -4
- package/dist/builtin-memory/05-kinds/explore/00-base.md +2 -2
- package/dist/builtin-memory/05-kinds/explore/01-orchestrator.md +4 -2
- package/dist/builtin-memory/05-kinds/review/security-findings.md +1 -1
- package/dist/builtin-memory/explore/exploration-doc.md +27 -0
- package/dist/clients/attach/__tests__/completion-frecency.test.d.ts +1 -0
- package/dist/clients/attach/__tests__/completion-frecency.test.js +35 -0
- package/dist/clients/attach/__tests__/ref-autocomplete.test.js +3 -1
- package/dist/clients/attach/__tests__/titled-editor-preview.test.js +2 -1
- package/dist/clients/attach/input/completion-frecency.d.ts +45 -0
- package/dist/clients/attach/input/completion-frecency.js +141 -0
- package/dist/clients/attach/input/controller.d.ts +17 -0
- package/dist/clients/attach/input/controller.js +40 -0
- package/dist/clients/attach/input/ref-autocomplete.d.ts +3 -1
- package/dist/clients/attach/input/ref-autocomplete.js +49 -8
- package/dist/clients/attach/input/titled-editor.d.ts +3 -0
- package/dist/clients/attach/input/titled-editor.js +5 -0
- package/dist/clients/attach/session/editor-inventory.d.ts +3 -0
- package/dist/clients/attach/session/editor-inventory.js +1 -1
- package/dist/clients/attach/session/input-wiring.d.ts +3 -0
- package/dist/clients/attach/session/input-wiring.js +2 -0
- package/dist/clients/attach/viewer.js +578 -578
- package/dist/commands/node/lifecycle.js +21 -6
- package/dist/core/__tests__/canvas-inbox-watcher-hold.test.js +4 -2
- package/dist/core/__tests__/canvas-inbox-watcher-naming.test.js +4 -1
- package/dist/core/__tests__/canvas-inbox-watcher.test.js +4 -1
- package/dist/core/__tests__/fixtures/fake-engine.d.ts +3 -1
- package/dist/core/__tests__/fixtures/fake-engine.js +7 -1
- package/dist/core/__tests__/integration/deferred-no-wake.test.js +4 -1
- package/dist/core/__tests__/migration.test.js +57 -11
- package/dist/core/__tests__/seam/broker-provider-retry.test.js +17 -2
- package/dist/core/__tests__/seam/dormancy-release.test.js +1 -0
- package/dist/core/__tests__/watchdog-abort-arms-retry.test.js +9 -2
- package/dist/core/canvas/migrations.js +63 -24
- package/dist/core/runtime/broker/engine-drive.js +7 -1
- package/dist/core/runtime/broker/fault-retry.js +31 -13
- package/dist/core/runtime/broker/held-deferred-inbox.d.ts +4 -0
- package/dist/core/runtime/broker/held-deferred-inbox.js +7 -2
- package/dist/core/runtime/broker.js +19 -0
- package/dist/core/runtime/close.js +2 -2
- package/dist/core/runtime/fault.js +8 -5
- package/dist/core/runtime/recycle.js +63 -41
- package/dist/core/runtime/reset.d.ts +1 -1
- package/dist/core/runtime/reset.js +25 -2
- package/dist/core/runtime/revive.js +6 -1
- package/dist/daemon/__tests__/helpers/source-daemon.js +1 -0
- package/dist/daemon/api/handlers/messages.js +1 -1
- package/dist/daemon/api/handlers/nodes.js +124 -4
- package/dist/daemon/api/handlers/reports.js +1 -1
- package/dist/daemon/api/map.d.ts +1 -1
- package/dist/daemon/api/map.js +3 -2
- package/dist/daemon/cron/sinks.js +1 -1
- package/dist/daemon/messaging/node-message.js +1 -1
- package/dist/pi-extensions/canvas-inbox-watcher.js +5 -2
- package/package.json +1 -1
- package/runtime.lock.json +2 -2
|
@@ -58,19 +58,26 @@ export class FaultRetry {
|
|
|
58
58
|
generation.stagedRefreshAbort = false;
|
|
59
59
|
generation.stagedOverflowFailure = null;
|
|
60
60
|
const settledSession = generation.session;
|
|
61
|
-
//
|
|
62
|
-
//
|
|
63
|
-
//
|
|
64
|
-
// each retry rewinds to the same fork point instead of stacking recovery
|
|
65
|
-
// prompts, and so backoff/exhaustion actually accumulate.
|
|
61
|
+
// turn_end clears the ordinary marker before this settlement. A failed
|
|
62
|
+
// admitted retry therefore carries from its durable episode, while an
|
|
63
|
+
// initial provider failure still reads from the ordinary marker.
|
|
66
64
|
const prior = readFault(this.deps.nodeId);
|
|
67
|
-
const
|
|
68
|
-
|
|
69
|
-
|
|
65
|
+
const durableEpisode = readProviderRetryEpisode(this.deps.nodeId);
|
|
66
|
+
const episodeFault = this.isActiveAutoFault(prior) && prior.link === 'pi→provider'
|
|
67
|
+
? prior
|
|
68
|
+
: durableEpisode?.state === 'admitted' &&
|
|
69
|
+
durableEpisode.sessionFile === this.sessionFile(settledSession) &&
|
|
70
|
+
this.isActiveAutoFault(durableEpisode.fault)
|
|
71
|
+
? durableEpisode.fault
|
|
72
|
+
: null;
|
|
73
|
+
const episode = episodeFault === null
|
|
74
|
+
? null
|
|
75
|
+
: { since: episodeFault.since, anchorEntryId: episodeFault.anchorEntryId, attempt: episodeFault.retry.attempt };
|
|
70
76
|
const messages = Array.isArray(agentEnd?.messages) ? agentEnd.messages : [];
|
|
71
77
|
const last = [...messages].reverse().find((message) => typeof message === 'object' && message !== null && message.role === 'assistant');
|
|
72
78
|
if (overflowFailure !== null) {
|
|
73
79
|
clearFault(this.deps.nodeId, { link: 'pi→provider' });
|
|
80
|
+
clearProviderRetryEpisode(this.deps.nodeId);
|
|
74
81
|
recordFault(this.deps.nodeId, {
|
|
75
82
|
link: 'pi→provider', op: 'context overflow recovery', kind: 'context-overflow', retry: { disposition: 'fatal' },
|
|
76
83
|
message: overflowFailure.errorMessage, anchorEntryId: settledSession.sessionManager.getLeafId?.() ?? undefined,
|
|
@@ -87,21 +94,30 @@ export class FaultRetry {
|
|
|
87
94
|
});
|
|
88
95
|
return;
|
|
89
96
|
}
|
|
90
|
-
if (last?.stopReason !== 'error')
|
|
97
|
+
if (last?.stopReason !== 'error') {
|
|
98
|
+
if (last !== undefined)
|
|
99
|
+
clearProviderRetryEpisode(this.deps.nodeId);
|
|
91
100
|
return;
|
|
92
|
-
|
|
101
|
+
}
|
|
102
|
+
if (refreshAbort) {
|
|
103
|
+
clearProviderRetryEpisode(this.deps.nodeId);
|
|
93
104
|
return;
|
|
105
|
+
}
|
|
94
106
|
const raw = typeof last.errorMessage === 'string' ? last.errorMessage : 'engine error (no errorMessage recorded)';
|
|
95
107
|
// An abort landing between requests reaches pi's catch as a bare AbortError
|
|
96
108
|
// and settles as an error rather than `aborted`. Something asked that turn
|
|
97
109
|
// to stop, so there is no provider fault to record.
|
|
98
|
-
if (endedByAbort({ stopReason: 'error', errorMessage: raw }))
|
|
110
|
+
if (endedByAbort({ stopReason: 'error', errorMessage: raw })) {
|
|
111
|
+
clearProviderRetryEpisode(this.deps.nodeId);
|
|
99
112
|
return;
|
|
113
|
+
}
|
|
100
114
|
const uncompactedOverflow = isContextOverflow(last, settledSession.model?.contextWindow);
|
|
101
115
|
const classified = classify('pi→provider', last);
|
|
102
116
|
const coolingDeadline = extractCoolingDeadline(last);
|
|
103
|
-
if (uncompactedOverflow)
|
|
117
|
+
if (uncompactedOverflow) {
|
|
104
118
|
clearFault(this.deps.nodeId, { link: 'pi→provider' });
|
|
119
|
+
clearProviderRetryEpisode(this.deps.nodeId);
|
|
120
|
+
}
|
|
105
121
|
const faultInput = {
|
|
106
122
|
link: 'pi→provider', op: 'generation turn', kind: uncompactedOverflow ? 'context-overflow' : classified.kind,
|
|
107
123
|
retry: uncompactedOverflow
|
|
@@ -121,8 +137,10 @@ export class FaultRetry {
|
|
|
121
137
|
};
|
|
122
138
|
if (faultInput.retry.disposition === 'auto')
|
|
123
139
|
this.recordPendingProviderFault(settledSession, faultInput);
|
|
124
|
-
else
|
|
140
|
+
else {
|
|
141
|
+
clearProviderRetryEpisode(this.deps.nodeId);
|
|
125
142
|
recordFault(this.deps.nodeId, faultInput);
|
|
143
|
+
}
|
|
126
144
|
}
|
|
127
145
|
/** Startup re-drive reads only a durable pending episode. Revive deliberately
|
|
128
146
|
* clears the ordinary fault marker before this boundary, so the episode is
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export declare const HELD_DEFERRED_INBOX_PROMPT_JOIN: unique symbol;
|
|
2
|
+
export declare const BROKER_IDLE_TURN_START: unique symbol;
|
|
2
3
|
export type HeldDeferredInboxPromptJoin = (session: {
|
|
3
4
|
sendCustomMessage: (message: {
|
|
4
5
|
customType: string;
|
|
@@ -7,3 +8,6 @@ export type HeldDeferredInboxPromptJoin = (session: {
|
|
|
7
8
|
}) => Promise<void>;
|
|
8
9
|
}) => Promise<void>;
|
|
9
10
|
export declare function heldDeferredInboxPromptJoin(): HeldDeferredInboxPromptJoin | undefined;
|
|
11
|
+
/** Starts an inbox-delivered idle turn through the broker's admission gate. */
|
|
12
|
+
export type BrokerIdleTurnStart = (text: string) => void;
|
|
13
|
+
export declare function brokerIdleTurnStart(): BrokerIdleTurnStart | undefined;
|
|
@@ -1,8 +1,13 @@
|
|
|
1
1
|
// The inbox watcher is a Pi extension loaded through Jiti, while the broker is
|
|
2
|
-
// native ESM. They therefore cannot share a module singleton.
|
|
3
|
-
//
|
|
2
|
+
// native ESM. They therefore cannot share a module singleton. Process-global
|
|
3
|
+
// symbols carry their narrow bridge functions across that loader boundary.
|
|
4
4
|
export const HELD_DEFERRED_INBOX_PROMPT_JOIN = Symbol.for('@crouton-kit/crtr:held-deferred-inbox-prompt-join');
|
|
5
|
+
export const BROKER_IDLE_TURN_START = Symbol.for('@crouton-kit/crtr:broker-idle-turn-start');
|
|
5
6
|
export function heldDeferredInboxPromptJoin() {
|
|
6
7
|
const join = globalThis[HELD_DEFERRED_INBOX_PROMPT_JOIN];
|
|
7
8
|
return typeof join === 'function' ? join : undefined;
|
|
8
9
|
}
|
|
10
|
+
export function brokerIdleTurnStart() {
|
|
11
|
+
const start = globalThis[BROKER_IDLE_TURN_START];
|
|
12
|
+
return typeof start === 'function' ? start : undefined;
|
|
13
|
+
}
|
|
@@ -47,6 +47,7 @@ import { ToolGroupTracker } from './broker/tool-groups.js';
|
|
|
47
47
|
import { onNodeNamed } from './broker/node-named.js';
|
|
48
48
|
import { FaultRetry } from './broker/fault-retry.js';
|
|
49
49
|
import { createTurnAdmissionGate } from './broker/turn-admission.js';
|
|
50
|
+
import { BROKER_IDLE_TURN_START } from './broker/held-deferred-inbox.js';
|
|
50
51
|
import { EventProjection } from './broker/event-projection.js';
|
|
51
52
|
import { createFrameDispatchContext, handleFrame, } from './broker/frame-dispatch.js';
|
|
52
53
|
import { hydratePersistedAdvertisedCommandMessages, installAdvertisedCommandInvocationContract, } from './advertised-command-invocation.js';
|
|
@@ -327,6 +328,24 @@ export async function runBroker(nodeId, startupAt = process.hrtime.bigint(), sta
|
|
|
327
328
|
registry.broadcast({ type: 'model_changed', model: isUnknownModel(liveSession.model) ? undefined : liveSession.model, spec });
|
|
328
329
|
};
|
|
329
330
|
const turnAdmission = createTurnAdmissionGate();
|
|
331
|
+
// The inbox watcher is Jiti-loaded and cannot import this native module's
|
|
332
|
+
// gate. Its idle delivery therefore calls this narrow process-global bridge:
|
|
333
|
+
// acquire admission first, then re-read the live session before prompt() so
|
|
334
|
+
// a viewer turn that won the race is joined rather than double-started.
|
|
335
|
+
const startIdleInboxTurn = (text) => {
|
|
336
|
+
void turnAdmission.admit((release) => {
|
|
337
|
+
const liveSession = rebind.session();
|
|
338
|
+
return liveSession
|
|
339
|
+
.prompt(text, {
|
|
340
|
+
preflightResult: release,
|
|
341
|
+
...(liveSession.isStreaming ? { streamingBehavior: 'steer' } : {}),
|
|
342
|
+
})
|
|
343
|
+
.catch((error) => {
|
|
344
|
+
emitEvent({ level: 'error', event: 'broker.inbox.idle_prompt_failed', error });
|
|
345
|
+
});
|
|
346
|
+
});
|
|
347
|
+
};
|
|
348
|
+
globalThis[BROKER_IDLE_TURN_START] = startIdleInboxTurn;
|
|
330
349
|
faultRetry = new FaultRetry({
|
|
331
350
|
nodeId,
|
|
332
351
|
cfg,
|
|
@@ -49,12 +49,12 @@ export function fanDoctrineWake(fromId, subscribers, label, data, tier = 'normal
|
|
|
49
49
|
try {
|
|
50
50
|
const notice = { from: fromId, tier, kind: 'message', label, data };
|
|
51
51
|
if (sub.active) {
|
|
52
|
-
appendInbox(sub.node_id, notice);
|
|
52
|
+
const entry = appendInbox(sub.node_id, notice);
|
|
53
53
|
// A wake-capable notice consumes the subscriber's cancel-on-wake
|
|
54
54
|
// deadline: the thing it was waiting for arrived. `deferred` is the
|
|
55
55
|
// exception — it never wakes a node, riding the next natural cycle
|
|
56
56
|
// instead, so the deadline it was racing must survive.
|
|
57
|
-
if (tier !== 'deferred')
|
|
57
|
+
if (entry.tier !== 'deferred')
|
|
58
58
|
cancelCronsOnWake(sub.node_id);
|
|
59
59
|
}
|
|
60
60
|
else
|
|
@@ -229,11 +229,14 @@ export function clearFault(nodeId, opts) {
|
|
|
229
229
|
if (opts?.link !== undefined && current.link !== opts.link)
|
|
230
230
|
return false;
|
|
231
231
|
rmSync(faultPath(nodeId));
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
232
|
+
const episode = currentProviderRetryEpisode(nodeId);
|
|
233
|
+
// turn_end precedes agent_settled. An admitted retry keeps its durable
|
|
234
|
+
// coordinate while the broker classifies an error settlement; an invalidated
|
|
235
|
+
// coordinate also remains durable so replacement startup cannot replay it.
|
|
236
|
+
if (opts?.preserveProviderRetryEpisode !== true &&
|
|
237
|
+
episode?.fault.since === current.since &&
|
|
238
|
+
episode.state === 'pending')
|
|
239
|
+
rmSync(providerRetryPath(nodeId), { force: true });
|
|
237
240
|
const operationId = resolveOperationId();
|
|
238
241
|
emitEvent({
|
|
239
242
|
level: 'info',
|
|
@@ -5,8 +5,8 @@
|
|
|
5
5
|
//
|
|
6
6
|
// 1. Finalize — push the agent's last surfaced message as a `final` report so
|
|
7
7
|
// every subscriber/manager waiting on it is unblocked, and mark it done.
|
|
8
|
-
// 2.
|
|
9
|
-
// 3.
|
|
8
|
+
// 2. Recycle — boot a fresh resident broker root and prove it accepts viewers.
|
|
9
|
+
// 3. Close — tear the finalized broker down, then re-attach the pane to the replacement.
|
|
10
10
|
//
|
|
11
11
|
// NOT to be confused with `node lifecycle demote` (flip-to-terminal IN PLACE, which keeps
|
|
12
12
|
// the agent focused and running): recycle ENDS this agent and boots a brand-new
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
// message) — falling back to a short note when it never reported.
|
|
18
18
|
import { readdirSync, readFileSync, statSync } from 'node:fs';
|
|
19
19
|
import { join } from 'node:path';
|
|
20
|
-
import { getNode, setPresence, setFocusOccupant, fullName } from '../canvas/index.js';
|
|
20
|
+
import { getNode, setPresence, setFocusOccupant, fullName, recordLaunch, recordPid } from '../canvas/index.js';
|
|
21
21
|
import { reportsDir } from '../canvas/paths.js';
|
|
22
22
|
import { pushFinal } from '../feed/feed.js';
|
|
23
23
|
import { spawnNode, rootOfSpine } from './nodes.js';
|
|
@@ -25,6 +25,7 @@ import { buildLaunchSpecAsync, buildPiArgv } from './launch.js';
|
|
|
25
25
|
import { focusOf, respawnPaneSync, setPaneOption } from './placement.js';
|
|
26
26
|
import { waitForBrokerViewSocket, viewerSplitEnv } from './placement-tmux.js';
|
|
27
27
|
import { headlessBrokerHost } from './host.js';
|
|
28
|
+
import { transition } from './lifecycle.js';
|
|
28
29
|
import { ensureDaemon } from '../../daemon/manage.js';
|
|
29
30
|
/** The agent's most recent surfaced message: the newest reports/*.md body with
|
|
30
31
|
* its YAML frontmatter stripped. Empty string when the node never reported. */
|
|
@@ -74,23 +75,11 @@ export async function recycleNode(nodeId, callerPane) {
|
|
|
74
75
|
finalized = true;
|
|
75
76
|
}
|
|
76
77
|
catch { /* recycle the pane even if the report failed */ }
|
|
77
|
-
//
|
|
78
|
-
//
|
|
79
|
-
//
|
|
80
|
-
//
|
|
81
|
-
// above (crash-safe order: the daemon won't revive a done node).
|
|
82
|
-
try {
|
|
83
|
-
headlessBrokerHost.teardown(nodeId);
|
|
84
|
-
}
|
|
85
|
-
catch { /* best-effort */ }
|
|
86
|
-
// Capture M's focus viewport (if any) BEFORE nulling — the fresh root inherits
|
|
87
|
-
// it (the SAME focus row + pane). The demoted node no longer holds a pane: it is
|
|
88
|
-
// being reclaimed.
|
|
78
|
+
// Capture M's focus viewport (if any) before booting its replacement. The
|
|
79
|
+
// old broker remains live until the new broker proves it can accept a viewer;
|
|
80
|
+
// a failed replacement must not strand this finished conversation without an
|
|
81
|
+
// engine.
|
|
89
82
|
const f = focusOf(nodeId);
|
|
90
|
-
try {
|
|
91
|
-
setPresence(nodeId, { pane: null, window: null, tmux_session: null });
|
|
92
|
-
}
|
|
93
|
-
catch { /* best-effort */ }
|
|
94
83
|
// 2 + 3. Recycle — boot a fresh resident BROKER root for the SAME pane: the
|
|
95
84
|
// viewer pane re-attaches to the fresh broker (broker-is-the-host — the viewer
|
|
96
85
|
// pane stays a viewer, never becomes an engine pane).
|
|
@@ -113,36 +102,69 @@ export async function recycleNode(nodeId, callerPane) {
|
|
|
113
102
|
profile_id: meta.profile_id,
|
|
114
103
|
launch,
|
|
115
104
|
});
|
|
116
|
-
|
|
117
|
-
|
|
105
|
+
const fresh = getNode(root.node_id);
|
|
106
|
+
const inv = buildPiArgv(fresh);
|
|
107
|
+
// CRTR_SUBTREE groups the fresh root's subtree; FRONT_DOOR is set by the broker
|
|
108
|
+
// host itself, so it is not added here.
|
|
109
|
+
inv.env = { ...inv.env, CRTR_SUBTREE: rootOfSpine(root.node_id) };
|
|
110
|
+
if (!(await launchRecycledBroker(fresh, inv))) {
|
|
111
|
+
return { recycled: false, finalized, newRoot: root.node_id, delivered };
|
|
112
|
+
}
|
|
113
|
+
// COMMIT after replacement readiness: the node's pane is a VIEWER, so tear
|
|
114
|
+
// down the old detached broker only now that the fresh broker can serve it.
|
|
115
|
+
try {
|
|
116
|
+
headlessBrokerHost.teardown(nodeId);
|
|
117
|
+
}
|
|
118
|
+
catch { /* best-effort */ }
|
|
119
|
+
try {
|
|
120
|
+
setPresence(nodeId, { pane: null, window: null, tmux_session: null });
|
|
121
|
+
}
|
|
122
|
+
catch { /* best-effort */ }
|
|
118
123
|
if (f !== null) {
|
|
119
124
|
try {
|
|
120
125
|
setFocusOccupant(f.focus_id, root.node_id);
|
|
121
126
|
}
|
|
122
127
|
catch { /* best-effort */ }
|
|
123
128
|
}
|
|
124
|
-
const
|
|
125
|
-
const inv = buildPiArgv(fresh);
|
|
126
|
-
// CRTR_SUBTREE groups the fresh root's subtree; FRONT_DOOR is set by the broker
|
|
127
|
-
// host itself, so it is not added here.
|
|
128
|
-
inv.env = { ...inv.env, CRTR_SUBTREE: rootOfSpine(root.node_id) };
|
|
129
|
-
const ok = await recycleBrokerViewer(fresh, pane, inv);
|
|
129
|
+
const ok = respawnRecycledViewer(fresh, pane);
|
|
130
130
|
return { recycled: ok, finalized, newRoot: root.node_id, delivered };
|
|
131
131
|
}
|
|
132
|
-
/**
|
|
133
|
-
*
|
|
134
|
-
*
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
132
|
+
/** Launch a recycled root and prove its viewer socket before recycling the old
|
|
133
|
+
* broker. This direct birth path writes the same launch/pid coordinates as
|
|
134
|
+
* reviveNode, and a failed readiness probe terminalizes and tears it down. */
|
|
135
|
+
async function launchRecycledBroker(fresh, inv) {
|
|
136
|
+
let placed;
|
|
137
|
+
try {
|
|
138
|
+
placed = headlessBrokerHost.launch(fresh.node_id, inv, { cwd: fresh.cwd, name: fullName(fresh), resuming: false });
|
|
139
|
+
if (placed.pid === null)
|
|
140
|
+
throw new Error('broker host returned no pid');
|
|
141
|
+
recordLaunch(fresh.node_id, new Date().toISOString());
|
|
142
|
+
recordPid(fresh.node_id, placed.pid);
|
|
143
|
+
if (await waitForBrokerViewSocket(fresh.node_id, placed.exited))
|
|
144
|
+
return true;
|
|
145
|
+
}
|
|
146
|
+
catch {
|
|
147
|
+
// The replacement never became usable; the finished root's broker remains
|
|
148
|
+
// intact and this half-born row cannot remain active without a fleet handle.
|
|
149
|
+
}
|
|
150
|
+
try {
|
|
151
|
+
transitionReplacementToDead(fresh.node_id);
|
|
152
|
+
}
|
|
153
|
+
catch { /* best-effort */ }
|
|
154
|
+
try {
|
|
155
|
+
headlessBrokerHost.teardown(fresh.node_id);
|
|
156
|
+
}
|
|
157
|
+
catch { /* best-effort */ }
|
|
158
|
+
return false;
|
|
159
|
+
}
|
|
160
|
+
function transitionReplacementToDead(nodeId) {
|
|
161
|
+
// spawnNode births active rows, so crash is the authoritative failed-launch
|
|
162
|
+
// outcome whether the host returned no pid or exited before readiness.
|
|
163
|
+
const replacement = getNode(nodeId);
|
|
164
|
+
if (replacement?.status === 'active' || replacement?.status === 'idle')
|
|
165
|
+
transition(nodeId, 'crash');
|
|
166
|
+
}
|
|
167
|
+
function respawnRecycledViewer(fresh, pane) {
|
|
146
168
|
// Clear the finalized node's stale `@crtr_node` tag before respawn so the tag
|
|
147
169
|
// never names a done node during the gap before the new `crtr surface attach` re-tags
|
|
148
170
|
// on connect. Node ids are shell-safe identifiers; no quoting needed.
|
|
@@ -25,7 +25,7 @@ export interface RelaunchDeps {
|
|
|
25
25
|
}>) => boolean | Promise<boolean>;
|
|
26
26
|
/** Re-exec the viewer pane onto the new node. Default: respawnPaneSync. */
|
|
27
27
|
respawnViewer?: typeof respawnPaneSync;
|
|
28
|
-
/** Tear
|
|
28
|
+
/** Tear down a broker after a failed replacement or a committed old-root replacement. */
|
|
29
29
|
teardownBroker?: typeof headlessBrokerHost.teardown;
|
|
30
30
|
}
|
|
31
31
|
export interface RelaunchRootResult {
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
// reserved for finish).
|
|
21
21
|
//
|
|
22
22
|
// Best-effort throughout: a tmux/fs failure on one node never aborts the reap.
|
|
23
|
-
import { getNode, updateNode, fullName, closeFocusRow, view, } from '../canvas/index.js';
|
|
23
|
+
import { getNode, updateNode, recordLaunch, recordPid, fullName, closeFocusRow, view, } from '../canvas/index.js';
|
|
24
24
|
import { transition } from './lifecycle.js';
|
|
25
25
|
import { headlessBrokerHost, requestBrokerTeardown } from './host.js';
|
|
26
26
|
import { tearDownNode, focusOf, registerViewerFocus, respawnPaneSync, windowOfPane, renameWindow, } from './placement.js';
|
|
@@ -129,7 +129,30 @@ export async function relaunchRoot(oldId, deps = {}) {
|
|
|
129
129
|
transition(newMeta.node_id, 'crash');
|
|
130
130
|
return null;
|
|
131
131
|
}
|
|
132
|
-
|
|
132
|
+
// A replacement is a managed broker just like a revive: the fleet owns its
|
|
133
|
+
// real exit, while the row gets the matching launch and pid coordinates before
|
|
134
|
+
// readiness can observe an early exit.
|
|
135
|
+
recordLaunch(newMeta.node_id, new Date().toISOString());
|
|
136
|
+
recordPid(newMeta.node_id, placed.pid);
|
|
137
|
+
let ready = false;
|
|
138
|
+
try {
|
|
139
|
+
ready = await waitForViewSocket(newMeta.node_id, placed.exited);
|
|
140
|
+
}
|
|
141
|
+
catch {
|
|
142
|
+
// A readiness probe error means the replacement is not proven usable.
|
|
143
|
+
}
|
|
144
|
+
if (!ready) {
|
|
145
|
+
// Do not let a broker that can still bind later sit behind a dead row, and
|
|
146
|
+
// do not commit any old-root teardown without a usable replacement.
|
|
147
|
+
const replacement = getNode(newMeta.node_id);
|
|
148
|
+
if (replacement?.status === 'active' || replacement?.status === 'idle')
|
|
149
|
+
transition(newMeta.node_id, 'crash');
|
|
150
|
+
try {
|
|
151
|
+
teardownBroker(newMeta.node_id);
|
|
152
|
+
}
|
|
153
|
+
catch { /* best-effort */ }
|
|
154
|
+
return null;
|
|
155
|
+
}
|
|
133
156
|
// --- COMMIT: park + reap the old root, re-point the viewer, kill the old
|
|
134
157
|
// broker. Past this point the new node is the live root. ---
|
|
135
158
|
reapDescendants(oldId); // old workers → canceled + torn down
|
|
@@ -230,10 +230,15 @@ function launchRevive(nodeId, meta, opts) {
|
|
|
230
230
|
// fail deep inside the crash/doctrine-wake catch below, where it would
|
|
231
231
|
// surface as an opaque `internal error` (toErrorBody masks every 5xx).
|
|
232
232
|
if (!existsSync(meta.cwd)) {
|
|
233
|
+
const next = meta.fork_from != null
|
|
234
|
+
? `Restore ${meta.cwd}, then revive the fork normally with \`crtr node lifecycle revive ${nodeId}\`.`
|
|
235
|
+
: meta.managed_worktree?.state === 'open'
|
|
236
|
+
? 'Restore or reconcile the recorded checkout, then use the managed-worktree close or abandon recovery.'
|
|
237
|
+
: `Recreate ${meta.cwd}, or repair it with \`crtr node config --cwd <path> --node ${nodeId}\`; durable node work resumes in a fresh conversation. Then retry \`crtr node lifecycle revive ${nodeId}\`.`;
|
|
233
238
|
throw new InputError({
|
|
234
239
|
error: 'cwd_missing',
|
|
235
240
|
message: `${nodeId} cannot be revived — its recorded cwd no longer exists: ${meta.cwd}`,
|
|
236
|
-
next
|
|
241
|
+
next,
|
|
237
242
|
});
|
|
238
243
|
}
|
|
239
244
|
// Lazy host_kind coerce (§C): every launch now uses the broker host. Persist
|
|
@@ -327,7 +327,7 @@ async function handleMessage(ctx) {
|
|
|
327
327
|
const entry = deliver();
|
|
328
328
|
// Deferred mail normally waits for a natural cycle, but a frozen row already
|
|
329
329
|
// owes a thaw; consume its deadline when that durable future wake arrives.
|
|
330
|
-
if (tier !== 'deferred' || meta.frozen_at !== null)
|
|
330
|
+
if (entry.tier !== 'deferred' || meta.frozen_at !== null)
|
|
331
331
|
cancelCronsOnWake(id);
|
|
332
332
|
// A wake-capable tier revives a dormant target so its inbox-watcher delivers
|
|
333
333
|
// this; deferred never wakes — it rides the target's next natural cycle. The
|
|
@@ -7,8 +7,8 @@
|
|
|
7
7
|
//
|
|
8
8
|
// `{id}` is the target node.
|
|
9
9
|
import { envProfileId } from '../../../shared/env.js';
|
|
10
|
-
import { readdirSync } from 'node:fs';
|
|
11
|
-
import { basename } from 'node:path';
|
|
10
|
+
import { existsSync, readdirSync, statSync } from 'node:fs';
|
|
11
|
+
import { basename, isAbsolute, resolve } from 'node:path';
|
|
12
12
|
import { appendSituationalContext, formatSituationalProse } from '../../../core/runtime/situational-context.js';
|
|
13
13
|
import { closeNode } from '../../../core/runtime/close.js';
|
|
14
14
|
import { reviveNode } from '../../../core/runtime/revive.js';
|
|
@@ -35,6 +35,7 @@ import { BrokerUnreachableError } from '../../../core/runtime/broker-request.js'
|
|
|
35
35
|
import { readFault } from '../../../core/runtime/fault.js';
|
|
36
36
|
import { activeFaultForDisplay } from '../../../core/canvas/render-source.js';
|
|
37
37
|
import { isBrokerLive, persistDormantModel, setModelLive } from '../../../core/runtime/model-swap.js';
|
|
38
|
+
import { boundFleet } from '../../../core/runtime/fleet.js';
|
|
38
39
|
import { writeOutputSchema } from '../../../core/runtime/structured-output.js';
|
|
39
40
|
import { claimWarmNode, refillWarmPool, warmPoolEnabled } from '../../../core/runtime/warm-pool.js';
|
|
40
41
|
import { resolveLaunchTarget } from '../../../core/runtime/launch-target.js';
|
|
@@ -45,6 +46,7 @@ import { cronWakeOrigin } from '../../../core/runtime/bearings.js';
|
|
|
45
46
|
import { notFound, usage } from '../../../core/errors.js';
|
|
46
47
|
import { toCloseResultDTO, toNodeDetailDTO, toNodeSummaryDTO, toReviveResultDTO } from '../map.js';
|
|
47
48
|
import { ApiError } from '../../../api/index.js';
|
|
49
|
+
import { InputError } from '../../../core/io.js';
|
|
48
50
|
// Shared helpers
|
|
49
51
|
function requireMeta(id) {
|
|
50
52
|
const meta = getNode(id);
|
|
@@ -622,11 +624,129 @@ function handleReviveAll() {
|
|
|
622
624
|
};
|
|
623
625
|
return { status: 200, body };
|
|
624
626
|
}
|
|
625
|
-
// PATCH /v1/nodes/{id}/config —
|
|
627
|
+
// PATCH /v1/nodes/{id}/config — durable config and repair-only cwd (spec §6.2)
|
|
626
628
|
export async function handleConfig(ctx, deps = {}) {
|
|
627
629
|
const id = ctx.params['id'];
|
|
630
|
+
const body = ctx.body;
|
|
631
|
+
let patch;
|
|
632
|
+
if (body === undefined) {
|
|
633
|
+
patch = {};
|
|
634
|
+
}
|
|
635
|
+
else if (typeof body === 'object' && body !== null && !Array.isArray(body)) {
|
|
636
|
+
patch = body;
|
|
637
|
+
}
|
|
638
|
+
else {
|
|
639
|
+
throw new InputError({
|
|
640
|
+
error: 'cwd_invalid',
|
|
641
|
+
message: 'cwd repair requires a non-null object with a non-empty cwd string',
|
|
642
|
+
received: body,
|
|
643
|
+
field: 'cwd',
|
|
644
|
+
next: 'Send a config patch with a non-empty absolute cwd string.',
|
|
645
|
+
});
|
|
646
|
+
}
|
|
647
|
+
if (Object.prototype.hasOwnProperty.call(patch, 'cwd')) {
|
|
648
|
+
const cwd = patch.cwd;
|
|
649
|
+
if (typeof cwd !== 'string' || cwd === '') {
|
|
650
|
+
throw new InputError({
|
|
651
|
+
error: 'cwd_invalid',
|
|
652
|
+
message: 'cwd must be a non-empty string',
|
|
653
|
+
received: cwd,
|
|
654
|
+
field: 'cwd',
|
|
655
|
+
next: 'Send a non-empty absolute cwd string.',
|
|
656
|
+
});
|
|
657
|
+
}
|
|
658
|
+
if (Object.keys(patch).some((key) => key !== 'cwd' && patch[key] !== undefined)) {
|
|
659
|
+
throw new InputError({
|
|
660
|
+
error: 'cwd_not_exclusive',
|
|
661
|
+
message: 'cwd repair cannot be combined with another config field',
|
|
662
|
+
received: patch,
|
|
663
|
+
field: 'cwd',
|
|
664
|
+
next: 'Send cwd as the only config field.',
|
|
665
|
+
});
|
|
666
|
+
}
|
|
667
|
+
const meta = requireMeta(id);
|
|
668
|
+
if (boundFleet().has(id)) {
|
|
669
|
+
const message = `${id} cannot repair its cwd while its broker is live`;
|
|
670
|
+
throw new ApiError(409, 'cwd_node_live', message, {
|
|
671
|
+
error: 'cwd_node_live',
|
|
672
|
+
message,
|
|
673
|
+
received: cwd,
|
|
674
|
+
field: 'cwd',
|
|
675
|
+
next: 'Let the node become dormant, then retry the cwd repair.',
|
|
676
|
+
});
|
|
677
|
+
}
|
|
678
|
+
if (meta.fork_from != null) {
|
|
679
|
+
throw new InputError({
|
|
680
|
+
error: 'cwd_node_forked',
|
|
681
|
+
message: `${id} is a fork and cannot repair its cwd; recorded cwd: ${meta.cwd}`,
|
|
682
|
+
received: cwd,
|
|
683
|
+
field: 'cwd',
|
|
684
|
+
next: `Restore ${meta.cwd}, then revive the fork normally.`,
|
|
685
|
+
});
|
|
686
|
+
}
|
|
687
|
+
if (meta.managed_worktree?.state === 'open') {
|
|
688
|
+
throw new InputError({
|
|
689
|
+
error: 'cwd_node_managed_worktree_open',
|
|
690
|
+
message: `${id} owns an open managed worktree and cannot repair its cwd`,
|
|
691
|
+
received: cwd,
|
|
692
|
+
field: 'cwd',
|
|
693
|
+
next: 'Restore or reconcile the recorded checkout, then use the managed-worktree close or abandon flow.',
|
|
694
|
+
});
|
|
695
|
+
}
|
|
696
|
+
if (existsSync(meta.cwd)) {
|
|
697
|
+
throw new InputError({
|
|
698
|
+
error: 'cwd_still_present',
|
|
699
|
+
message: `${id} recorded cwd still exists: ${meta.cwd}; cwd repair is not a relocation command`,
|
|
700
|
+
received: cwd,
|
|
701
|
+
field: 'cwd',
|
|
702
|
+
next: 'This command only repairs a node whose recorded cwd is gone.',
|
|
703
|
+
});
|
|
704
|
+
}
|
|
705
|
+
if (!isAbsolute(cwd)) {
|
|
706
|
+
throw new InputError({
|
|
707
|
+
error: 'cwd_not_absolute',
|
|
708
|
+
message: `cwd must be an absolute path: ${cwd}`,
|
|
709
|
+
received: cwd,
|
|
710
|
+
field: 'cwd',
|
|
711
|
+
next: 'Send an absolute path.',
|
|
712
|
+
});
|
|
713
|
+
}
|
|
714
|
+
const resolved = resolve(cwd);
|
|
715
|
+
if (!existsSync(resolved)) {
|
|
716
|
+
throw new InputError({
|
|
717
|
+
error: 'cwd_missing',
|
|
718
|
+
message: `cwd does not exist: ${resolved}`,
|
|
719
|
+
received: cwd,
|
|
720
|
+
field: 'cwd',
|
|
721
|
+
next: 'Create the replacement directory or send an existing absolute directory.',
|
|
722
|
+
});
|
|
723
|
+
}
|
|
724
|
+
try {
|
|
725
|
+
if (!statSync(resolved).isDirectory()) {
|
|
726
|
+
throw new InputError({
|
|
727
|
+
error: 'cwd_not_a_directory',
|
|
728
|
+
message: `cwd is not a directory: ${resolved}`,
|
|
729
|
+
received: cwd,
|
|
730
|
+
field: 'cwd',
|
|
731
|
+
next: 'Send an existing absolute directory.',
|
|
732
|
+
});
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
catch (error) {
|
|
736
|
+
if (error instanceof InputError)
|
|
737
|
+
throw error;
|
|
738
|
+
throw new InputError({
|
|
739
|
+
error: 'cwd_missing',
|
|
740
|
+
message: `cwd does not exist: ${resolved}`,
|
|
741
|
+
received: cwd,
|
|
742
|
+
field: 'cwd',
|
|
743
|
+
next: 'Create the replacement directory or send an existing absolute directory.',
|
|
744
|
+
});
|
|
745
|
+
}
|
|
746
|
+
updateNode(id, { cwd: resolved, pi_session_id: null, pi_session_file: null });
|
|
747
|
+
return { status: 200, body: detailOf(id) };
|
|
748
|
+
}
|
|
628
749
|
let meta = requireMeta(id);
|
|
629
|
-
const patch = ctx.body ?? {};
|
|
630
750
|
if (patch.situational_context !== undefined) {
|
|
631
751
|
appendSituationalContext(id, formatSituationalProse(patch.situational_context));
|
|
632
752
|
}
|
|
@@ -33,7 +33,7 @@ function parsePushBody(body) {
|
|
|
33
33
|
if (deliveryTier !== undefined && deliveryTier !== 'deferred' && deliveryTier !== 'normal' && deliveryTier !== 'urgent') {
|
|
34
34
|
throw usage(`invalid delivery_tier: ${String(deliveryTier)} (expected deferred|normal|urgent)`);
|
|
35
35
|
}
|
|
36
|
-
if (tier
|
|
36
|
+
if (tier !== 'update' && deliveryTier !== undefined) {
|
|
37
37
|
throw usage('delivery_tier is only valid for update reports');
|
|
38
38
|
}
|
|
39
39
|
if (typeof text !== 'string' || text.trim() === '') {
|
package/dist/daemon/api/map.d.ts
CHANGED
|
@@ -112,7 +112,7 @@ export declare function errorToStatus(thrown: unknown): {
|
|
|
112
112
|
};
|
|
113
113
|
/** Build the uniform JSON error body (spec §8) for a thrown value, using
|
|
114
114
|
* `errorToStatus` for the status/code. Structured `details` are surfaced for a
|
|
115
|
-
* CrtrError (they let the client reconstruct the error verbatim). A 500 still
|
|
115
|
+
* CrtrError or ApiError (they let the client reconstruct the error verbatim). A 500 still
|
|
116
116
|
* omits `details` for hygiene, but NO LONGER strips the message to a bare
|
|
117
117
|
* `internal error`: it surfaces the bounded cause (`boundedCause`) so a caller
|
|
118
118
|
* is never left with a causeless internal, while the router logs the full
|
package/dist/daemon/api/map.js
CHANGED
|
@@ -17,6 +17,7 @@ import { NodeIdConflictError } from '../../core/runtime/nodes.js';
|
|
|
17
17
|
import { readContextTokens } from '../../core/canvas/telemetry.js';
|
|
18
18
|
import { CrtrError } from '../../core/errors.js';
|
|
19
19
|
import { WorktreeError } from '../../core/worktree.js';
|
|
20
|
+
import { ApiError } from '../../api/errors.js';
|
|
20
21
|
import { ExitCode } from '../../types.js';
|
|
21
22
|
import { sinkDisplay } from '../cron-sink.js';
|
|
22
23
|
import { responsePath } from '../../core/human/convention.js';
|
|
@@ -563,7 +564,7 @@ function boundedCause(err) {
|
|
|
563
564
|
}
|
|
564
565
|
/** Build the uniform JSON error body (spec §8) for a thrown value, using
|
|
565
566
|
* `errorToStatus` for the status/code. Structured `details` are surfaced for a
|
|
566
|
-
* CrtrError (they let the client reconstruct the error verbatim). A 500 still
|
|
567
|
+
* CrtrError or ApiError (they let the client reconstruct the error verbatim). A 500 still
|
|
567
568
|
* omits `details` for hygiene, but NO LONGER strips the message to a bare
|
|
568
569
|
* `internal error`: it surfaces the bounded cause (`boundedCause`) so a caller
|
|
569
570
|
* is never left with a causeless internal, while the router logs the full
|
|
@@ -577,7 +578,7 @@ export function toErrorBody(thrown) {
|
|
|
577
578
|
const message = err instanceof Error ? err.message : String(err);
|
|
578
579
|
const details = err instanceof ReviewOperationErrorClass
|
|
579
580
|
? reviewErrorDetails(err)
|
|
580
|
-
: err instanceof CrtrError && err.details !== undefined
|
|
581
|
+
: (err instanceof CrtrError || err instanceof ApiError) && err.details !== undefined
|
|
581
582
|
? err.details
|
|
582
583
|
: undefined;
|
|
583
584
|
const body = { error: { code, message } };
|
|
@@ -31,7 +31,7 @@ function deliverNodeSink(c, target, stdout) {
|
|
|
31
31
|
label: `⏰ cron ${c.name}`,
|
|
32
32
|
data: { body: stdout },
|
|
33
33
|
});
|
|
34
|
-
if (
|
|
34
|
+
if (entry.tier !== 'deferred' || meta.frozen_at !== null)
|
|
35
35
|
cancelCronsOnWake(target);
|
|
36
36
|
// The APPENDED tier decides the wake, not the cron's requested one: a deferred
|
|
37
37
|
// cron aimed at a terminal node was raised out of `deferred` by the append.
|