@north-light/crouter 0.3.250 → 0.3.252
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builtin-memory/02-turn-lifecycle/00-ending-a-turn.md +1 -5
- package/dist/builtin-memory/05-kinds/review/security-findings.md +1 -1
- package/dist/clients/attach/__tests__/completion-frecency.test.d.ts +1 -0
- package/dist/clients/attach/__tests__/completion-frecency.test.js +35 -0
- package/dist/clients/attach/__tests__/ref-autocomplete.test.js +3 -1
- package/dist/clients/attach/__tests__/titled-editor-preview.test.js +2 -1
- package/dist/clients/attach/input/completion-frecency.d.ts +45 -0
- package/dist/clients/attach/input/completion-frecency.js +141 -0
- package/dist/clients/attach/input/controller.d.ts +17 -0
- package/dist/clients/attach/input/controller.js +40 -0
- package/dist/clients/attach/input/ref-autocomplete.d.ts +3 -1
- package/dist/clients/attach/input/ref-autocomplete.js +49 -8
- package/dist/clients/attach/input/titled-editor.d.ts +3 -0
- package/dist/clients/attach/input/titled-editor.js +5 -0
- package/dist/clients/attach/session/editor-inventory.d.ts +3 -0
- package/dist/clients/attach/session/editor-inventory.js +1 -1
- package/dist/clients/attach/session/input-wiring.d.ts +3 -0
- package/dist/clients/attach/session/input-wiring.js +2 -0
- package/dist/clients/attach/viewer.js +578 -578
- package/dist/core/__tests__/canvas-inbox-watcher-hold.test.js +98 -1
- package/dist/core/__tests__/canvas-inbox-watcher-naming.test.js +4 -1
- package/dist/core/__tests__/canvas-inbox-watcher.test.js +4 -1
- package/dist/core/__tests__/fixtures/fake-engine.d.ts +3 -1
- package/dist/core/__tests__/fixtures/fake-engine.js +16 -3
- package/dist/core/__tests__/integration/deferred-no-wake.test.js +4 -1
- package/dist/core/__tests__/migration.test.js +57 -11
- package/dist/core/__tests__/seam/broker-provider-retry.test.js +17 -2
- package/dist/core/__tests__/seam/dormancy-release.test.js +1 -0
- package/dist/core/__tests__/seam/held-deferred-human-prompt.test.d.ts +1 -0
- package/dist/core/__tests__/seam/held-deferred-human-prompt.test.js +63 -0
- package/dist/core/__tests__/watchdog-abort-arms-retry.test.js +9 -2
- package/dist/core/canvas/migrations.js +63 -24
- package/dist/core/runtime/broker/engine-drive.js +20 -3
- package/dist/core/runtime/broker/fault-retry.js +31 -13
- package/dist/core/runtime/broker/held-deferred-inbox.d.ts +13 -0
- package/dist/core/runtime/broker/held-deferred-inbox.js +13 -0
- package/dist/core/runtime/broker.js +19 -0
- package/dist/core/runtime/close.js +2 -2
- package/dist/core/runtime/fault.js +8 -5
- package/dist/core/runtime/recycle.js +63 -41
- package/dist/core/runtime/reset.d.ts +1 -1
- package/dist/core/runtime/reset.js +25 -2
- package/dist/daemon/__tests__/helpers/source-daemon.js +1 -0
- package/dist/daemon/api/handlers/messages.js +1 -1
- package/dist/daemon/api/handlers/reports.js +1 -1
- package/dist/daemon/cron/sinks.js +1 -1
- package/dist/daemon/messaging/node-message.js +1 -1
- package/dist/pi-extensions/canvas-inbox-watcher.js +109 -5
- package/package.json +1 -1
- package/runtime.lock.json +2 -2
|
@@ -26,14 +26,19 @@ import { basename, join, resolve } from 'node:path';
|
|
|
26
26
|
import { crtrHome, isSafeNodeId, nodesRoot, nodeMetaPath } from './paths.js';
|
|
27
27
|
import { STATE_MIGRATIONS } from '../../migrations/registry.js';
|
|
28
28
|
import { customExtensionPaths } from './extensions.js';
|
|
29
|
-
import { updateJsonFileDurably } from './meta-file.js';
|
|
29
|
+
import { readMetaObject, updateJsonFileDurably, withMetaLock, writeMetaObjectLocked } from './meta-file.js';
|
|
30
30
|
// Schema as a forward-only migration list
|
|
31
31
|
//
|
|
32
32
|
// The schema is the migration list: one place a schema change is expressed,
|
|
33
33
|
// one gate (`PRAGMA user_version`) that applies it. `migrate()` runs every
|
|
34
34
|
// pending step in order and bumps `user_version` after each, so a fresh db and
|
|
35
35
|
// the live fleet (all at `user_version 0`) converge on the same final shape.
|
|
36
|
-
// Migrations are append-only and forward-only — never edit a shipped step.
|
|
36
|
+
// Migrations are append-only and forward-only — never edit a shipped step. The
|
|
37
|
+
// v36 body below is the sole corrective exception: v0.3.250 shipped a destructive
|
|
38
|
+
// stale-row decision, so v36 is narrowed for databases still below its gate while
|
|
39
|
+
// appended v37 repairs databases that already crossed it. Do not repeat this
|
|
40
|
+
// exception; an appended migration is required unless it would run too late to
|
|
41
|
+
// prevent destruction at an existing version boundary.
|
|
37
42
|
/** v1 — the baseline tables + indexes. `IF NOT EXISTS` makes this a no-op on
|
|
38
43
|
* any existing db (the live fleet already has these tables). */
|
|
39
44
|
function baselineSchema(db) {
|
|
@@ -1215,32 +1220,43 @@ function addNodeSessionFileIndex(db) {
|
|
|
1215
1220
|
}
|
|
1216
1221
|
db.exec('CREATE INDEX IF NOT EXISTS idx_nodes_pi_session_file ON nodes(pi_session_file);');
|
|
1217
1222
|
}
|
|
1218
|
-
/**
|
|
1219
|
-
*
|
|
1220
|
-
*
|
|
1221
|
-
*
|
|
1222
|
-
function
|
|
1223
|
+
/** Reproject the derived session-file index from authoritative node metadata.
|
|
1224
|
+
* v36 may resolve duplicate authoritative claimants because it runs before the
|
|
1225
|
+
* first UNIQUE index exists. v37 must retain every surviving meta claim: a
|
|
1226
|
+
* conflict at that later boundary is irreconcilable and fails instead. */
|
|
1227
|
+
function reconcileNodeSessionFileOwners(db, clearDuplicateMetaClaimants) {
|
|
1223
1228
|
const canonical = (sessionFile) => {
|
|
1224
1229
|
const absolute = resolve(sessionFile);
|
|
1225
1230
|
return existsSync(absolute) ? realpathSync(absolute) : absolute;
|
|
1226
1231
|
};
|
|
1227
|
-
const rows = db.prepare(`
|
|
1228
|
-
SELECT node_id, pi_session_file
|
|
1229
|
-
FROM nodes
|
|
1230
|
-
WHERE pi_session_file IS NOT NULL
|
|
1231
|
-
ORDER BY created ASC, node_id ASC
|
|
1232
|
-
`).all();
|
|
1233
1232
|
const setFile = db.prepare('UPDATE nodes SET pi_session_file = ? WHERE node_id = ?');
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1233
|
+
db.exec('DROP INDEX IF EXISTS idx_nodes_pi_session_file;');
|
|
1234
|
+
const rows = db.prepare('SELECT node_id FROM nodes ORDER BY created ASC, node_id ASC').all();
|
|
1235
|
+
for (const { node_id } of rows) {
|
|
1236
|
+
const metaPath = nodeMetaPath(node_id);
|
|
1237
|
+
// Missing or unreadable metadata has no authoritative coordinate. Avoid
|
|
1238
|
+
// taking a file lock there: old rows can outlive a removed node directory.
|
|
1239
|
+
if (readMetaObject(metaPath) === null) {
|
|
1240
|
+
setFile.run(null, node_id);
|
|
1237
1241
|
continue;
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
meta
|
|
1241
|
-
|
|
1242
|
+
}
|
|
1243
|
+
const sessionFile = withMetaLock(metaPath, () => {
|
|
1244
|
+
const meta = readMetaObject(metaPath);
|
|
1245
|
+
if (meta === null)
|
|
1246
|
+
throw new Error(`node ${node_id} session metadata changed during ownership migration`);
|
|
1247
|
+
if (typeof meta['pi_session_file'] !== 'string')
|
|
1248
|
+
return null;
|
|
1249
|
+
const storedSessionFile = meta['pi_session_file'];
|
|
1250
|
+
const normalizedSessionFile = canonical(storedSessionFile);
|
|
1251
|
+
if (normalizedSessionFile !== storedSessionFile) {
|
|
1252
|
+
meta['pi_session_file'] = normalizedSessionFile;
|
|
1253
|
+
writeMetaObjectLocked(metaPath, meta);
|
|
1254
|
+
}
|
|
1255
|
+
return normalizedSessionFile;
|
|
1242
1256
|
});
|
|
1243
|
-
|
|
1257
|
+
// meta.json is authoritative: a crash after its write can leave this row
|
|
1258
|
+
// claiming another node's old session coordinate.
|
|
1259
|
+
setFile.run(sessionFile, node_id);
|
|
1244
1260
|
}
|
|
1245
1261
|
const duplicates = db.prepare(`
|
|
1246
1262
|
SELECT pi_session_file
|
|
@@ -1249,6 +1265,10 @@ WHERE pi_session_file IS NOT NULL
|
|
|
1249
1265
|
GROUP BY pi_session_file
|
|
1250
1266
|
HAVING COUNT(*) > 1
|
|
1251
1267
|
`).all();
|
|
1268
|
+
if (!clearDuplicateMetaClaimants && duplicates.length > 0) {
|
|
1269
|
+
const coordinates = duplicates.map(({ pi_session_file }) => pi_session_file).join(', ');
|
|
1270
|
+
throw new Error(`cannot restore unique Pi session ownership: authoritative metadata has concurrent claimants for ${coordinates}`);
|
|
1271
|
+
}
|
|
1252
1272
|
const clearRow = db.prepare('UPDATE nodes SET pi_session_file = NULL WHERE node_id = ?');
|
|
1253
1273
|
for (const { pi_session_file } of duplicates) {
|
|
1254
1274
|
const owners = db.prepare(`
|
|
@@ -1259,18 +1279,36 @@ ORDER BY created ASC, node_id ASC
|
|
|
1259
1279
|
`).all(pi_session_file);
|
|
1260
1280
|
for (const { node_id } of owners.slice(1)) {
|
|
1261
1281
|
// Clear both durable coordinates so a later reindex cannot restore the
|
|
1262
|
-
// duplicate projection.
|
|
1263
|
-
|
|
1282
|
+
// duplicate projection. The checked write fails rather than erasing a
|
|
1283
|
+
// concurrent metadata change.
|
|
1284
|
+
const cleared = updateJsonFileDurably(nodeMetaPath(node_id), (meta) => {
|
|
1285
|
+
const currentSessionFile = meta['pi_session_file'];
|
|
1286
|
+
if (typeof currentSessionFile !== 'string' || canonical(currentSessionFile) !== pi_session_file) {
|
|
1287
|
+
throw new Error(`node ${node_id} session metadata changed during ownership migration`);
|
|
1288
|
+
}
|
|
1264
1289
|
meta['pi_session_id'] = null;
|
|
1265
1290
|
meta['pi_session_file'] = null;
|
|
1266
1291
|
return true;
|
|
1267
1292
|
});
|
|
1293
|
+
if (!cleared)
|
|
1294
|
+
throw new Error(`node ${node_id} session metadata changed during ownership migration`);
|
|
1268
1295
|
clearRow.run(node_id);
|
|
1269
1296
|
}
|
|
1270
1297
|
}
|
|
1271
|
-
db.exec('DROP INDEX IF EXISTS idx_nodes_pi_session_file;');
|
|
1272
1298
|
db.exec('CREATE UNIQUE INDEX IF NOT EXISTS idx_nodes_pi_session_file ON nodes(pi_session_file);');
|
|
1273
1299
|
}
|
|
1300
|
+
/** v36 — one Pi transcript may have exactly one node owner. This is the narrow
|
|
1301
|
+
* corrective body for clients below v36; it reprojects metadata before choosing
|
|
1302
|
+
* the earliest created duplicate claimant (node_id breaks equal timestamps). */
|
|
1303
|
+
function makeNodeSessionFileUnique(db) {
|
|
1304
|
+
reconcileNodeSessionFileOwners(db, true);
|
|
1305
|
+
}
|
|
1306
|
+
/** v37 — repair v0.3.250's already-applied v36 projection without inventing
|
|
1307
|
+
* coordinates v36 already cleared. Surviving authoritative metadata wins over
|
|
1308
|
+
* stale rows; concurrent surviving claims fail loudly rather than clearing one. */
|
|
1309
|
+
function repairNodeSessionFileUniqueIndex(db) {
|
|
1310
|
+
reconcileNodeSessionFileOwners(db, false);
|
|
1311
|
+
}
|
|
1274
1312
|
/** v35 — durable node lifecycle execution state and terminal cause. */
|
|
1275
1313
|
function addNodeLifecycleColumns(db) {
|
|
1276
1314
|
const columns = nodeColumns(db);
|
|
@@ -1340,6 +1378,7 @@ export const MIGRATIONS = [
|
|
|
1340
1378
|
/* v34 */ addNodeSessionFileIndex,
|
|
1341
1379
|
/* v35 */ addNodeLifecycleColumns,
|
|
1342
1380
|
/* v36 */ makeNodeSessionFileUnique,
|
|
1381
|
+
/* v37 */ repairNodeSessionFileUniqueIndex,
|
|
1343
1382
|
...JOURNALED_STATE_MIGRATIONS,
|
|
1344
1383
|
];
|
|
1345
1384
|
/** Migration indexes that manage their OWN transaction and therefore must not
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { emitEvent } from '../../events/emit.js';
|
|
2
2
|
import { operationIdContext } from '../../events/operation-id.js';
|
|
3
3
|
import { promptWithAdvertisedCommandInvocation } from '../advertised-command-invocation.js';
|
|
4
|
+
import { heldDeferredInboxPromptJoin } from './held-deferred-inbox.js';
|
|
4
5
|
import { chooseGuidanceDeliveryMode, promptWithNextTurnGuidance, resolveEngineRoute, resolveSteerCall, } from './engine-routing.js';
|
|
5
6
|
export function createEngineDriver(deps) {
|
|
6
7
|
const { registry, replies, session, dispatches, memoryRefs, authReloadGate, rebind, projection, toolGroups, turnAdmission, } = deps;
|
|
@@ -44,14 +45,30 @@ export function createEngineDriver(deps) {
|
|
|
44
45
|
// false. Admission waits for that preflight, then routes against the
|
|
45
46
|
// state it actually established instead of its stale snapshot.
|
|
46
47
|
const route = resolveEngineRoute(frame, session().isStreaming);
|
|
48
|
+
const promptOptions = route.call === 'prompt' ? route.options : undefined;
|
|
47
49
|
const guidance = memoryRefs.guidanceFor(frame.text);
|
|
50
|
+
const prompt = async () => {
|
|
51
|
+
// Both human surfaces converge at this admitted boundary. The watcher
|
|
52
|
+
// claims held deferred mail before the prompt body, but its cursor stays
|
|
53
|
+
// uncommitted until the resulting turn's agent_settled event.
|
|
54
|
+
if (route.call === 'prompt' && !session().isStreaming) {
|
|
55
|
+
try {
|
|
56
|
+
await heldDeferredInboxPromptJoin()?.(session());
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
// The watcher emitted the failed durable delivery and retains it for
|
|
60
|
+
// replay. A person's live prompt is not replayable, so it must run.
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
await session().prompt(frame.text, { ...promptOptions, preflightResult: release });
|
|
64
|
+
};
|
|
48
65
|
if (guidance === null) {
|
|
49
66
|
if (route.call === 'prompt') {
|
|
50
67
|
// A routed prompt STARTS a turn — including an idle follow_up
|
|
51
68
|
// escalated by m-C — so it gets a fresh operation id. A true
|
|
52
69
|
// (mid-stream) followUp rides the current turn's operation, like
|
|
53
70
|
// steer.
|
|
54
|
-
return settle(dispatches.track(operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text,
|
|
71
|
+
return settle(dispatches.track(operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text, prompt).catch(relayError))));
|
|
55
72
|
}
|
|
56
73
|
// C1: pi's followUp() takes images as a POSITIONAL 2nd arg (`followUp(text, images?)`).
|
|
57
74
|
release();
|
|
@@ -67,9 +84,9 @@ export function createEngineDriver(deps) {
|
|
|
67
84
|
// ATOMICALLY clears its own pending entry if prompt() settles
|
|
68
85
|
// without ever reaching that drain (an early throw/return leaves
|
|
69
86
|
// nothing stranded to leak into a later, unrelated turn).
|
|
70
|
-
return settle(dispatches.track(promptWithNextTurnGuidance(session(), guidance, () => operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text,
|
|
87
|
+
return settle(dispatches.track(promptWithNextTurnGuidance(session(), guidance, () => operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text, prompt))).catch(relayError)));
|
|
71
88
|
}
|
|
72
|
-
return settle(dispatches.track(operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text,
|
|
89
|
+
return settle(dispatches.track(operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text, prompt)
|
|
73
90
|
.catch(relayError)
|
|
74
91
|
.then(() => sendGuidance(guidance, deliverAs)))));
|
|
75
92
|
}
|
|
@@ -58,19 +58,26 @@ export class FaultRetry {
|
|
|
58
58
|
generation.stagedRefreshAbort = false;
|
|
59
59
|
generation.stagedOverflowFailure = null;
|
|
60
60
|
const settledSession = generation.session;
|
|
61
|
-
//
|
|
62
|
-
//
|
|
63
|
-
//
|
|
64
|
-
// each retry rewinds to the same fork point instead of stacking recovery
|
|
65
|
-
// prompts, and so backoff/exhaustion actually accumulate.
|
|
61
|
+
// turn_end clears the ordinary marker before this settlement. A failed
|
|
62
|
+
// admitted retry therefore carries from its durable episode, while an
|
|
63
|
+
// initial provider failure still reads from the ordinary marker.
|
|
66
64
|
const prior = readFault(this.deps.nodeId);
|
|
67
|
-
const
|
|
68
|
-
|
|
69
|
-
|
|
65
|
+
const durableEpisode = readProviderRetryEpisode(this.deps.nodeId);
|
|
66
|
+
const episodeFault = this.isActiveAutoFault(prior) && prior.link === 'pi→provider'
|
|
67
|
+
? prior
|
|
68
|
+
: durableEpisode?.state === 'admitted' &&
|
|
69
|
+
durableEpisode.sessionFile === this.sessionFile(settledSession) &&
|
|
70
|
+
this.isActiveAutoFault(durableEpisode.fault)
|
|
71
|
+
? durableEpisode.fault
|
|
72
|
+
: null;
|
|
73
|
+
const episode = episodeFault === null
|
|
74
|
+
? null
|
|
75
|
+
: { since: episodeFault.since, anchorEntryId: episodeFault.anchorEntryId, attempt: episodeFault.retry.attempt };
|
|
70
76
|
const messages = Array.isArray(agentEnd?.messages) ? agentEnd.messages : [];
|
|
71
77
|
const last = [...messages].reverse().find((message) => typeof message === 'object' && message !== null && message.role === 'assistant');
|
|
72
78
|
if (overflowFailure !== null) {
|
|
73
79
|
clearFault(this.deps.nodeId, { link: 'pi→provider' });
|
|
80
|
+
clearProviderRetryEpisode(this.deps.nodeId);
|
|
74
81
|
recordFault(this.deps.nodeId, {
|
|
75
82
|
link: 'pi→provider', op: 'context overflow recovery', kind: 'context-overflow', retry: { disposition: 'fatal' },
|
|
76
83
|
message: overflowFailure.errorMessage, anchorEntryId: settledSession.sessionManager.getLeafId?.() ?? undefined,
|
|
@@ -87,21 +94,30 @@ export class FaultRetry {
|
|
|
87
94
|
});
|
|
88
95
|
return;
|
|
89
96
|
}
|
|
90
|
-
if (last?.stopReason !== 'error')
|
|
97
|
+
if (last?.stopReason !== 'error') {
|
|
98
|
+
if (last !== undefined)
|
|
99
|
+
clearProviderRetryEpisode(this.deps.nodeId);
|
|
91
100
|
return;
|
|
92
|
-
|
|
101
|
+
}
|
|
102
|
+
if (refreshAbort) {
|
|
103
|
+
clearProviderRetryEpisode(this.deps.nodeId);
|
|
93
104
|
return;
|
|
105
|
+
}
|
|
94
106
|
const raw = typeof last.errorMessage === 'string' ? last.errorMessage : 'engine error (no errorMessage recorded)';
|
|
95
107
|
// An abort landing between requests reaches pi's catch as a bare AbortError
|
|
96
108
|
// and settles as an error rather than `aborted`. Something asked that turn
|
|
97
109
|
// to stop, so there is no provider fault to record.
|
|
98
|
-
if (endedByAbort({ stopReason: 'error', errorMessage: raw }))
|
|
110
|
+
if (endedByAbort({ stopReason: 'error', errorMessage: raw })) {
|
|
111
|
+
clearProviderRetryEpisode(this.deps.nodeId);
|
|
99
112
|
return;
|
|
113
|
+
}
|
|
100
114
|
const uncompactedOverflow = isContextOverflow(last, settledSession.model?.contextWindow);
|
|
101
115
|
const classified = classify('pi→provider', last);
|
|
102
116
|
const coolingDeadline = extractCoolingDeadline(last);
|
|
103
|
-
if (uncompactedOverflow)
|
|
117
|
+
if (uncompactedOverflow) {
|
|
104
118
|
clearFault(this.deps.nodeId, { link: 'pi→provider' });
|
|
119
|
+
clearProviderRetryEpisode(this.deps.nodeId);
|
|
120
|
+
}
|
|
105
121
|
const faultInput = {
|
|
106
122
|
link: 'pi→provider', op: 'generation turn', kind: uncompactedOverflow ? 'context-overflow' : classified.kind,
|
|
107
123
|
retry: uncompactedOverflow
|
|
@@ -121,8 +137,10 @@ export class FaultRetry {
|
|
|
121
137
|
};
|
|
122
138
|
if (faultInput.retry.disposition === 'auto')
|
|
123
139
|
this.recordPendingProviderFault(settledSession, faultInput);
|
|
124
|
-
else
|
|
140
|
+
else {
|
|
141
|
+
clearProviderRetryEpisode(this.deps.nodeId);
|
|
125
142
|
recordFault(this.deps.nodeId, faultInput);
|
|
143
|
+
}
|
|
126
144
|
}
|
|
127
145
|
/** Startup re-drive reads only a durable pending episode. Revive deliberately
|
|
128
146
|
* clears the ordinary fault marker before this boundary, so the episode is
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export declare const HELD_DEFERRED_INBOX_PROMPT_JOIN: unique symbol;
|
|
2
|
+
export declare const BROKER_IDLE_TURN_START: unique symbol;
|
|
3
|
+
export type HeldDeferredInboxPromptJoin = (session: {
|
|
4
|
+
sendCustomMessage: (message: {
|
|
5
|
+
customType: string;
|
|
6
|
+
content: string;
|
|
7
|
+
display: boolean;
|
|
8
|
+
}) => Promise<void>;
|
|
9
|
+
}) => Promise<void>;
|
|
10
|
+
export declare function heldDeferredInboxPromptJoin(): HeldDeferredInboxPromptJoin | undefined;
|
|
11
|
+
/** Starts an inbox-delivered idle turn through the broker's admission gate. */
|
|
12
|
+
export type BrokerIdleTurnStart = (text: string) => void;
|
|
13
|
+
export declare function brokerIdleTurnStart(): BrokerIdleTurnStart | undefined;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
// The inbox watcher is a Pi extension loaded through Jiti, while the broker is
|
|
2
|
+
// native ESM. They therefore cannot share a module singleton. Process-global
|
|
3
|
+
// symbols carry their narrow bridge functions across that loader boundary.
|
|
4
|
+
export const HELD_DEFERRED_INBOX_PROMPT_JOIN = Symbol.for('@crouton-kit/crtr:held-deferred-inbox-prompt-join');
|
|
5
|
+
export const BROKER_IDLE_TURN_START = Symbol.for('@crouton-kit/crtr:broker-idle-turn-start');
|
|
6
|
+
export function heldDeferredInboxPromptJoin() {
|
|
7
|
+
const join = globalThis[HELD_DEFERRED_INBOX_PROMPT_JOIN];
|
|
8
|
+
return typeof join === 'function' ? join : undefined;
|
|
9
|
+
}
|
|
10
|
+
export function brokerIdleTurnStart() {
|
|
11
|
+
const start = globalThis[BROKER_IDLE_TURN_START];
|
|
12
|
+
return typeof start === 'function' ? start : undefined;
|
|
13
|
+
}
|
|
@@ -47,6 +47,7 @@ import { ToolGroupTracker } from './broker/tool-groups.js';
|
|
|
47
47
|
import { onNodeNamed } from './broker/node-named.js';
|
|
48
48
|
import { FaultRetry } from './broker/fault-retry.js';
|
|
49
49
|
import { createTurnAdmissionGate } from './broker/turn-admission.js';
|
|
50
|
+
import { BROKER_IDLE_TURN_START } from './broker/held-deferred-inbox.js';
|
|
50
51
|
import { EventProjection } from './broker/event-projection.js';
|
|
51
52
|
import { createFrameDispatchContext, handleFrame, } from './broker/frame-dispatch.js';
|
|
52
53
|
import { hydratePersistedAdvertisedCommandMessages, installAdvertisedCommandInvocationContract, } from './advertised-command-invocation.js';
|
|
@@ -327,6 +328,24 @@ export async function runBroker(nodeId, startupAt = process.hrtime.bigint(), sta
|
|
|
327
328
|
registry.broadcast({ type: 'model_changed', model: isUnknownModel(liveSession.model) ? undefined : liveSession.model, spec });
|
|
328
329
|
};
|
|
329
330
|
const turnAdmission = createTurnAdmissionGate();
|
|
331
|
+
// The inbox watcher is Jiti-loaded and cannot import this native module's
|
|
332
|
+
// gate. Its idle delivery therefore calls this narrow process-global bridge:
|
|
333
|
+
// acquire admission first, then re-read the live session before prompt() so
|
|
334
|
+
// a viewer turn that won the race is joined rather than double-started.
|
|
335
|
+
const startIdleInboxTurn = (text) => {
|
|
336
|
+
void turnAdmission.admit((release) => {
|
|
337
|
+
const liveSession = rebind.session();
|
|
338
|
+
return liveSession
|
|
339
|
+
.prompt(text, {
|
|
340
|
+
preflightResult: release,
|
|
341
|
+
...(liveSession.isStreaming ? { streamingBehavior: 'steer' } : {}),
|
|
342
|
+
})
|
|
343
|
+
.catch((error) => {
|
|
344
|
+
emitEvent({ level: 'error', event: 'broker.inbox.idle_prompt_failed', error });
|
|
345
|
+
});
|
|
346
|
+
});
|
|
347
|
+
};
|
|
348
|
+
globalThis[BROKER_IDLE_TURN_START] = startIdleInboxTurn;
|
|
330
349
|
faultRetry = new FaultRetry({
|
|
331
350
|
nodeId,
|
|
332
351
|
cfg,
|
|
@@ -49,12 +49,12 @@ export function fanDoctrineWake(fromId, subscribers, label, data, tier = 'normal
|
|
|
49
49
|
try {
|
|
50
50
|
const notice = { from: fromId, tier, kind: 'message', label, data };
|
|
51
51
|
if (sub.active) {
|
|
52
|
-
appendInbox(sub.node_id, notice);
|
|
52
|
+
const entry = appendInbox(sub.node_id, notice);
|
|
53
53
|
// A wake-capable notice consumes the subscriber's cancel-on-wake
|
|
54
54
|
// deadline: the thing it was waiting for arrived. `deferred` is the
|
|
55
55
|
// exception — it never wakes a node, riding the next natural cycle
|
|
56
56
|
// instead, so the deadline it was racing must survive.
|
|
57
|
-
if (tier !== 'deferred')
|
|
57
|
+
if (entry.tier !== 'deferred')
|
|
58
58
|
cancelCronsOnWake(sub.node_id);
|
|
59
59
|
}
|
|
60
60
|
else
|
|
@@ -229,11 +229,14 @@ export function clearFault(nodeId, opts) {
|
|
|
229
229
|
if (opts?.link !== undefined && current.link !== opts.link)
|
|
230
230
|
return false;
|
|
231
231
|
rmSync(faultPath(nodeId));
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
232
|
+
const episode = currentProviderRetryEpisode(nodeId);
|
|
233
|
+
// turn_end precedes agent_settled. An admitted retry keeps its durable
|
|
234
|
+
// coordinate while the broker classifies an error settlement; an invalidated
|
|
235
|
+
// coordinate also remains durable so replacement startup cannot replay it.
|
|
236
|
+
if (opts?.preserveProviderRetryEpisode !== true &&
|
|
237
|
+
episode?.fault.since === current.since &&
|
|
238
|
+
episode.state === 'pending')
|
|
239
|
+
rmSync(providerRetryPath(nodeId), { force: true });
|
|
237
240
|
const operationId = resolveOperationId();
|
|
238
241
|
emitEvent({
|
|
239
242
|
level: 'info',
|
|
@@ -5,8 +5,8 @@
|
|
|
5
5
|
//
|
|
6
6
|
// 1. Finalize — push the agent's last surfaced message as a `final` report so
|
|
7
7
|
// every subscriber/manager waiting on it is unblocked, and mark it done.
|
|
8
|
-
// 2.
|
|
9
|
-
// 3.
|
|
8
|
+
// 2. Recycle — boot a fresh resident broker root and prove it accepts viewers.
|
|
9
|
+
// 3. Close — tear the finalized broker down, then re-attach the pane to the replacement.
|
|
10
10
|
//
|
|
11
11
|
// NOT to be confused with `node lifecycle demote` (flip-to-terminal IN PLACE, which keeps
|
|
12
12
|
// the agent focused and running): recycle ENDS this agent and boots a brand-new
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
// message) — falling back to a short note when it never reported.
|
|
18
18
|
import { readdirSync, readFileSync, statSync } from 'node:fs';
|
|
19
19
|
import { join } from 'node:path';
|
|
20
|
-
import { getNode, setPresence, setFocusOccupant, fullName } from '../canvas/index.js';
|
|
20
|
+
import { getNode, setPresence, setFocusOccupant, fullName, recordLaunch, recordPid } from '../canvas/index.js';
|
|
21
21
|
import { reportsDir } from '../canvas/paths.js';
|
|
22
22
|
import { pushFinal } from '../feed/feed.js';
|
|
23
23
|
import { spawnNode, rootOfSpine } from './nodes.js';
|
|
@@ -25,6 +25,7 @@ import { buildLaunchSpecAsync, buildPiArgv } from './launch.js';
|
|
|
25
25
|
import { focusOf, respawnPaneSync, setPaneOption } from './placement.js';
|
|
26
26
|
import { waitForBrokerViewSocket, viewerSplitEnv } from './placement-tmux.js';
|
|
27
27
|
import { headlessBrokerHost } from './host.js';
|
|
28
|
+
import { transition } from './lifecycle.js';
|
|
28
29
|
import { ensureDaemon } from '../../daemon/manage.js';
|
|
29
30
|
/** The agent's most recent surfaced message: the newest reports/*.md body with
|
|
30
31
|
* its YAML frontmatter stripped. Empty string when the node never reported. */
|
|
@@ -74,23 +75,11 @@ export async function recycleNode(nodeId, callerPane) {
|
|
|
74
75
|
finalized = true;
|
|
75
76
|
}
|
|
76
77
|
catch { /* recycle the pane even if the report failed */ }
|
|
77
|
-
//
|
|
78
|
-
//
|
|
79
|
-
//
|
|
80
|
-
//
|
|
81
|
-
// above (crash-safe order: the daemon won't revive a done node).
|
|
82
|
-
try {
|
|
83
|
-
headlessBrokerHost.teardown(nodeId);
|
|
84
|
-
}
|
|
85
|
-
catch { /* best-effort */ }
|
|
86
|
-
// Capture M's focus viewport (if any) BEFORE nulling — the fresh root inherits
|
|
87
|
-
// it (the SAME focus row + pane). The demoted node no longer holds a pane: it is
|
|
88
|
-
// being reclaimed.
|
|
78
|
+
// Capture M's focus viewport (if any) before booting its replacement. The
|
|
79
|
+
// old broker remains live until the new broker proves it can accept a viewer;
|
|
80
|
+
// a failed replacement must not strand this finished conversation without an
|
|
81
|
+
// engine.
|
|
89
82
|
const f = focusOf(nodeId);
|
|
90
|
-
try {
|
|
91
|
-
setPresence(nodeId, { pane: null, window: null, tmux_session: null });
|
|
92
|
-
}
|
|
93
|
-
catch { /* best-effort */ }
|
|
94
83
|
// 2 + 3. Recycle — boot a fresh resident BROKER root for the SAME pane: the
|
|
95
84
|
// viewer pane re-attaches to the fresh broker (broker-is-the-host — the viewer
|
|
96
85
|
// pane stays a viewer, never becomes an engine pane).
|
|
@@ -113,36 +102,69 @@ export async function recycleNode(nodeId, callerPane) {
|
|
|
113
102
|
profile_id: meta.profile_id,
|
|
114
103
|
launch,
|
|
115
104
|
});
|
|
116
|
-
|
|
117
|
-
|
|
105
|
+
const fresh = getNode(root.node_id);
|
|
106
|
+
const inv = buildPiArgv(fresh);
|
|
107
|
+
// CRTR_SUBTREE groups the fresh root's subtree; FRONT_DOOR is set by the broker
|
|
108
|
+
// host itself, so it is not added here.
|
|
109
|
+
inv.env = { ...inv.env, CRTR_SUBTREE: rootOfSpine(root.node_id) };
|
|
110
|
+
if (!(await launchRecycledBroker(fresh, inv))) {
|
|
111
|
+
return { recycled: false, finalized, newRoot: root.node_id, delivered };
|
|
112
|
+
}
|
|
113
|
+
// COMMIT after replacement readiness: the node's pane is a VIEWER, so tear
|
|
114
|
+
// down the old detached broker only now that the fresh broker can serve it.
|
|
115
|
+
try {
|
|
116
|
+
headlessBrokerHost.teardown(nodeId);
|
|
117
|
+
}
|
|
118
|
+
catch { /* best-effort */ }
|
|
119
|
+
try {
|
|
120
|
+
setPresence(nodeId, { pane: null, window: null, tmux_session: null });
|
|
121
|
+
}
|
|
122
|
+
catch { /* best-effort */ }
|
|
118
123
|
if (f !== null) {
|
|
119
124
|
try {
|
|
120
125
|
setFocusOccupant(f.focus_id, root.node_id);
|
|
121
126
|
}
|
|
122
127
|
catch { /* best-effort */ }
|
|
123
128
|
}
|
|
124
|
-
const
|
|
125
|
-
const inv = buildPiArgv(fresh);
|
|
126
|
-
// CRTR_SUBTREE groups the fresh root's subtree; FRONT_DOOR is set by the broker
|
|
127
|
-
// host itself, so it is not added here.
|
|
128
|
-
inv.env = { ...inv.env, CRTR_SUBTREE: rootOfSpine(root.node_id) };
|
|
129
|
-
const ok = await recycleBrokerViewer(fresh, pane, inv);
|
|
129
|
+
const ok = respawnRecycledViewer(fresh, pane);
|
|
130
130
|
return { recycled: ok, finalized, newRoot: root.node_id, delivered };
|
|
131
131
|
}
|
|
132
|
-
/**
|
|
133
|
-
*
|
|
134
|
-
*
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
132
|
+
/** Launch a recycled root and prove its viewer socket before recycling the old
|
|
133
|
+
* broker. This direct birth path writes the same launch/pid coordinates as
|
|
134
|
+
* reviveNode, and a failed readiness probe terminalizes and tears it down. */
|
|
135
|
+
async function launchRecycledBroker(fresh, inv) {
|
|
136
|
+
let placed;
|
|
137
|
+
try {
|
|
138
|
+
placed = headlessBrokerHost.launch(fresh.node_id, inv, { cwd: fresh.cwd, name: fullName(fresh), resuming: false });
|
|
139
|
+
if (placed.pid === null)
|
|
140
|
+
throw new Error('broker host returned no pid');
|
|
141
|
+
recordLaunch(fresh.node_id, new Date().toISOString());
|
|
142
|
+
recordPid(fresh.node_id, placed.pid);
|
|
143
|
+
if (await waitForBrokerViewSocket(fresh.node_id, placed.exited))
|
|
144
|
+
return true;
|
|
145
|
+
}
|
|
146
|
+
catch {
|
|
147
|
+
// The replacement never became usable; the finished root's broker remains
|
|
148
|
+
// intact and this half-born row cannot remain active without a fleet handle.
|
|
149
|
+
}
|
|
150
|
+
try {
|
|
151
|
+
transitionReplacementToDead(fresh.node_id);
|
|
152
|
+
}
|
|
153
|
+
catch { /* best-effort */ }
|
|
154
|
+
try {
|
|
155
|
+
headlessBrokerHost.teardown(fresh.node_id);
|
|
156
|
+
}
|
|
157
|
+
catch { /* best-effort */ }
|
|
158
|
+
return false;
|
|
159
|
+
}
|
|
160
|
+
function transitionReplacementToDead(nodeId) {
|
|
161
|
+
// spawnNode births active rows, so crash is the authoritative failed-launch
|
|
162
|
+
// outcome whether the host returned no pid or exited before readiness.
|
|
163
|
+
const replacement = getNode(nodeId);
|
|
164
|
+
if (replacement?.status === 'active' || replacement?.status === 'idle')
|
|
165
|
+
transition(nodeId, 'crash');
|
|
166
|
+
}
|
|
167
|
+
function respawnRecycledViewer(fresh, pane) {
|
|
146
168
|
// Clear the finalized node's stale `@crtr_node` tag before respawn so the tag
|
|
147
169
|
// never names a done node during the gap before the new `crtr surface attach` re-tags
|
|
148
170
|
// on connect. Node ids are shell-safe identifiers; no quoting needed.
|
|
@@ -25,7 +25,7 @@ export interface RelaunchDeps {
|
|
|
25
25
|
}>) => boolean | Promise<boolean>;
|
|
26
26
|
/** Re-exec the viewer pane onto the new node. Default: respawnPaneSync. */
|
|
27
27
|
respawnViewer?: typeof respawnPaneSync;
|
|
28
|
-
/** Tear
|
|
28
|
+
/** Tear down a broker after a failed replacement or a committed old-root replacement. */
|
|
29
29
|
teardownBroker?: typeof headlessBrokerHost.teardown;
|
|
30
30
|
}
|
|
31
31
|
export interface RelaunchRootResult {
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
// reserved for finish).
|
|
21
21
|
//
|
|
22
22
|
// Best-effort throughout: a tmux/fs failure on one node never aborts the reap.
|
|
23
|
-
import { getNode, updateNode, fullName, closeFocusRow, view, } from '../canvas/index.js';
|
|
23
|
+
import { getNode, updateNode, recordLaunch, recordPid, fullName, closeFocusRow, view, } from '../canvas/index.js';
|
|
24
24
|
import { transition } from './lifecycle.js';
|
|
25
25
|
import { headlessBrokerHost, requestBrokerTeardown } from './host.js';
|
|
26
26
|
import { tearDownNode, focusOf, registerViewerFocus, respawnPaneSync, windowOfPane, renameWindow, } from './placement.js';
|
|
@@ -129,7 +129,30 @@ export async function relaunchRoot(oldId, deps = {}) {
|
|
|
129
129
|
transition(newMeta.node_id, 'crash');
|
|
130
130
|
return null;
|
|
131
131
|
}
|
|
132
|
-
|
|
132
|
+
// A replacement is a managed broker just like a revive: the fleet owns its
|
|
133
|
+
// real exit, while the row gets the matching launch and pid coordinates before
|
|
134
|
+
// readiness can observe an early exit.
|
|
135
|
+
recordLaunch(newMeta.node_id, new Date().toISOString());
|
|
136
|
+
recordPid(newMeta.node_id, placed.pid);
|
|
137
|
+
let ready = false;
|
|
138
|
+
try {
|
|
139
|
+
ready = await waitForViewSocket(newMeta.node_id, placed.exited);
|
|
140
|
+
}
|
|
141
|
+
catch {
|
|
142
|
+
// A readiness probe error means the replacement is not proven usable.
|
|
143
|
+
}
|
|
144
|
+
if (!ready) {
|
|
145
|
+
// Do not let a broker that can still bind later sit behind a dead row, and
|
|
146
|
+
// do not commit any old-root teardown without a usable replacement.
|
|
147
|
+
const replacement = getNode(newMeta.node_id);
|
|
148
|
+
if (replacement?.status === 'active' || replacement?.status === 'idle')
|
|
149
|
+
transition(newMeta.node_id, 'crash');
|
|
150
|
+
try {
|
|
151
|
+
teardownBroker(newMeta.node_id);
|
|
152
|
+
}
|
|
153
|
+
catch { /* best-effort */ }
|
|
154
|
+
return null;
|
|
155
|
+
}
|
|
133
156
|
// --- COMMIT: park + reap the old root, re-point the viewer, kill the old
|
|
134
157
|
// broker. Past this point the new node is the live root. ---
|
|
135
158
|
reapDescendants(oldId); // old workers → canceled + torn down
|
|
@@ -327,7 +327,7 @@ async function handleMessage(ctx) {
|
|
|
327
327
|
const entry = deliver();
|
|
328
328
|
// Deferred mail normally waits for a natural cycle, but a frozen row already
|
|
329
329
|
// owes a thaw; consume its deadline when that durable future wake arrives.
|
|
330
|
-
if (tier !== 'deferred' || meta.frozen_at !== null)
|
|
330
|
+
if (entry.tier !== 'deferred' || meta.frozen_at !== null)
|
|
331
331
|
cancelCronsOnWake(id);
|
|
332
332
|
// A wake-capable tier revives a dormant target so its inbox-watcher delivers
|
|
333
333
|
// this; deferred never wakes — it rides the target's next natural cycle. The
|
|
@@ -33,7 +33,7 @@ function parsePushBody(body) {
|
|
|
33
33
|
if (deliveryTier !== undefined && deliveryTier !== 'deferred' && deliveryTier !== 'normal' && deliveryTier !== 'urgent') {
|
|
34
34
|
throw usage(`invalid delivery_tier: ${String(deliveryTier)} (expected deferred|normal|urgent)`);
|
|
35
35
|
}
|
|
36
|
-
if (tier
|
|
36
|
+
if (tier !== 'update' && deliveryTier !== undefined) {
|
|
37
37
|
throw usage('delivery_tier is only valid for update reports');
|
|
38
38
|
}
|
|
39
39
|
if (typeof text !== 'string' || text.trim() === '') {
|
|
@@ -31,7 +31,7 @@ function deliverNodeSink(c, target, stdout) {
|
|
|
31
31
|
label: `⏰ cron ${c.name}`,
|
|
32
32
|
data: { body: stdout },
|
|
33
33
|
});
|
|
34
|
-
if (
|
|
34
|
+
if (entry.tier !== 'deferred' || meta.frozen_at !== null)
|
|
35
35
|
cancelCronsOnWake(target);
|
|
36
36
|
// The APPENDED tier decides the wake, not the cron's requested one: a deferred
|
|
37
37
|
// cron aimed at a terminal node was raised out of `deferred` by the append.
|
|
@@ -59,7 +59,7 @@ export async function deliverNodeMessage(args) {
|
|
|
59
59
|
label: args.label,
|
|
60
60
|
data: { ...args.data, body: args.body },
|
|
61
61
|
}));
|
|
62
|
-
if (
|
|
62
|
+
if (entry.tier !== 'deferred' || target.frozen_at !== null)
|
|
63
63
|
cancelCronsOnWake(args.node_id);
|
|
64
64
|
// The APPENDED entry's tier decides the wake: a quiet send to a terminal
|
|
65
65
|
// target was raised out of `deferred` by the append, and must be delivered.
|