@north-light/crouter 0.3.249 → 0.3.251
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/api/__tests__/integration/client.test.js +28 -7
- package/dist/api/client.d.ts +1 -1
- package/dist/api/client.js +2 -2
- package/dist/api/dto/lifecycle.d.ts +3 -0
- package/dist/api/dto/messages.d.ts +4 -0
- package/dist/api/dto/nodes.d.ts +4 -0
- package/dist/api/dto/reports.d.ts +7 -2
- package/dist/builtin-memory/00-runtime-base/01-escalation.md +1 -1
- package/dist/builtin-memory/05-kinds/review/01-orchestrator.md +3 -0
- package/dist/builtin-memory/internal/nodes-and-canvas.md +1 -1
- package/dist/builtin-pi-packages/pi-crtr-extensions/README.md +0 -1
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js +4 -3
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +4 -3
- package/dist/clients/attach/session/chrome-refresh.d.ts +3 -0
- package/dist/clients/attach/session/chrome-refresh.js +1 -0
- package/dist/clients/attach/viewer.js +532 -532
- package/dist/clients/inbox/__tests__/integration/stale-row-activation.test.d.ts +1 -0
- package/dist/clients/inbox/__tests__/integration/stale-row-activation.test.js +53 -0
- package/dist/clients/inbox/controller.js +32 -5
- package/dist/clients/inbox/tui/ansi.d.ts +3 -1
- package/dist/clients/inbox/tui/ansi.js +14 -10
- package/dist/commands/__tests__/api-client-cold-start-diagnostic.test.js +7 -7
- package/dist/commands/api-client.d.ts +2 -2
- package/dist/commands/api-client.js +5 -5
- package/dist/commands/canvas-browse.js +1 -1
- package/dist/commands/node/create.js +3 -1
- package/dist/commands/node/inspect.js +1 -0
- package/dist/commands/node/lifecycle.js +1 -1
- package/dist/commands/node/message.js +6 -4
- package/dist/commands/node/wait.js +1 -1
- package/dist/commands/push.js +36 -37
- package/dist/commands/sys/daemon.js +21 -8
- package/dist/commands/sys/setup-core.js +2 -2
- package/dist/core/__tests__/broker-stream-watchdog-floor.test.d.ts +1 -0
- package/dist/core/__tests__/broker-stream-watchdog-floor.test.js +84 -0
- package/dist/core/__tests__/broker-turn-admission.test.d.ts +1 -0
- package/dist/core/__tests__/broker-turn-admission.test.js +44 -0
- package/dist/core/__tests__/canvas-inbox-watcher-hold.test.js +100 -1
- package/dist/core/__tests__/child-death-wake.test.js +11 -7
- package/dist/core/__tests__/collectible.test.d.ts +1 -0
- package/dist/core/__tests__/collectible.test.js +88 -0
- package/dist/core/__tests__/context-intro.test.js +4 -41
- package/dist/core/__tests__/daemon-boot.test.js +30 -14
- package/dist/core/__tests__/daemon-wedge.test.js +25 -19
- package/dist/core/__tests__/fixtures/fake-engine.js +20 -2
- package/dist/core/__tests__/human-deliver.test.js +5 -5
- package/dist/core/__tests__/integration/deferred-no-wake.test.js +65 -4
- package/dist/core/__tests__/integration/human-deliver-e2e.test.js +3 -3
- package/dist/core/__tests__/integration/revive.test.js +48 -0
- package/dist/core/__tests__/integration/spawn-root.test.js +36 -1
- package/dist/core/__tests__/integration/worktree-land.test.js +82 -3
- package/dist/core/__tests__/integration/worktree-reap.test.js +44 -6
- package/dist/core/__tests__/migration.test.js +30 -0
- package/dist/core/__tests__/passive-subscription.test.js +39 -1
- package/dist/core/__tests__/prune.test.js +12 -3
- package/dist/core/__tests__/relaunch-root.test.js +1 -1
- package/dist/core/__tests__/respawn-throttle.test.js +16 -146
- package/dist/core/__tests__/seam/broker-crash-teardown.test.js +23 -11
- package/dist/core/__tests__/seam/broker-provider-retry.test.js +81 -2
- package/dist/core/__tests__/seam/dormancy-release.test.js +1 -1
- package/dist/core/__tests__/seam/held-deferred-human-prompt.test.d.ts +1 -0
- package/dist/core/__tests__/seam/held-deferred-human-prompt.test.js +63 -0
- package/dist/core/__tests__/seam/self-close-teardown.test.d.ts +1 -0
- package/dist/core/__tests__/seam/self-close-teardown.test.js +49 -0
- package/dist/core/__tests__/seam/yield-refresh-transaction.test.js +18 -10
- package/dist/core/canvas/canvas.d.ts +9 -15
- package/dist/core/canvas/canvas.js +41 -47
- package/dist/core/canvas/crons.d.ts +5 -8
- package/dist/core/canvas/crons.js +5 -8
- package/dist/core/canvas/history.d.ts +5 -0
- package/dist/core/canvas/history.js +12 -1
- package/dist/core/canvas/migrations.js +58 -1
- package/dist/core/canvas/pid.d.ts +1 -1
- package/dist/core/canvas/pid.js +5 -2
- package/dist/core/canvas/types.d.ts +4 -0
- package/dist/core/feed/feed.d.ts +4 -0
- package/dist/core/feed/feed.js +13 -8
- package/dist/core/feed/inbox.d.ts +2 -1
- package/dist/core/feed/inbox.js +22 -3
- package/dist/core/git.d.ts +3 -0
- package/dist/core/git.js +3 -1
- package/dist/core/human/feedback-companion.js +1 -0
- package/dist/core/preview-registry.js +0 -1
- package/dist/core/review/companion.js +1 -0
- package/dist/core/review/realize.js +1 -0
- package/dist/core/runtime/broker/auth-reload.d.ts +2 -0
- package/dist/core/runtime/broker/auth-reload.js +9 -3
- package/dist/core/runtime/broker/engine-drive.d.ts +2 -0
- package/dist/core/runtime/broker/engine-drive.js +39 -23
- package/dist/core/runtime/broker/event-projection.d.ts +4 -0
- package/dist/core/runtime/broker/event-projection.js +43 -23
- package/dist/core/runtime/broker/fault-retry.d.ts +9 -0
- package/dist/core/runtime/broker/fault-retry.js +94 -8
- package/dist/core/runtime/broker/frame-dispatch.d.ts +2 -0
- package/dist/core/runtime/broker/frame-dispatch.js +4 -1
- package/dist/core/runtime/broker/held-deferred-inbox.d.ts +9 -0
- package/dist/core/runtime/broker/held-deferred-inbox.js +8 -0
- package/dist/core/runtime/broker/inbox.js +4 -0
- package/dist/core/runtime/broker/rebind.js +18 -0
- package/dist/core/runtime/broker/turn-admission.d.ts +11 -0
- package/dist/core/runtime/broker/turn-admission.js +31 -0
- package/dist/core/runtime/broker/turn-ignition.d.ts +2 -0
- package/dist/core/runtime/broker/turn-ignition.js +25 -15
- package/dist/core/runtime/broker.js +21 -2
- package/dist/core/runtime/close.d.ts +11 -9
- package/dist/core/runtime/close.js +23 -19
- package/dist/core/runtime/fault.d.ts +21 -0
- package/dist/core/runtime/fault.js +129 -19
- package/dist/core/runtime/fleet.d.ts +0 -6
- package/dist/core/runtime/host.d.ts +7 -1
- package/dist/core/runtime/host.js +20 -2
- package/dist/core/runtime/nodes.d.ts +5 -1
- package/dist/core/runtime/nodes.js +24 -1
- package/dist/core/runtime/placement.d.ts +6 -6
- package/dist/core/runtime/placement.js +15 -8
- package/dist/core/runtime/reset.js +2 -2
- package/dist/core/runtime/revive.js +40 -4
- package/dist/core/runtime/spawn.d.ts +3 -0
- package/dist/core/runtime/spawn.js +1 -0
- package/dist/core/runtime/warm-pool.d.ts +4 -0
- package/dist/core/runtime/warm-pool.js +2 -0
- package/dist/core/worktree.d.ts +4 -3
- package/dist/core/worktree.js +73 -27
- package/dist/daemon/__tests__/integration/api-startup-readiness.test.d.ts +1 -0
- package/dist/daemon/__tests__/integration/api-startup-readiness.test.js +81 -0
- package/dist/daemon/api/__tests__/reopen-delivery.test.js +38 -0
- package/dist/daemon/api/__tests__/seam/api-server.test.js +1 -0
- package/dist/daemon/api/__tests__/seam/leaf-api-parity.test.js +13 -2
- package/dist/daemon/api/handlers/bash-jobs.js +2 -0
- package/dist/daemon/api/handlers/broker-ops.js +23 -2
- package/dist/daemon/api/handlers/human.js +1 -1
- package/dist/daemon/api/handlers/messages.js +11 -2
- package/dist/daemon/api/handlers/nodes.js +13 -1
- package/dist/daemon/api/handlers/reports.js +14 -6
- package/dist/daemon/api/map.js +1 -0
- package/dist/daemon/api/server.d.ts +9 -12
- package/dist/daemon/api/server.js +30 -19
- package/dist/daemon/companion-retire.js +3 -9
- package/dist/daemon/cron/sinks.js +7 -2
- package/dist/daemon/crtrd.js +4 -1
- package/dist/daemon/fleet.d.ts +2 -16
- package/dist/daemon/fleet.js +63 -150
- package/dist/daemon/human/finish.js +2 -0
- package/dist/daemon/manage.d.ts +11 -0
- package/dist/daemon/manage.js +32 -3
- package/dist/daemon/messaging/node-message.js +7 -1
- package/dist/daemon/reconcilers/bash-deadline.js +2 -0
- package/dist/daemon/reconcilers/broker-supervision.d.ts +1 -0
- package/dist/daemon/reconcilers/broker-supervision.js +22 -9
- package/dist/daemon/reconcilers/live-obligation.d.ts +16 -6
- package/dist/daemon/reconcilers/live-obligation.js +18 -6
- package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.d.ts +15 -4
- package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.js +34 -3
- package/dist/daemon/reconcilers/node-lifecycle/terminating.d.ts +4 -1
- package/dist/daemon/reconcilers/node-lifecycle/terminating.js +11 -2
- package/dist/daemon/reconcilers/node-lifecycle/tick.js +14 -7
- package/dist/daemon/reconcilers/storage-maintenance.d.ts +2 -2
- package/dist/daemon/reconcilers/storage-maintenance.js +3 -3
- package/dist/pi-extensions/__tests__/canvas-stophook-agentend.test.js +4 -4
- package/dist/pi-extensions/canvas-context-intro.d.ts +3 -34
- package/dist/pi-extensions/canvas-context-intro.js +4 -76
- package/dist/pi-extensions/canvas-inbox-watcher.js +105 -4
- package/dist/pi-extensions/canvas-review-boundary.d.ts +3 -24
- package/dist/pi-extensions/canvas-review-boundary.js +4 -29
- package/dist/pi-extensions/canvas-stophook.js +1 -6
- package/dist/shared/birth-announcement.d.ts +12 -0
- package/dist/shared/birth-announcement.js +25 -0
- package/dist/shared/generated-context.d.ts +8 -4
- package/dist/shared/generated-context.js +15 -9
- package/package.json +4 -4
- package/runtime.lock.json +2 -2
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/statusline.ts +0 -254
- package/dist/pi-extensions/truncate.d.ts +0 -5
- package/dist/pi-extensions/truncate.js +0 -14
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
// `db.ts` itself keeps only connection management (`openDb`/`withCanvasWrite`
|
|
22
22
|
// /`closeDb`) and re-exports `migrate`/`MIGRATIONS` from here so external
|
|
23
23
|
// imports are unaffected by this split.
|
|
24
|
-
import { existsSync, readFileSync, readdirSync } from 'node:fs';
|
|
24
|
+
import { existsSync, readFileSync, readdirSync, realpathSync } from 'node:fs';
|
|
25
25
|
import { basename, join, resolve } from 'node:path';
|
|
26
26
|
import { crtrHome, isSafeNodeId, nodesRoot, nodeMetaPath } from './paths.js';
|
|
27
27
|
import { STATE_MIGRATIONS } from '../../migrations/registry.js';
|
|
@@ -1215,6 +1215,62 @@ function addNodeSessionFileIndex(db) {
|
|
|
1215
1215
|
}
|
|
1216
1216
|
db.exec('CREATE INDEX IF NOT EXISTS idx_nodes_pi_session_file ON nodes(pi_session_file);');
|
|
1217
1217
|
}
|
|
1218
|
+
/** v36 — one Pi transcript may have exactly one node owner. Normalize every
|
|
1219
|
+
* stored path first (including meta), then retain the earliest created row
|
|
1220
|
+
* (node_id breaks equal timestamps) and clear both durable session coordinates
|
|
1221
|
+
* from every later claimant before creating the UNIQUE index. */
|
|
1222
|
+
function makeNodeSessionFileUnique(db) {
|
|
1223
|
+
const canonical = (sessionFile) => {
|
|
1224
|
+
const absolute = resolve(sessionFile);
|
|
1225
|
+
return existsSync(absolute) ? realpathSync(absolute) : absolute;
|
|
1226
|
+
};
|
|
1227
|
+
const rows = db.prepare(`
|
|
1228
|
+
SELECT node_id, pi_session_file
|
|
1229
|
+
FROM nodes
|
|
1230
|
+
WHERE pi_session_file IS NOT NULL
|
|
1231
|
+
ORDER BY created ASC, node_id ASC
|
|
1232
|
+
`).all();
|
|
1233
|
+
const setFile = db.prepare('UPDATE nodes SET pi_session_file = ? WHERE node_id = ?');
|
|
1234
|
+
for (const row of rows) {
|
|
1235
|
+
const sessionFile = canonical(row.pi_session_file);
|
|
1236
|
+
if (sessionFile === row.pi_session_file)
|
|
1237
|
+
continue;
|
|
1238
|
+
// Identity is authoritative, so normalize both stores before grouping.
|
|
1239
|
+
updateJsonFileDurably(nodeMetaPath(row.node_id), (meta) => {
|
|
1240
|
+
meta['pi_session_file'] = sessionFile;
|
|
1241
|
+
return true;
|
|
1242
|
+
});
|
|
1243
|
+
setFile.run(sessionFile, row.node_id);
|
|
1244
|
+
}
|
|
1245
|
+
const duplicates = db.prepare(`
|
|
1246
|
+
SELECT pi_session_file
|
|
1247
|
+
FROM nodes
|
|
1248
|
+
WHERE pi_session_file IS NOT NULL
|
|
1249
|
+
GROUP BY pi_session_file
|
|
1250
|
+
HAVING COUNT(*) > 1
|
|
1251
|
+
`).all();
|
|
1252
|
+
const clearRow = db.prepare('UPDATE nodes SET pi_session_file = NULL WHERE node_id = ?');
|
|
1253
|
+
for (const { pi_session_file } of duplicates) {
|
|
1254
|
+
const owners = db.prepare(`
|
|
1255
|
+
SELECT node_id
|
|
1256
|
+
FROM nodes
|
|
1257
|
+
WHERE pi_session_file = ?
|
|
1258
|
+
ORDER BY created ASC, node_id ASC
|
|
1259
|
+
`).all(pi_session_file);
|
|
1260
|
+
for (const { node_id } of owners.slice(1)) {
|
|
1261
|
+
// Clear both durable coordinates so a later reindex cannot restore the
|
|
1262
|
+
// duplicate projection.
|
|
1263
|
+
updateJsonFileDurably(nodeMetaPath(node_id), (meta) => {
|
|
1264
|
+
meta['pi_session_id'] = null;
|
|
1265
|
+
meta['pi_session_file'] = null;
|
|
1266
|
+
return true;
|
|
1267
|
+
});
|
|
1268
|
+
clearRow.run(node_id);
|
|
1269
|
+
}
|
|
1270
|
+
}
|
|
1271
|
+
db.exec('DROP INDEX IF EXISTS idx_nodes_pi_session_file;');
|
|
1272
|
+
db.exec('CREATE UNIQUE INDEX IF NOT EXISTS idx_nodes_pi_session_file ON nodes(pi_session_file);');
|
|
1273
|
+
}
|
|
1218
1274
|
/** v35 — durable node lifecycle execution state and terminal cause. */
|
|
1219
1275
|
function addNodeLifecycleColumns(db) {
|
|
1220
1276
|
const columns = nodeColumns(db);
|
|
@@ -1283,6 +1339,7 @@ export const MIGRATIONS = [
|
|
|
1283
1339
|
/* v33 */ addHumanActionDeliveries,
|
|
1284
1340
|
/* v34 */ addNodeSessionFileIndex,
|
|
1285
1341
|
/* v35 */ addNodeLifecycleColumns,
|
|
1342
|
+
/* v36 */ makeNodeSessionFileUnique,
|
|
1286
1343
|
...JOURNALED_STATE_MIGRATIONS,
|
|
1287
1344
|
];
|
|
1288
1345
|
/** Migration indexes that manage their OWN transaction and therefore must not
|
|
@@ -252,7 +252,7 @@ export interface TeardownSnapshot {
|
|
|
252
252
|
* before this snapshot was taken. Callers must skip ALL signaling when true. */
|
|
253
253
|
reused: boolean;
|
|
254
254
|
}
|
|
255
|
-
export declare function captureTeardownSnapshot(rootPid: number, expectedIdentity: string | null): TeardownSnapshot;
|
|
255
|
+
export declare function captureTeardownSnapshot(rootPid: number, expectedIdentity: string | null, excludePid?: number): TeardownSnapshot;
|
|
256
256
|
/** Best-effort signal against every pid in an ALREADY-CAPTURED tree snapshot
|
|
257
257
|
* (typically `[rootPid, ...descendantPids(rootPid)]`, taken ONCE by the
|
|
258
258
|
* caller before sending any signal — see the doc on `isProcessTreeAlive` for
|
package/dist/core/canvas/pid.js
CHANGED
|
@@ -500,7 +500,7 @@ export function capturePidCommand(pid) {
|
|
|
500
500
|
return null;
|
|
501
501
|
}
|
|
502
502
|
}
|
|
503
|
-
export function captureTeardownSnapshot(rootPid, expectedIdentity) {
|
|
503
|
+
export function captureTeardownSnapshot(rootPid, expectedIdentity, excludePid) {
|
|
504
504
|
if (!Number.isInteger(rootPid) || rootPid <= 0)
|
|
505
505
|
return { tree: [], identities: null, reused: false };
|
|
506
506
|
const table = captureLivenessSnapshot();
|
|
@@ -512,7 +512,10 @@ export function captureTeardownSnapshot(rootPid, expectedIdentity) {
|
|
|
512
512
|
!identitiesMatch(currentRootIdentity, expectedIdentity);
|
|
513
513
|
if (reused)
|
|
514
514
|
return { tree: [], identities: null, reused: true };
|
|
515
|
-
const
|
|
515
|
+
const excluded = excludePid !== undefined && Number.isInteger(excludePid) && excludePid > 0
|
|
516
|
+
? new Set([excludePid, ...descendantPidsFrom(excludePid, table.childrenOf)])
|
|
517
|
+
: undefined;
|
|
518
|
+
const tree = [rootPid, ...descendantPidsFrom(rootPid, table.childrenOf)].filter((pid) => !excluded?.has(pid));
|
|
516
519
|
const identities = new Map();
|
|
517
520
|
for (const pid of tree) {
|
|
518
521
|
const id = table.processes.get(pid)?.identity;
|
|
@@ -143,6 +143,10 @@ export interface NodeIdentity {
|
|
|
143
143
|
* `parent` so an INDEPENDENT root (parent=null) still records its lineage.
|
|
144
144
|
* Audit only; null for a user-opened root. Defaults to `parent` for a child. */
|
|
145
145
|
spawned_by?: string | null;
|
|
146
|
+
/** Actor that created this node; null when an external process did. Unlike
|
|
147
|
+
* `parent` (manager) and `spawned_by` (provenance edge), this determines
|
|
148
|
+
* whether birth is announced to the manager. */
|
|
149
|
+
creator?: string | null;
|
|
146
150
|
/** Raw `--fork-from` operand retained for identity framing and audit. */
|
|
147
151
|
fork_from?: string | null;
|
|
148
152
|
/** Resolved absolute source session file used for every pre-bind fork retry. */
|
package/dist/core/feed/feed.d.ts
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import type { InboxDisposition } from './inbox.js';
|
|
2
2
|
export type PushKind = 'update' | 'urgent' | 'final';
|
|
3
|
+
export type PushDeliveryTier = 'deferred' | 'normal' | 'urgent';
|
|
3
4
|
export interface PushOpts {
|
|
4
5
|
kind: PushKind;
|
|
6
|
+
/** Update-only subscriber delivery urgency; report kind remains `kind`. */
|
|
7
|
+
deliveryTier?: PushDeliveryTier;
|
|
5
8
|
body: string;
|
|
6
9
|
from?: string;
|
|
7
10
|
/** Attribution snapshotted by the human-ticket settlement paths; absent means an ordinary report. */
|
|
@@ -25,6 +28,7 @@ export declare class FinalDeliveryError extends Error {
|
|
|
25
28
|
export declare function push(nodeId: string, opts: PushOpts): Promise<PushResult>;
|
|
26
29
|
export declare function pushUpdate(nodeId: string, body: string, opts?: {
|
|
27
30
|
from?: string;
|
|
31
|
+
deliveryTier?: PushDeliveryTier;
|
|
28
32
|
}): Promise<PushResult>;
|
|
29
33
|
export declare function pushUrgent(nodeId: string, body: string, opts?: {
|
|
30
34
|
from?: string;
|
package/dist/core/feed/feed.js
CHANGED
|
@@ -51,18 +51,21 @@ function firstLine(text, maxLen = 80) {
|
|
|
51
51
|
const line = text.split('\n')[0] ?? '';
|
|
52
52
|
return line.length > maxLen ? `${line.slice(0, maxLen - 1)}…` : line;
|
|
53
53
|
}
|
|
54
|
-
|
|
55
|
-
|
|
54
|
+
/** Requested update delivery urgency wins; legacy report-kind mapping is the fallback. */
|
|
55
|
+
function tierFor(kind, requestedTier) {
|
|
56
|
+
return requestedTier ?? (kind === 'urgent' ? 'urgent' : 'normal');
|
|
56
57
|
}
|
|
57
|
-
function deliver(subscribers, from, fromName, kind, reportRef, body, disposition) {
|
|
58
|
+
function deliver(subscribers, from, fromName, kind, requestedTier, reportRef, body, disposition) {
|
|
58
59
|
const deliveredTo = [];
|
|
59
60
|
const failedTo = [];
|
|
60
61
|
const label = firstLine(body);
|
|
61
62
|
for (const sub of subscribers) {
|
|
62
63
|
try {
|
|
63
|
-
const entry = { from, ...(fromName === undefined ? {} : { from_name: fromName }), tier: tierFor(kind), kind, ref: reportRef, label, ...(disposition === undefined ? {} : { disposition }) };
|
|
64
|
-
if (sub.active)
|
|
64
|
+
const entry = { from, ...(fromName === undefined ? {} : { from_name: fromName }), tier: tierFor(kind, requestedTier), kind, ref: reportRef, label, ...(disposition === undefined ? {} : { disposition }) };
|
|
65
|
+
if (sub.active) {
|
|
65
66
|
appendInbox(sub.node_id, entry);
|
|
67
|
+
cancelCronsOnWake(sub.node_id);
|
|
68
|
+
}
|
|
66
69
|
else
|
|
67
70
|
appendPassive(sub.node_id, entry);
|
|
68
71
|
deliveredTo.push(sub.node_id);
|
|
@@ -108,6 +111,8 @@ function commitCanonicalFinal(nodeId, staged) {
|
|
|
108
111
|
async function pushWithContext(nodeId, opts) {
|
|
109
112
|
if (!nodeId)
|
|
110
113
|
throw new Error(`push: empty nodeId (kind=${opts.kind}) — a report must belong to a node`);
|
|
114
|
+
if (opts.kind === 'final' && opts.deliveryTier !== undefined)
|
|
115
|
+
throw new Error('push final does not accept a delivery tier');
|
|
111
116
|
const staged = stageReport(nodeId, opts.kind, opts.body);
|
|
112
117
|
const from = opts.from ?? nodeId;
|
|
113
118
|
const fromName = getNode(from)?.name;
|
|
@@ -121,14 +126,14 @@ async function pushWithContext(nodeId, opts) {
|
|
|
121
126
|
// cancel-on-wake rows anchored here go; standing crons survive. Outside
|
|
122
127
|
// commitCanonicalFinal's transaction — it takes its own write boundary.
|
|
123
128
|
cancelCronsOnWake(nodeId);
|
|
124
|
-
const { deliveredTo, failedTo } = deliver(subscribers, from, fromName, opts.kind, `${nodeId}:reports/${staged.basename}`, opts.body, opts.disposition);
|
|
129
|
+
const { deliveredTo, failedTo } = deliver(subscribers, from, fromName, opts.kind, undefined, `${nodeId}:reports/${staged.basename}`, opts.body, opts.disposition);
|
|
125
130
|
if (failedTo.length > 0)
|
|
126
131
|
throw new FinalDeliveryError(staged.finalPath, deliveredTo, failedTo);
|
|
127
132
|
return { reportPath: staged.finalPath, deliveredTo };
|
|
128
133
|
}
|
|
129
134
|
try {
|
|
130
135
|
install(staged);
|
|
131
|
-
const { deliveredTo, failedTo } = deliver(subscribersOf(nodeId), from, fromName, opts.kind, `${nodeId}:reports/${staged.basename}`, opts.body, opts.disposition);
|
|
136
|
+
const { deliveredTo, failedTo } = deliver(subscribersOf(nodeId), from, fromName, opts.kind, opts.deliveryTier, `${nodeId}:reports/${staged.basename}`, opts.body, opts.disposition);
|
|
132
137
|
if (failedTo.length > 0)
|
|
133
138
|
throw new Error(`report published at ${staged.finalPath}; delivery failed for ${failedTo.join(', ')}`);
|
|
134
139
|
return { reportPath: staged.finalPath, deliveredTo };
|
|
@@ -156,7 +161,7 @@ export function redeliverFinalReport(nodeId, originNodeId, reportBasename, body,
|
|
|
156
161
|
if (reportBasename === '' || reportBasename.includes('/') || reportBasename.includes('\\')) {
|
|
157
162
|
throw new Error(`invalid canonical final report basename: ${reportBasename}`);
|
|
158
163
|
}
|
|
159
|
-
const { deliveredTo, failedTo } = deliver([{ node_id: originNodeId, active: true, created: '' }], nodeId, getNode(nodeId)?.name, 'final', `${nodeId}:reports/${reportBasename}`, body, opts?.disposition);
|
|
164
|
+
const { deliveredTo, failedTo } = deliver([{ node_id: originNodeId, active: true, created: '' }], nodeId, getNode(nodeId)?.name, 'final', undefined, `${nodeId}:reports/${reportBasename}`, body, opts?.disposition);
|
|
160
165
|
const reportPath = join(reportsDir(nodeId), reportBasename);
|
|
161
166
|
if (failedTo.length > 0)
|
|
162
167
|
throw new FinalDeliveryError(reportPath, deliveredTo, failedTo);
|
|
@@ -41,7 +41,8 @@ export type InboxEntryInput = Omit<InboxEntry, 'entry_id' | 'ts' | 'operation_id
|
|
|
41
41
|
};
|
|
42
42
|
/** Complete an entry exactly once before any spill or append work. */
|
|
43
43
|
export declare function finalizeInboxEntry(input: InboxEntryInput): InboxEntry;
|
|
44
|
-
/** Atomically append one finalized inbox entry to `nodes/<nodeId>/inbox.jsonl
|
|
44
|
+
/** Atomically append one finalized inbox entry to `nodes/<nodeId>/inbox.jsonl`,
|
|
45
|
+
* at the tier `effectiveTier` resolves for this target. */
|
|
45
46
|
export declare function appendInbox(nodeId: string, entry: InboxEntryInput): InboxEntry;
|
|
46
47
|
/** Deliver the one system notice for a controller that can no longer satisfy a
|
|
47
48
|
* wait. Returns null when another durable operation already settled or re-armed
|
package/dist/core/feed/inbox.js
CHANGED
|
@@ -13,12 +13,13 @@
|
|
|
13
13
|
import { appendFileSync, existsSync, readFileSync, writeFileSync, renameSync, mkdirSync, } from 'node:fs';
|
|
14
14
|
import { dirname, join } from 'node:path';
|
|
15
15
|
import { randomBytes, randomUUID } from 'node:crypto';
|
|
16
|
-
import { deliverAndSettleMessageWait, inboxPath, messagesDir, settleDeadMessageWait } from '../canvas/index.js';
|
|
16
|
+
import { deliverAndSettleMessageWait, getRow, inboxPath, messagesDir, settleDeadMessageWait } from '../canvas/index.js';
|
|
17
17
|
import { resolveRef } from '../canvas/history.js';
|
|
18
18
|
import { emitEvent } from '../events/emit.js';
|
|
19
19
|
import { errorClassFromError } from '../events/errors.js';
|
|
20
20
|
import { generateOperationId, operationIdContext, validateOperationId } from '../events/operation-id.js';
|
|
21
21
|
import { formatInboxCard } from '../../shared/generated-context.js';
|
|
22
|
+
import { renderBirthAnnouncementBody } from '../../shared/birth-announcement.js';
|
|
22
23
|
const INBOX_ENTRY_ID = /^(?!0{32}$)[0-9a-f]{32}$/;
|
|
23
24
|
function validateInboxEntryId(value) {
|
|
24
25
|
if (typeof value !== 'string' || !INBOX_ENTRY_ID.test(value))
|
|
@@ -64,9 +65,24 @@ function writeMessageBody(nodeId, ts, body) {
|
|
|
64
65
|
renameSync(tmpPath, finalPath);
|
|
65
66
|
return finalPath;
|
|
66
67
|
}
|
|
67
|
-
/**
|
|
68
|
+
/** The tier a live watcher injects into the turn a node is ALREADY running
|
|
69
|
+
* (`visibleRoute`, canvas-inbox-watcher). `normal` and `deferred` both wait
|
|
70
|
+
* for a later cycle instead. */
|
|
71
|
+
const STEERING_TIER = 'urgent';
|
|
72
|
+
/** The tier an entry is actually appended at. A terminal node owes one final
|
|
73
|
+
* result and reaps on it, so the later cycle `normal` and `deferred` wait for
|
|
74
|
+
* is one it may never take — mail handed to such a node is raised to the
|
|
75
|
+
* steering tier so the turn it is running now carries it, and so a dormant one
|
|
76
|
+
* is woken for it rather than holding it forever. */
|
|
77
|
+
function effectiveTier(nodeId, tier) {
|
|
78
|
+
if (tier !== 'normal' && tier !== 'deferred')
|
|
79
|
+
return tier;
|
|
80
|
+
return getRow(nodeId)?.lifecycle === 'terminal' ? STEERING_TIER : tier;
|
|
81
|
+
}
|
|
82
|
+
/** Atomically append one finalized inbox entry to `nodes/<nodeId>/inbox.jsonl`,
|
|
83
|
+
* at the tier `effectiveTier` resolves for this target. */
|
|
68
84
|
export function appendInbox(nodeId, entry) {
|
|
69
|
-
const full = finalizeInboxEntry(entry);
|
|
85
|
+
const full = finalizeInboxEntry({ ...entry, tier: effectiveTier(nodeId, entry.tier) });
|
|
70
86
|
try {
|
|
71
87
|
// A push pointer already carries its canonical history `ref` — its body
|
|
72
88
|
// lives in the report file, never inline. A direct message / system notice has no ref yet: when
|
|
@@ -353,6 +369,9 @@ function inlineReport(e) {
|
|
|
353
369
|
* behavior.
|
|
354
370
|
*/
|
|
355
371
|
function entryBody(e) {
|
|
372
|
+
const birth = renderBirthAnnouncementBody(e.data, e.ts);
|
|
373
|
+
if (birth !== null)
|
|
374
|
+
return birth;
|
|
356
375
|
const report = inlineReport(e);
|
|
357
376
|
if (report !== null)
|
|
358
377
|
return report.trimEnd();
|
package/dist/core/git.d.ts
CHANGED
|
@@ -3,6 +3,9 @@ export interface GitResult {
|
|
|
3
3
|
stdout: string;
|
|
4
4
|
stderr: string;
|
|
5
5
|
timedOut?: boolean;
|
|
6
|
+
/** The child did not exit with a numeric status (for example, spawn failure or timeout). */
|
|
7
|
+
executionFailed?: true;
|
|
8
|
+
processError?: string;
|
|
6
9
|
}
|
|
7
10
|
export interface GitSyncOptions {
|
|
8
11
|
timeoutMs?: number;
|
package/dist/core/git.js
CHANGED
|
@@ -3,10 +3,12 @@ import { network } from './errors.js';
|
|
|
3
3
|
export function gitSync(args, cwd, input, options = {}) {
|
|
4
4
|
const res = spawnSync('git', args, { cwd, encoding: 'utf8', input, timeout: options.timeoutMs });
|
|
5
5
|
const status = typeof res.status === 'number' ? res.status : 1;
|
|
6
|
+
const completed = typeof res.status === 'number';
|
|
6
7
|
const stdout = typeof res.stdout === 'string' ? res.stdout : '';
|
|
7
8
|
const stderr = typeof res.stderr === 'string' ? res.stderr : '';
|
|
9
|
+
const processError = res.error === undefined ? undefined : String(res.error);
|
|
8
10
|
const timedOut = res.error?.code === 'ETIMEDOUT';
|
|
9
|
-
return { status, stdout, stderr, ...(timedOut ? { timedOut: true } : {}) };
|
|
11
|
+
return { status, stdout, stderr, ...(completed ? {} : { executionFailed: true }), ...(processError === undefined ? {} : { processError }), ...(timedOut ? { timedOut: true } : {}) };
|
|
10
12
|
}
|
|
11
13
|
export async function gitAsync(args, cwd) {
|
|
12
14
|
return new Promise((resolve) => {
|
|
@@ -4,6 +4,7 @@ import type { BrokerClient } from './client-registry.js';
|
|
|
4
4
|
import type { FrameReplies } from './frame-replies.js';
|
|
5
5
|
import type { RebindDriver } from './rebind.js';
|
|
6
6
|
import type { BrokerSession } from './read-ops.js';
|
|
7
|
+
import type { TurnAdmissionGate } from './turn-admission.js';
|
|
7
8
|
type ResolvedLaunchModel = {
|
|
8
9
|
model: NonNullable<ReturnType<ModelRegistry['find']>>;
|
|
9
10
|
thinkingLevel: string | undefined;
|
|
@@ -35,5 +36,6 @@ export declare function createAuthReloadGate(deps: {
|
|
|
35
36
|
resolveLaunchModel: (services: AgentSessionServices, cfg: BrokerSdkConfig) => Promise<ResolvedLaunchModel>;
|
|
36
37
|
/** A reconcile that finally resolves a model releases the held launch prompt. */
|
|
37
38
|
driveKickoff: () => void;
|
|
39
|
+
turnAdmission: TurnAdmissionGate;
|
|
38
40
|
}): AuthReloadGate;
|
|
39
41
|
export {};
|
|
@@ -4,7 +4,7 @@ import { emitEvent } from '../../events/emit.js';
|
|
|
4
4
|
import { clearFault, readFault } from '../fault.js';
|
|
5
5
|
import { isUnknownModel } from '../model-selection.js';
|
|
6
6
|
export function createAuthReloadGate(deps) {
|
|
7
|
-
const { nodeId, cfg, replies, rebind, session, services, broadcastModelChanged, formatModelSpec, resolveLaunchModel, driveKickoff, } = deps;
|
|
7
|
+
const { nodeId, cfg, replies, rebind, session, services, broadcastModelChanged, formatModelSpec, resolveLaunchModel, driveKickoff, turnAdmission, } = deps;
|
|
8
8
|
let reloadInFlight = null;
|
|
9
9
|
let faultRedriveInFlight = false;
|
|
10
10
|
/** True once the engine this work started against has been replaced — every
|
|
@@ -112,8 +112,14 @@ export function createAuthReloadGate(deps) {
|
|
|
112
112
|
fresh.kind !== 'auth' ||
|
|
113
113
|
fresh.since !== authFault.since)
|
|
114
114
|
return;
|
|
115
|
-
|
|
116
|
-
|
|
115
|
+
await turnAdmission.admit((release) => {
|
|
116
|
+
if (supersededBy(generation, faultSession) || faultSession.isStreaming) {
|
|
117
|
+
release();
|
|
118
|
+
return Promise.resolve();
|
|
119
|
+
}
|
|
120
|
+
clearFault(nodeId, { link: 'pi→provider' });
|
|
121
|
+
return faultSession.prompt(formatCard('recovery', {}, AUTH_FAULT_RECOVERY_BODY), { preflightResult: release });
|
|
122
|
+
});
|
|
117
123
|
}
|
|
118
124
|
catch (err) {
|
|
119
125
|
emitEvent({
|
|
@@ -8,6 +8,7 @@ import type { BrokerSession } from './read-ops.js';
|
|
|
8
8
|
import type { RebindDriver } from './rebind.js';
|
|
9
9
|
import type { ToolGroupTracker } from './tool-groups.js';
|
|
10
10
|
import type { PromptDispatchTracker } from './turn-ignition.js';
|
|
11
|
+
import type { TurnAdmissionGate } from './turn-admission.js';
|
|
11
12
|
/** The frames that drive the live engine — the viewer's input vocabulary. */
|
|
12
13
|
export type DriveFrame = Extract<ClientToBroker, {
|
|
13
14
|
type: 'prompt' | 'steer' | 'follow_up' | 'abort' | 'bash';
|
|
@@ -23,4 +24,5 @@ export declare function createEngineDriver(deps: {
|
|
|
23
24
|
rebind: RebindDriver;
|
|
24
25
|
projection: EventProjection;
|
|
25
26
|
toolGroups: ToolGroupTracker;
|
|
27
|
+
turnAdmission: TurnAdmissionGate;
|
|
26
28
|
}): EngineDriver;
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { emitEvent } from '../../events/emit.js';
|
|
2
2
|
import { operationIdContext } from '../../events/operation-id.js';
|
|
3
3
|
import { promptWithAdvertisedCommandInvocation } from '../advertised-command-invocation.js';
|
|
4
|
+
import { heldDeferredInboxPromptJoin } from './held-deferred-inbox.js';
|
|
4
5
|
import { chooseGuidanceDeliveryMode, promptWithNextTurnGuidance, resolveEngineRoute, resolveSteerCall, } from './engine-routing.js';
|
|
5
6
|
export function createEngineDriver(deps) {
|
|
6
|
-
const { registry, replies, session, dispatches, memoryRefs, authReloadGate, rebind, projection, toolGroups, } = deps;
|
|
7
|
+
const { registry, replies, session, dispatches, memoryRefs, authReloadGate, rebind, projection, toolGroups, turnAdmission, } = deps;
|
|
7
8
|
const { ackTo, engineErrorTo } = replies;
|
|
8
9
|
const errorRelay = (client) => {
|
|
9
10
|
let failure;
|
|
@@ -39,18 +40,32 @@ export function createEngineDriver(deps) {
|
|
|
39
40
|
// before the caught promise resolves, so reading it after the chain is
|
|
40
41
|
// the call's verdict.
|
|
41
42
|
const settle = (p) => p.then(() => errors.failure());
|
|
42
|
-
const dispatch = () => {
|
|
43
|
+
const dispatch = () => turnAdmission.admit((release) => {
|
|
44
|
+
// A prior prompt can be in asynchronous preflight while isStreaming is
|
|
45
|
+
// false. Admission waits for that preflight, then routes against the
|
|
46
|
+
// state it actually established instead of its stale snapshot.
|
|
43
47
|
const route = resolveEngineRoute(frame, session().isStreaming);
|
|
48
|
+
const promptOptions = route.call === 'prompt' ? route.options : undefined;
|
|
44
49
|
const guidance = memoryRefs.guidanceFor(frame.text);
|
|
50
|
+
const prompt = async () => {
|
|
51
|
+
// Both human surfaces converge at this admitted boundary. The watcher
|
|
52
|
+
// claims held deferred mail before the prompt body, but its cursor stays
|
|
53
|
+
// uncommitted until the resulting turn's agent_settled event.
|
|
54
|
+
if (route.call === 'prompt' && !session().isStreaming) {
|
|
55
|
+
await heldDeferredInboxPromptJoin()?.(session());
|
|
56
|
+
}
|
|
57
|
+
await session().prompt(frame.text, { ...promptOptions, preflightResult: release });
|
|
58
|
+
};
|
|
45
59
|
if (guidance === null) {
|
|
46
60
|
if (route.call === 'prompt') {
|
|
47
61
|
// A routed prompt STARTS a turn — including an idle follow_up
|
|
48
62
|
// escalated by m-C — so it gets a fresh operation id. A true
|
|
49
63
|
// (mid-stream) followUp rides the current turn's operation, like
|
|
50
64
|
// steer.
|
|
51
|
-
return settle(dispatches.track(operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text,
|
|
65
|
+
return settle(dispatches.track(operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text, prompt).catch(relayError))));
|
|
52
66
|
}
|
|
53
67
|
// C1: pi's followUp() takes images as a POSITIONAL 2nd arg (`followUp(text, images?)`).
|
|
68
|
+
release();
|
|
54
69
|
return settle(session().followUp(frame.text, frame.images).catch(relayError));
|
|
55
70
|
}
|
|
56
71
|
const deliverAs = chooseGuidanceDeliveryMode(frame.type, route.call, session().isStreaming);
|
|
@@ -63,19 +78,20 @@ export function createEngineDriver(deps) {
|
|
|
63
78
|
// ATOMICALLY clears its own pending entry if prompt() settles
|
|
64
79
|
// without ever reaching that drain (an early throw/return leaves
|
|
65
80
|
// nothing stranded to leak into a later, unrelated turn).
|
|
66
|
-
return settle(dispatches.track(promptWithNextTurnGuidance(session(), guidance, () => operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text,
|
|
81
|
+
return settle(dispatches.track(promptWithNextTurnGuidance(session(), guidance, () => operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text, prompt))).catch(relayError)));
|
|
67
82
|
}
|
|
68
|
-
return settle(dispatches.track(operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text,
|
|
83
|
+
return settle(dispatches.track(operationIdContext.fresh(() => promptWithAdvertisedCommandInvocation(session(), frame.text, prompt)
|
|
69
84
|
.catch(relayError)
|
|
70
85
|
.then(() => sendGuidance(guidance, deliverAs)))));
|
|
71
86
|
}
|
|
72
87
|
// deliverAs is always 'followUp' here per chooseGuidanceDeliveryMode's
|
|
73
88
|
// contract (a followUp routeCall never yields 'nextTurn').
|
|
89
|
+
release();
|
|
74
90
|
return settle(session()
|
|
75
91
|
.followUp(frame.text, frame.images)
|
|
76
92
|
.catch(relayError)
|
|
77
93
|
.then(() => sendGuidance(guidance, deliverAs)));
|
|
78
|
-
};
|
|
94
|
+
});
|
|
79
95
|
// /login acknowledges before its asynchronous local refresh completes.
|
|
80
96
|
// An idle turn must not overtake that refresh and launch on the stale
|
|
81
97
|
// route; true mid-stream follow-ups keep their immediate queue behavior.
|
|
@@ -130,23 +146,23 @@ export function createEngineDriver(deps) {
|
|
|
130
146
|
*/
|
|
131
147
|
const driveSteer = (frame, errors) => {
|
|
132
148
|
const relayError = errors.relay;
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
149
|
+
void turnAdmission.admit((release) => {
|
|
150
|
+
if (resolveSteerCall(frame.text, session()) === 'prompt') {
|
|
151
|
+
return promptWithAdvertisedCommandInvocation(session(), frame.text, () => session().prompt(frame.text, {
|
|
152
|
+
images: frame.images,
|
|
153
|
+
streamingBehavior: 'steer',
|
|
154
|
+
preflightResult: release,
|
|
155
|
+
}));
|
|
156
|
+
}
|
|
157
|
+
release();
|
|
158
|
+
const guidance = memoryRefs.guidanceFor(frame.text);
|
|
159
|
+
// C1: pi's steer() takes images as a POSITIONAL 2nd arg (`steer(text, images?)`).
|
|
160
|
+
if (guidance === null)
|
|
161
|
+
return session().steer(frame.text, frame.images);
|
|
162
|
+
return session()
|
|
163
|
+
.steer(frame.text, frame.images)
|
|
164
|
+
.then(() => sendGuidance(guidance, 'steer'));
|
|
165
|
+
}).catch(relayError);
|
|
150
166
|
};
|
|
151
167
|
/**
|
|
152
168
|
* esc/abort cancels whatever is live: a running `!` command (abortBash)
|
|
@@ -21,6 +21,7 @@ export declare class EventProjection {
|
|
|
21
21
|
private pendingUpdateTimer;
|
|
22
22
|
private workingActivity;
|
|
23
23
|
private readonly streamWatchdog;
|
|
24
|
+
private readonly toolExecutionWatchdog;
|
|
24
25
|
private readonly streamWatchdogMs;
|
|
25
26
|
private pendingBroadcast;
|
|
26
27
|
private broadcastGeneration;
|
|
@@ -30,8 +31,11 @@ export declare class EventProjection {
|
|
|
30
31
|
flush(): void;
|
|
31
32
|
reset(): void;
|
|
32
33
|
dispose(): void;
|
|
34
|
+
/** Arms the applicable watchdog floor after a streaming event or rebind. */
|
|
35
|
+
rearmIfStreaming(generation: SubscriptionGeneration): void;
|
|
33
36
|
knownStreamWait(deadlineMs: number): () => void;
|
|
34
37
|
relayEvent(event: BrokerToClient, generation: SubscriptionGeneration): void;
|
|
38
|
+
private abortStalledStream;
|
|
35
39
|
private clearPendingUpdate;
|
|
36
40
|
private flushPendingUpdate;
|
|
37
41
|
private invalidatePendingBroadcasts;
|
|
@@ -6,6 +6,9 @@ import { withoutYieldAbort } from '../session-cycles.js';
|
|
|
6
6
|
import { DEFAULT_WORKING_ACTIVITY, randomWorkingActivity } from '../../../shared/working-activity.js';
|
|
7
7
|
import { hydrateLiveAdvertisedCommandMessage, prepareAdvertisedCommandPersistence, } from '../advertised-command-invocation.js';
|
|
8
8
|
const MESSAGE_UPDATE_COALESCE_MS = 75;
|
|
9
|
+
// Match the daemon's wedge grace: a tool may legitimately be silent longer than
|
|
10
|
+
// a provider stream, but it cannot leave the broker unguarded indefinitely.
|
|
11
|
+
const TOOL_EXECUTION_WATCHDOG_MS = 20 * 60_000;
|
|
9
12
|
/** Projects one live engine event stream to broker viewers and owns its relay timers. */
|
|
10
13
|
export class EventProjection {
|
|
11
14
|
deps;
|
|
@@ -13,25 +16,18 @@ export class EventProjection {
|
|
|
13
16
|
pendingUpdateTimer = null;
|
|
14
17
|
workingActivity = DEFAULT_WORKING_ACTIVITY;
|
|
15
18
|
streamWatchdog;
|
|
19
|
+
toolExecutionWatchdog;
|
|
16
20
|
streamWatchdogMs = envStreamWatchdogMs();
|
|
17
21
|
pendingBroadcast = null;
|
|
18
22
|
broadcastGeneration = 0;
|
|
19
23
|
constructor(deps) {
|
|
20
24
|
this.deps = deps;
|
|
21
25
|
this.streamWatchdog = new StreamWatchdog((armedGeneration) => {
|
|
22
|
-
|
|
23
|
-
if (generation === undefined || armedGeneration !== generation.id)
|
|
24
|
-
return;
|
|
25
|
-
emitEvent({
|
|
26
|
-
level: 'debug',
|
|
27
|
-
event: 'broker.runtime.diagnostic',
|
|
28
|
-
fields: { message: `stream watchdog: no engine event for ${this.streamWatchdogMs}ms while streaming — aborting stalled provider stream` },
|
|
29
|
-
});
|
|
30
|
-
generation.stagedWatchdogAbort = true;
|
|
31
|
-
void generation.session.abort().catch((err) => {
|
|
32
|
-
emitEvent({ level: 'debug', event: 'broker.runtime.diagnostic', fields: { message: `stream watchdog abort failed: ${String(err)}` } });
|
|
33
|
-
});
|
|
26
|
+
this.abortStalledStream(armedGeneration, this.streamWatchdogMs, 'while streaming — aborting stalled provider stream');
|
|
34
27
|
}, this.streamWatchdogMs);
|
|
28
|
+
this.toolExecutionWatchdog = new StreamWatchdog((armedGeneration) => {
|
|
29
|
+
this.abortStalledStream(armedGeneration, TOOL_EXECUTION_WATCHDOG_MS, 'while a tool is executing — aborting stalled tool execution');
|
|
30
|
+
}, TOOL_EXECUTION_WATCHDOG_MS);
|
|
35
31
|
}
|
|
36
32
|
currentWorkingActivity() {
|
|
37
33
|
return this.workingActivity;
|
|
@@ -46,11 +42,25 @@ export class EventProjection {
|
|
|
46
42
|
this.clearPendingUpdate();
|
|
47
43
|
this.invalidatePendingBroadcasts();
|
|
48
44
|
this.streamWatchdog.reset();
|
|
45
|
+
this.toolExecutionWatchdog.reset();
|
|
49
46
|
}
|
|
50
47
|
dispose() {
|
|
51
48
|
this.clearPendingUpdate();
|
|
52
49
|
this.invalidatePendingBroadcasts();
|
|
53
50
|
this.streamWatchdog.reset();
|
|
51
|
+
this.toolExecutionWatchdog.reset();
|
|
52
|
+
}
|
|
53
|
+
/** Arms the applicable watchdog floor after a streaming event or rebind. */
|
|
54
|
+
rearmIfStreaming(generation) {
|
|
55
|
+
if (!generation.session.isStreaming)
|
|
56
|
+
return;
|
|
57
|
+
if (generation.executingTools.size > 0) {
|
|
58
|
+
this.streamWatchdog.clear();
|
|
59
|
+
this.toolExecutionWatchdog.poke(generation.id);
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
this.toolExecutionWatchdog.clear();
|
|
63
|
+
this.streamWatchdog.poke(generation.id);
|
|
54
64
|
}
|
|
55
65
|
knownStreamWait(deadlineMs) {
|
|
56
66
|
if (!Number.isFinite(deadlineMs) || deadlineMs <= Date.now())
|
|
@@ -99,18 +109,14 @@ export class EventProjection {
|
|
|
99
109
|
if (type === 'agent_end') {
|
|
100
110
|
generation.executingTools.clear();
|
|
101
111
|
this.streamWatchdog.reset();
|
|
112
|
+
this.toolExecutionWatchdog.reset();
|
|
102
113
|
}
|
|
103
|
-
else
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
if (generation.executingTools.size === 0 && generation.session.isStreaming)
|
|
110
|
-
this.streamWatchdog.poke(generation.id);
|
|
111
|
-
}
|
|
112
|
-
else if (generation.session.isStreaming && generation.executingTools.size === 0) {
|
|
113
|
-
this.streamWatchdog.poke(generation.id);
|
|
114
|
+
else {
|
|
115
|
+
if (type === 'tool_execution_start')
|
|
116
|
+
generation.executingTools.add(event.toolCallId ?? '');
|
|
117
|
+
else if (type === 'tool_execution_end')
|
|
118
|
+
generation.executingTools.delete(event.toolCallId ?? '');
|
|
119
|
+
this.rearmIfStreaming(generation);
|
|
114
120
|
}
|
|
115
121
|
if (type === 'message_update') {
|
|
116
122
|
this.pendingUpdate = event;
|
|
@@ -128,6 +134,20 @@ export class EventProjection {
|
|
|
128
134
|
}
|
|
129
135
|
this.broadcast(event);
|
|
130
136
|
}
|
|
137
|
+
abortStalledStream(armedGeneration, timeoutMs, context) {
|
|
138
|
+
const generation = this.deps.installedGeneration();
|
|
139
|
+
if (generation === undefined || armedGeneration !== generation.id)
|
|
140
|
+
return;
|
|
141
|
+
emitEvent({
|
|
142
|
+
level: 'debug',
|
|
143
|
+
event: 'broker.runtime.diagnostic',
|
|
144
|
+
fields: { message: `stream watchdog: no engine event for ${timeoutMs}ms ${context}` },
|
|
145
|
+
});
|
|
146
|
+
generation.stagedWatchdogAbort = true;
|
|
147
|
+
void generation.session.abort().catch((err) => {
|
|
148
|
+
emitEvent({ level: 'debug', event: 'broker.runtime.diagnostic', fields: { message: `stream watchdog abort failed: ${String(err)}` } });
|
|
149
|
+
});
|
|
150
|
+
}
|
|
131
151
|
clearPendingUpdate() {
|
|
132
152
|
if (this.pendingUpdateTimer !== null) {
|
|
133
153
|
clearTimeout(this.pendingUpdateTimer);
|