@north-light/crouter 0.3.282 → 0.3.284
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/client.d.ts +22 -4
- package/dist/api/client.js +34 -0
- package/dist/api/dto/broker-ops.d.ts +11 -0
- package/dist/api/dto/broker.d.ts +3 -8
- package/dist/api/dto/common.d.ts +1 -1
- package/dist/api/dto/messages.d.ts +2 -2
- package/dist/api/dto/node-outcomes.d.ts +80 -0
- package/dist/api/dto/node-outcomes.js +2 -0
- package/dist/api/dto/nodes.d.ts +9 -0
- package/dist/api/dto/profiles.d.ts +5 -0
- package/dist/api/dto/reports.d.ts +45 -0
- package/dist/api/index.d.ts +1 -0
- package/dist/api/index.js +1 -0
- package/dist/api/routes.d.ts +7 -0
- package/dist/api/routes.js +7 -0
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js +24 -3
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +31 -4
- package/dist/clients/attach/overlays/graph.d.ts +5 -0
- package/dist/clients/attach/overlays/graph.js +65 -47
- package/dist/clients/attach/overlays/pickers.js +8 -9
- package/dist/clients/attach/render/card-presentation.js +1 -1
- package/dist/clients/attach/viewer.js +1286 -1260
- package/dist/commands/api-client.js +22 -23
- package/dist/commands/node/create.js +38 -3
- package/dist/commands/node/inspect.js +3 -7
- package/dist/commands/node/message.js +3 -3
- package/dist/commands/node/outcome.d.ts +2 -0
- package/dist/commands/node/outcome.js +134 -0
- package/dist/commands/node.js +2 -1
- package/dist/commands/profile/pause.js +17 -14
- package/dist/commands/push.js +102 -1
- package/dist/commands/sys/__tests__/setup-core.test.js +4 -1
- package/dist/commands/sys/panels/broker-limits-panel.js +7 -1
- package/dist/commands/sys/panels/models-panel.js +2 -2
- package/dist/commands/sys/settings-shell.d.ts +1 -1
- package/dist/commands/sys/settings-shell.js +6 -0
- package/dist/commands/sys/settings.js +3 -3
- package/dist/core/__tests__/fixtures/fake-engine.d.ts +2 -0
- package/dist/core/__tests__/fixtures/fake-engine.js +42 -4
- package/dist/core/__tests__/helpers/harness.d.ts +1 -0
- package/dist/core/__tests__/helpers/harness.js +9 -0
- package/dist/core/__tests__/integration/profile-pause.test.d.ts +1 -0
- package/dist/core/__tests__/integration/profile-pause.test.js +115 -0
- package/dist/core/__tests__/node-outcome.test.d.ts +1 -0
- package/dist/core/__tests__/node-outcome.test.js +184 -0
- package/dist/core/__tests__/revive-parked-fresh.test.js +13 -20
- package/dist/core/__tests__/seam/broker-provider-retry.test.js +6 -2
- package/dist/core/__tests__/seam/dormancy-release.test.js +3 -3
- package/dist/core/__tests__/seam/node-outcome-delivery.test.d.ts +1 -0
- package/dist/core/__tests__/seam/node-outcome-delivery.test.js +153 -0
- package/dist/core/bash-jobs.d.ts +6 -0
- package/dist/core/bash-jobs.js +22 -0
- package/dist/core/canvas/canvas.d.ts +41 -3
- package/dist/core/canvas/canvas.js +153 -39
- package/dist/core/canvas/crons.js +2 -1
- package/dist/core/canvas/index.d.ts +1 -0
- package/dist/core/canvas/index.js +1 -0
- package/dist/core/canvas/migrations.js +47 -0
- package/dist/core/canvas/node-outcome-deliveries.d.ts +80 -0
- package/dist/core/canvas/node-outcome-deliveries.js +163 -0
- package/dist/core/canvas/pid.d.ts +7 -0
- package/dist/core/canvas/pid.js +17 -0
- package/dist/core/canvas/render-source.js +2 -2
- package/dist/core/canvas/types.d.ts +42 -1
- package/dist/core/canvas/types.js +2 -1
- package/dist/core/config.d.ts +4 -1
- package/dist/core/config.js +25 -1
- package/dist/core/feed/feed.d.ts +8 -0
- package/dist/core/feed/feed.js +25 -4
- package/dist/core/feed/inbox.d.ts +5 -0
- package/dist/core/feed/inbox.js +13 -0
- package/dist/core/keybindings/catalog.d.ts +1 -1
- package/dist/core/keybindings/catalog.js +3 -0
- package/dist/core/profiles/manifest.d.ts +3 -1
- package/dist/core/profiles/manifest.js +6 -1
- package/dist/core/runtime/broker/daemon-ops.d.ts +3 -1
- package/dist/core/runtime/broker/daemon-ops.js +6 -0
- package/dist/core/runtime/broker/engine-drive.d.ts +2 -3
- package/dist/core/runtime/broker/engine-drive.js +74 -27
- package/dist/core/runtime/broker/frame-dispatch.js +1 -0
- package/dist/core/runtime/broker/read-ops.js +14 -0
- package/dist/core/runtime/broker/rebind.js +0 -7
- package/dist/core/runtime/broker-protocol.d.ts +1 -6
- package/dist/core/runtime/fleet.d.ts +7 -0
- package/dist/core/runtime/headless-pi.d.ts +13 -0
- package/dist/core/runtime/headless-pi.js +10 -2
- package/dist/core/runtime/lifecycle.d.ts +6 -1
- package/dist/core/runtime/lifecycle.js +50 -1
- package/dist/core/runtime/nodes.d.ts +5 -1
- package/dist/core/runtime/nodes.js +2 -1
- package/dist/core/runtime/outcome-document.d.ts +70 -0
- package/dist/core/runtime/outcome-document.js +97 -0
- package/dist/core/runtime/revive.js +16 -21
- package/dist/core/runtime/spawn.d.ts +5 -1
- package/dist/core/runtime/spawn.js +18 -7
- package/dist/core/runtime/stamp/channel.d.ts +3 -5
- package/dist/core/runtime/stamp/channel.js +3 -10
- package/dist/core/runtime/stop-guard.js +2 -2
- package/dist/core/runtime/structured-output.d.ts +24 -3
- package/dist/core/runtime/structured-output.js +42 -3
- package/dist/core/runtime/tmux-bindings.js +4 -0
- package/dist/core/runtime/warm-pool.d.ts +5 -2
- package/dist/core/runtime/warm-pool.js +5 -5
- package/dist/core/subscription-state.js +7 -5
- package/dist/core/substrate/render.js +10 -10
- package/dist/core/user-settings.d.ts +18 -1
- package/dist/core/user-settings.js +37 -0
- package/dist/daemon/__tests__/node-deadline-lane.test.d.ts +1 -0
- package/dist/daemon/__tests__/node-deadline-lane.test.js +76 -0
- package/dist/daemon/__tests__/node-outcome-birth-invariants.test.d.ts +1 -0
- package/dist/daemon/__tests__/node-outcome-birth-invariants.test.js +122 -0
- package/dist/daemon/__tests__/node-outcome-delivery.test.d.ts +1 -0
- package/dist/daemon/__tests__/node-outcome-delivery.test.js +86 -0
- package/dist/daemon/api/__tests__/broker-settle-park.test.js +86 -18
- package/dist/daemon/api/__tests__/profile-launch-gates.test.js +10 -1
- package/dist/daemon/api/__tests__/seam/node-outcome-api.test.d.ts +1 -0
- package/dist/daemon/api/__tests__/seam/node-outcome-api.test.js +159 -0
- package/dist/daemon/api/__tests__/submit-result.test.d.ts +1 -0
- package/dist/daemon/api/__tests__/submit-result.test.js +214 -0
- package/dist/daemon/api/handlers/broker-ops.js +53 -24
- package/dist/daemon/api/handlers/canvas.js +2 -2
- package/dist/daemon/api/handlers/messages.js +12 -2
- package/dist/daemon/api/handlers/node-outcomes.d.ts +2 -0
- package/dist/daemon/api/handlers/node-outcomes.js +136 -0
- package/dist/daemon/api/handlers/nodes.js +67 -1
- package/dist/daemon/api/handlers/profiles.js +53 -1
- package/dist/daemon/api/handlers/reports.js +197 -33
- package/dist/daemon/api/map.d.ts +7 -2
- package/dist/daemon/api/map.js +78 -4
- package/dist/daemon/api/server.js +74 -7
- package/dist/daemon/crtrd.js +12 -0
- package/dist/daemon/fleet.d.ts +4 -8
- package/dist/daemon/fleet.js +48 -9
- package/dist/daemon/messaging/node-message.js +2 -1
- package/dist/daemon/node-outcome/deliver-outcome.d.ts +14 -0
- package/dist/daemon/node-outcome/deliver-outcome.js +155 -0
- package/dist/daemon/park-activity.d.ts +9 -0
- package/dist/daemon/park-activity.js +30 -0
- package/dist/daemon/park-pending.d.ts +10 -5
- package/dist/daemon/park-pending.js +18 -8
- package/dist/daemon/reconcilers/bash-deadline.js +6 -0
- package/dist/daemon/reconcilers/broker-supervision.d.ts +3 -7
- package/dist/daemon/reconcilers/broker-supervision.js +29 -33
- package/dist/daemon/reconcilers/node-deadline.d.ts +5 -0
- package/dist/daemon/reconcilers/node-deadline.js +82 -0
- package/dist/daemon/reconcilers/node-lifecycle/freeze-lane.js +9 -1
- package/dist/daemon/reconcilers/node-lifecycle/tick.js +12 -3
- package/dist/daemon/reconcilers/node-outcome-delivery-lane.d.ts +10 -0
- package/dist/daemon/reconcilers/node-outcome-delivery-lane.js +41 -0
- package/dist/pi-extensions/__tests__/canvas-structured-output.test.js +66 -66
- package/dist/pi-extensions/canvas-structured-output.d.ts +9 -5
- package/dist/pi-extensions/canvas-structured-output.js +23 -297
- package/dist/shared/env.d.ts +8 -0
- package/dist/shared/env.js +14 -0
- package/dist/shared/generated-context.d.ts +3 -9
- package/dist/shared/generated-context.js +8 -16
- package/dist/types.d.ts +12 -0
- package/dist/types.js +1 -0
- package/package.json +9 -8
- package/runtime.lock.json +227 -383
|
@@ -3,10 +3,14 @@
|
|
|
3
3
|
// Two `http.Server` listeners share one logical API (a Node server can bind one
|
|
4
4
|
// handle only):
|
|
5
5
|
// - the unix socket at `apiSocketPath()` — ALWAYS bound, owner-only (mode
|
|
6
|
-
// 0600 = the fs-perms-are-auth model);
|
|
6
|
+
// 0600 = the fs-perms-are-auth model); NEVER token-checked, token set or
|
|
7
|
+
// not;
|
|
7
8
|
// - an opt-in TCP listener, bound only when `--tcp <host:port>` / `CRTRD_TCP`
|
|
8
|
-
// is set
|
|
9
|
-
//
|
|
9
|
+
// is set. When `CRTRD_TOKEN` is ALSO set, every TCP request and WS upgrade
|
|
10
|
+
// must carry `Authorization: Bearer <token>` or the daemon answers 401
|
|
11
|
+
// with the standard `ErrorBody` (errors.ts) — otherwise (the default) TCP
|
|
12
|
+
// gets no auth check at all, unchanged from before this guard existed (a
|
|
13
|
+
// fronting edge owns auth there instead).
|
|
10
14
|
//
|
|
11
15
|
// Non-upgrade requests flow through the hand-rolled `Router` (A-6) loaded with
|
|
12
16
|
// every handler route table (A-7). `GET /v1/nodes/:id/attach` upgrades are
|
|
@@ -17,7 +21,7 @@ import { createServer } from 'node:http';
|
|
|
17
21
|
import { chmodSync, existsSync, unlinkSync } from 'node:fs';
|
|
18
22
|
import { apiSocketPath } from '../../core/canvas/paths.js';
|
|
19
23
|
import { emitEvent } from '../../core/events/emit.js';
|
|
20
|
-
import { envCrtrdTcp } from '../../shared/env.js';
|
|
24
|
+
import { envCrtrdTcp, envCrtrdToken } from '../../shared/env.js';
|
|
21
25
|
import { Router } from './router.js';
|
|
22
26
|
import { handleAttachUpgrade } from './bridge.js';
|
|
23
27
|
import { attachRoutes } from './handlers/attach.js';
|
|
@@ -38,6 +42,7 @@ import { messageRoutes } from './handlers/messages.js';
|
|
|
38
42
|
import { modelAuthRoutes } from './handlers/modelauth.js';
|
|
39
43
|
import { modelConfigRoutes } from './handlers/model-config.js';
|
|
40
44
|
import { nodeRoutes } from './handlers/nodes.js';
|
|
45
|
+
import { nodeOutcomeRoutes } from './handlers/node-outcomes.js';
|
|
41
46
|
import { profileRoutes } from './handlers/profiles.js';
|
|
42
47
|
import { prospectiveChatInventoryRoutes } from './handlers/prospective-chat-inventory.js';
|
|
43
48
|
import { reportRoutes } from './handlers/reports.js';
|
|
@@ -54,6 +59,7 @@ function buildRouter() {
|
|
|
54
59
|
.registerAll(healthRoutes)
|
|
55
60
|
.registerAll(daemonRoutes)
|
|
56
61
|
.registerAll(nodeRoutes)
|
|
62
|
+
.registerAll(nodeOutcomeRoutes)
|
|
57
63
|
.registerAll(bashJobRoutes)
|
|
58
64
|
.registerAll(brokerOperationRoutes)
|
|
59
65
|
.registerAll(worktreeRoutes)
|
|
@@ -112,6 +118,46 @@ function parseTcp(spec) {
|
|
|
112
118
|
return null;
|
|
113
119
|
return { host: host === '' ? '0.0.0.0' : host, port };
|
|
114
120
|
}
|
|
121
|
+
/** The standard `ErrorBody` (errors.ts) for a TCP request/upgrade missing or
|
|
122
|
+
* carrying a wrong `Authorization: Bearer <token>` header. Only ever written
|
|
123
|
+
* when `CRTRD_TOKEN` is set — an unset token never reaches this path. */
|
|
124
|
+
const UNAUTHORIZED_BODY = {
|
|
125
|
+
error: { code: 'unauthorized', message: 'missing or invalid bearer token' },
|
|
126
|
+
};
|
|
127
|
+
/** Does `req` carry the exact `Authorization: Bearer <token>` this TCP
|
|
128
|
+
* listener requires? */
|
|
129
|
+
function isAuthorized(req, token) {
|
|
130
|
+
return req.headers['authorization'] === `Bearer ${token}`;
|
|
131
|
+
}
|
|
132
|
+
/** Reject an unauthorized non-upgrade TCP request with the standard 401 body. */
|
|
133
|
+
function writeUnauthorized(res) {
|
|
134
|
+
const payload = JSON.stringify(UNAUTHORIZED_BODY);
|
|
135
|
+
res.writeHead(401, { 'content-type': 'application/json; charset=utf-8' });
|
|
136
|
+
res.end(payload);
|
|
137
|
+
}
|
|
138
|
+
/** Reject an unauthorized TCP WS-upgrade attempt: this is a raw pre-upgrade
|
|
139
|
+
* socket (no `ServerResponse`), so the 401 is written as a literal HTTP
|
|
140
|
+
* response — mirroring `bridge.ts`'s `refuseUpgrade` (`end()`, not
|
|
141
|
+
* `write()`+`destroy()`, so a slow peer can't truncate the body) — before the
|
|
142
|
+
* socket closes. */
|
|
143
|
+
function destroyUpgradeUnauthorized(socket) {
|
|
144
|
+
const payload = JSON.stringify(UNAUTHORIZED_BODY);
|
|
145
|
+
const response = `HTTP/1.1 401 Unauthorized\r\n` +
|
|
146
|
+
`content-type: application/json; charset=utf-8\r\n` +
|
|
147
|
+
`content-length: ${Buffer.byteLength(payload)}\r\n` +
|
|
148
|
+
`connection: close\r\n\r\n${payload}`;
|
|
149
|
+
try {
|
|
150
|
+
socket.end(response);
|
|
151
|
+
}
|
|
152
|
+
catch {
|
|
153
|
+
try {
|
|
154
|
+
socket.destroy();
|
|
155
|
+
}
|
|
156
|
+
catch {
|
|
157
|
+
/* already gone */
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
}
|
|
115
161
|
/** Close one http.Server, dropping idle keep-alives so the callback fires. */
|
|
116
162
|
function closeServer(server) {
|
|
117
163
|
return new Promise((resolve) => {
|
|
@@ -242,7 +288,9 @@ export function createApiServer(opts = {}) {
|
|
|
242
288
|
});
|
|
243
289
|
}
|
|
244
290
|
});
|
|
245
|
-
// TCP — opt-in only. A SECOND http.Server sharing the same handlers
|
|
291
|
+
// TCP — opt-in only. A SECOND http.Server sharing the same handlers, gated
|
|
292
|
+
// by CRTRD_TOKEN when set (never the unix socket, which shares onRequest/
|
|
293
|
+
// onUpgrade directly above with no wrapping at all).
|
|
246
294
|
let tcpServer;
|
|
247
295
|
const tcpSpec = opts.tcp ?? envCrtrdTcp();
|
|
248
296
|
if (tcpSpec !== undefined && tcpSpec !== '') {
|
|
@@ -255,8 +303,27 @@ export function createApiServer(opts = {}) {
|
|
|
255
303
|
});
|
|
256
304
|
}
|
|
257
305
|
else {
|
|
258
|
-
|
|
259
|
-
|
|
306
|
+
const token = envCrtrdToken();
|
|
307
|
+
const tcpOnRequest = token === undefined
|
|
308
|
+
? onRequest
|
|
309
|
+
: (req, res) => {
|
|
310
|
+
if (!isAuthorized(req, token)) {
|
|
311
|
+
writeUnauthorized(res);
|
|
312
|
+
return;
|
|
313
|
+
}
|
|
314
|
+
onRequest(req, res);
|
|
315
|
+
};
|
|
316
|
+
const tcpOnUpgrade = token === undefined
|
|
317
|
+
? onUpgrade
|
|
318
|
+
: (req, socket, head) => {
|
|
319
|
+
if (!isAuthorized(req, token)) {
|
|
320
|
+
destroyUpgradeUnauthorized(socket);
|
|
321
|
+
return;
|
|
322
|
+
}
|
|
323
|
+
onUpgrade(req, socket, head);
|
|
324
|
+
};
|
|
325
|
+
tcpServer = createServer(tcpOnRequest);
|
|
326
|
+
tcpServer.on('upgrade', tcpOnUpgrade);
|
|
260
327
|
bindErrorHandler(tcpServer);
|
|
261
328
|
tcpServer.listen(parsed.port, parsed.host);
|
|
262
329
|
}
|
package/dist/daemon/crtrd.js
CHANGED
|
@@ -50,6 +50,7 @@ import { listNodes, migrateLegacyPidIdentities, } from '../core/canvas/index.js'
|
|
|
50
50
|
import { createApiServer } from './api/server.js';
|
|
51
51
|
import { reconcileUndeliveredTickets } from './human/sweep.js';
|
|
52
52
|
import { recoverRunningActionDeliveries } from '../core/canvas/human-deliveries.js';
|
|
53
|
+
import { recoverRunningOutcomeDeliveries } from '../core/canvas/node-outcome-deliveries.js';
|
|
53
54
|
import { sweepReviews } from './review/sweep.js';
|
|
54
55
|
import { pidfilePath } from './pidfile.js';
|
|
55
56
|
import { rendererWarning } from '../core/runtime/package-health.js';
|
|
@@ -57,8 +58,10 @@ import { DEFAULT_INTERVAL_MS } from './supervise-cadence.js';
|
|
|
57
58
|
import { BrokerSupervisionReconciler } from './reconcilers/broker-supervision.js';
|
|
58
59
|
import { ControllerDeathReconciler } from './reconcilers/controller-death.js';
|
|
59
60
|
import { NodeLifecycleReconciler } from './reconcilers/node-lifecycle/tick.js';
|
|
61
|
+
import { NodeDeadlineReconciler } from './reconcilers/node-deadline.js';
|
|
60
62
|
import { CronLaneReconciler } from './reconcilers/cron-lane.js';
|
|
61
63
|
import { HumanDeliveryLaneReconciler } from './reconcilers/human-delivery-lane.js';
|
|
64
|
+
import { NodeOutcomeDeliveryLaneReconciler } from './reconcilers/node-outcome-delivery-lane.js';
|
|
62
65
|
import { PendingReviewSubmitReconciler } from './reconcilers/pending-review-submit.js';
|
|
63
66
|
import { StorageMaintenanceReconciler } from './reconcilers/storage-maintenance.js';
|
|
64
67
|
import { BashDeadlineReconciler } from './reconcilers/bash-deadline.js';
|
|
@@ -272,8 +275,10 @@ const directTickLifecycle = {
|
|
|
272
275
|
const brokerSupervision = new BrokerSupervisionReconciler();
|
|
273
276
|
const controllerDeath = new ControllerDeathReconciler();
|
|
274
277
|
const nodeLifecycle = new NodeLifecycleReconciler();
|
|
278
|
+
const nodeDeadline = new NodeDeadlineReconciler();
|
|
275
279
|
const cronLane = new CronLaneReconciler();
|
|
276
280
|
const humanDeliveryLane = new HumanDeliveryLaneReconciler();
|
|
281
|
+
const nodeOutcomeDeliveryLane = new NodeOutcomeDeliveryLaneReconciler();
|
|
277
282
|
const pendingReviewSubmit = new PendingReviewSubmitReconciler();
|
|
278
283
|
const storageMaintenance = new StorageMaintenanceReconciler();
|
|
279
284
|
const bashDeadline = new BashDeadlineReconciler();
|
|
@@ -301,8 +306,12 @@ export async function superviseTick(now = Date.now(), lifecycle = directTickLife
|
|
|
301
306
|
await brokerSupervision.run(now, { rows, fleet, lifecycle });
|
|
302
307
|
controllerDeath.run(now, { rows });
|
|
303
308
|
const frozen = await nodeLifecycle.run(now, { rows: [...rows, ...terminating], fleet });
|
|
309
|
+
// The universal backstop: a node past its deadline with no outcome is
|
|
310
|
+
// canceled and terminalized here, after the lifecycle pass has had its say.
|
|
311
|
+
nodeDeadline.run(now);
|
|
304
312
|
cronLane.run(now, { lifecycle });
|
|
305
313
|
humanDeliveryLane.run(now, { lifecycle });
|
|
314
|
+
nodeOutcomeDeliveryLane.run(now, { lifecycle });
|
|
306
315
|
pendingReviewSubmit.run(now, { lifecycle });
|
|
307
316
|
storageMaintenance.run(now, { lifecycle, fleet });
|
|
308
317
|
await bashDeadline.run(now, { rows });
|
|
@@ -640,6 +649,9 @@ export async function runDaemon(opts = {}) {
|
|
|
640
649
|
await reconcileUndeliveredTickets();
|
|
641
650
|
// Unlike a cron one-shot, a completion is at-least-once: destinations deduplicate request IDs, so a dead daemon's running delivery is re-queued immediately.
|
|
642
651
|
recoverRunningActionDeliveries(Date.now());
|
|
652
|
+
// Same at-least-once contract for node outcome deliveries: a delivery the
|
|
653
|
+
// dead daemon had marked running is re-queued rather than stranded.
|
|
654
|
+
recoverRunningOutcomeDeliveries(Date.now());
|
|
643
655
|
});
|
|
644
656
|
// Boot reconciliation clears stale process identities before any tick can
|
|
645
657
|
// inspect them, then migrates legacy identities only with proven same-boot
|
package/dist/daemon/fleet.d.ts
CHANGED
|
@@ -1,13 +1,7 @@
|
|
|
1
1
|
import { type NodeMeta } from '../core/canvas/index.js';
|
|
2
2
|
import { type FleetEntry, type FleetExitStatus, type FleetRegistry } from '../core/runtime/fleet.js';
|
|
3
|
+
import type { NodeOutcomeDetailV1, TerminalReason } from '../core/canvas/types.js';
|
|
3
4
|
import type { HostHandle } from '../core/runtime/host.js';
|
|
4
|
-
/** Surface a vehicle that never booted.
|
|
5
|
-
*
|
|
6
|
-
* A pi that dies before its first session_start (so `pi_session_id` was never
|
|
7
|
-
* recorded) is invisible to its parent — the spawn returned optimistically.
|
|
8
|
-
* When the daemon observes that death it errors LOUDLY up the spine: an
|
|
9
|
-
* urgent push so the parent learns the child failed to launch instead of just
|
|
10
|
-
* seeing a silent `dead`. */
|
|
11
5
|
export declare function surfaceBootFailure(meta: NodeMeta): Promise<void>;
|
|
12
6
|
/** Whether a dead relaunch should resume the saved session strictly, or retry
|
|
13
7
|
* as a fresh cycle instead. A cycling attempt (cycle_pending) that dies before
|
|
@@ -69,6 +63,8 @@ export declare class DaemonFleet implements FleetRegistry {
|
|
|
69
63
|
reserveSlot(nodeId: string): boolean;
|
|
70
64
|
releaseReservation(nodeId: string): void;
|
|
71
65
|
entries(): Iterable<[string, FleetEntry]>;
|
|
66
|
+
interruptForProfilePause(nodeId: string, pid?: number | undefined): void;
|
|
67
|
+
hasProfilePauseInterruption(nodeId: string): boolean;
|
|
72
68
|
forget(nodeId: string): void;
|
|
73
69
|
onChildExit(nodeId: string, status: FleetExitStatus): void;
|
|
74
70
|
deliverExit(nodeId: string, status: FleetExitStatus): void;
|
|
@@ -78,6 +74,6 @@ export declare class DaemonFleet implements FleetRegistry {
|
|
|
78
74
|
* with no fleet entry. Convert that otherwise-silent strand into the same
|
|
79
75
|
* terminal outcome as a boot failure. The fresh read avoids duplicating
|
|
80
76
|
* reviveNode's launch-failure crash path. */
|
|
81
|
-
terminalizeIfStillNonTerminal(nodeId: string, event: string, error: unknown): Promise<boolean>;
|
|
77
|
+
terminalizeIfStillNonTerminal(nodeId: string, event: string, error: unknown, reason: TerminalReason, detail: NodeOutcomeDetailV1): Promise<boolean>;
|
|
82
78
|
dispose(): void;
|
|
83
79
|
}
|
package/dist/daemon/fleet.js
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
// before every deliberate exit, and that durable state is robust to every kill
|
|
13
13
|
// path. Codes/signals are logged for diagnostics only.
|
|
14
14
|
import { getNode, getRow, subscribersOf } from '../core/canvas/index.js';
|
|
15
|
+
import { isProfilePaused } from '../core/profiles/manifest.js';
|
|
15
16
|
import { fullName } from '../core/canvas/labels.js';
|
|
16
17
|
import { isSafeNodeId } from '../core/canvas/paths.js';
|
|
17
18
|
import { transition } from '../core/runtime/lifecycle.js';
|
|
@@ -33,14 +34,16 @@ import { brokerThresholdsForDaemon } from '../core/runtime/fleet.js';
|
|
|
33
34
|
* When the daemon observes that death it errors LOUDLY up the spine: an
|
|
34
35
|
* urgent push so the parent learns the child failed to launch instead of just
|
|
35
36
|
* seeing a silent `dead`. */
|
|
36
|
-
|
|
37
|
-
|
|
37
|
+
function bootFailureBody(meta) {
|
|
38
|
+
return (`⚠ Spawn failed — \`${meta.name}\` (${meta.kind}) never started.\n\n` +
|
|
38
39
|
`Its pi vehicle exited before the session came up (no pi_session_id was ever ` +
|
|
39
40
|
`recorded), so the node produced no output. This is almost always a transient ` +
|
|
40
41
|
`launch failure — e.g. resource pressure when several nodes boot at once — not ` +
|
|
41
42
|
`a fault in the task itself.\n\n` +
|
|
42
|
-
`If the work still needs doing, re-spawn it; if spawns keep dying, spawn fewer at a time
|
|
43
|
-
|
|
43
|
+
`If the work still needs doing, re-spawn it; if spawns keep dying, spawn fewer at a time.`);
|
|
44
|
+
}
|
|
45
|
+
export async function surfaceBootFailure(meta) {
|
|
46
|
+
await pushUrgent(meta.node_id, bootFailureBody(meta), { from: meta.node_id });
|
|
44
47
|
}
|
|
45
48
|
async function surfaceCrashLoop(meta) {
|
|
46
49
|
const transportPoison = readTransportPoisonVerdict(meta.node_id);
|
|
@@ -131,6 +134,9 @@ export class DaemonFleet {
|
|
|
131
134
|
#map = new Map();
|
|
132
135
|
/** Slots claimed by launches that have not registered a broker yet. */
|
|
133
136
|
#reservations = new Set();
|
|
137
|
+
/** Brokers deliberately interrupted by profile pause. This belongs to the
|
|
138
|
+
* exact child, so a resume cannot turn its later SIGTERM exit into a crash. */
|
|
139
|
+
#profilePauseInterruptions = new Set();
|
|
134
140
|
#lastCapacityLogAt = Number.NEGATIVE_INFINITY;
|
|
135
141
|
constructor(opts) {
|
|
136
142
|
this.#epoch = opts.epoch;
|
|
@@ -191,6 +197,21 @@ export class DaemonFleet {
|
|
|
191
197
|
entries() {
|
|
192
198
|
return this.#map.entries();
|
|
193
199
|
}
|
|
200
|
+
interruptForProfilePause(nodeId, pid = this.#map.get(nodeId)?.pid) {
|
|
201
|
+
if (pid === undefined)
|
|
202
|
+
return;
|
|
203
|
+
this.#profilePauseInterruptions.add(nodeId);
|
|
204
|
+
try {
|
|
205
|
+
process.kill(pid, 'SIGTERM');
|
|
206
|
+
}
|
|
207
|
+
catch {
|
|
208
|
+
// A live handle or identity-checked PID can exit between its observation
|
|
209
|
+
// and signal. The marker still makes that exact observed exit harmless.
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
hasProfilePauseInterruption(nodeId) {
|
|
213
|
+
return this.#profilePauseInterruptions.has(nodeId);
|
|
214
|
+
}
|
|
194
215
|
forget(nodeId) {
|
|
195
216
|
this.#map.delete(nodeId);
|
|
196
217
|
this.#reservations.delete(nodeId);
|
|
@@ -217,12 +238,14 @@ export class DaemonFleet {
|
|
|
217
238
|
* with no fleet entry. Convert that otherwise-silent strand into the same
|
|
218
239
|
* terminal outcome as a boot failure. The fresh read avoids duplicating
|
|
219
240
|
* reviveNode's launch-failure crash path. */
|
|
220
|
-
async terminalizeIfStillNonTerminal(nodeId, event, error) {
|
|
241
|
+
async terminalizeIfStillNonTerminal(nodeId, event, error, reason, detail) {
|
|
221
242
|
const meta = getNode(nodeId);
|
|
222
243
|
if (meta === null || meta.status === 'done' || meta.status === 'canceled' || meta.status === 'dead')
|
|
223
244
|
return false;
|
|
224
245
|
await this.#terminalize(nodeId, meta, {
|
|
225
246
|
event,
|
|
247
|
+
reason,
|
|
248
|
+
detail,
|
|
226
249
|
wake: `Child crashed — ${fullName(meta)} (${nodeId}) could not be relaunched and is now dead. ` +
|
|
227
250
|
`It stays dead until you revive it — \`crtr node lifecycle revive ${nodeId}\`.`,
|
|
228
251
|
surface: () => surfaceRecoveryFailure(meta, error),
|
|
@@ -249,6 +272,17 @@ export class DaemonFleet {
|
|
|
249
272
|
}
|
|
250
273
|
async #decideAndEnact(nodeId) {
|
|
251
274
|
const meta = getNode(nodeId);
|
|
275
|
+
if (meta !== null && isProfilePaused(meta.profile_id)) {
|
|
276
|
+
clearRespawnPolicy(nodeId);
|
|
277
|
+
return 'forget';
|
|
278
|
+
}
|
|
279
|
+
// Consume one unpaused policy pass for the exact broker profile pause
|
|
280
|
+
// interrupted. The following tick owns ordinary recovery regardless of
|
|
281
|
+
// which dead-row verdict the still-recorded PID would otherwise select.
|
|
282
|
+
if (this.#profilePauseInterruptions.delete(nodeId)) {
|
|
283
|
+
clearRespawnPolicy(nodeId);
|
|
284
|
+
return 'forget';
|
|
285
|
+
}
|
|
252
286
|
// Two durable witnesses mean the dead broker had a turn in flight. A dirty
|
|
253
287
|
// death leaves the busy marker behind; a bounded graceful abort clears busy
|
|
254
288
|
// as the run settles, then writes the clean-abort certificate. Both need the
|
|
@@ -284,6 +318,8 @@ export class DaemonFleet {
|
|
|
284
318
|
case 'boot-failure':
|
|
285
319
|
await this.#terminalize(nodeId, meta, {
|
|
286
320
|
event: 'broker.boot.failed',
|
|
321
|
+
reason: 'boot_failed',
|
|
322
|
+
detail: { schema: 'crtr.node-outcome-detail/v1', message: bootFailureBody(meta) },
|
|
287
323
|
wake: `Child crashed — ${fullName(meta)} (${nodeId}) exited before its session ever came up and is now dead. ` +
|
|
288
324
|
`It stays dead until you revive it — \`crtr node lifecycle revive ${nodeId}\`.`,
|
|
289
325
|
});
|
|
@@ -328,10 +364,13 @@ export class DaemonFleet {
|
|
|
328
364
|
? recordRespawnPolicy(row, this.#now())
|
|
329
365
|
: { failures: priorFailures, notBefore: row.respawn_not_before, exhausted: priorFailures >= MAX_RESPAWN_ATTEMPTS };
|
|
330
366
|
if (attempt.exhausted) {
|
|
367
|
+
const wake = `Child crashed — ${fullName(meta)} (${nodeId}) kept dying without settling a turn (${attempt.failures} consecutive ` +
|
|
368
|
+
`launches) and is now dead. It stays dead until you revive it — \`crtr node lifecycle revive ${nodeId}\`.`;
|
|
331
369
|
await this.#terminalize(nodeId, meta, {
|
|
332
370
|
event: 'broker.respawn.exhausted',
|
|
333
|
-
|
|
334
|
-
|
|
371
|
+
reason: 'crash_looped',
|
|
372
|
+
detail: { schema: 'crtr.node-outcome-detail/v1', message: wake, respawn_failures: attempt.failures },
|
|
373
|
+
wake,
|
|
335
374
|
...(meta.pi_session_id != null ? { surface: () => surfaceCrashLoop(meta) } : {}),
|
|
336
375
|
});
|
|
337
376
|
return;
|
|
@@ -392,7 +431,7 @@ export class DaemonFleet {
|
|
|
392
431
|
// Launch failures terminalize inside reviveNode, but its preflight
|
|
393
432
|
// refusals (for example cwd_missing) happen before that catch. Re-read
|
|
394
433
|
// before terminalizing so the launch path's settled terminal row wins.
|
|
395
|
-
await this.terminalizeIfStillNonTerminal(nodeId, 'broker.respawn.refused', err);
|
|
434
|
+
await this.terminalizeIfStillNonTerminal(nodeId, 'broker.respawn.refused', err, 'launch_failed', { schema: 'crtr.node-outcome-detail/v1', message: err instanceof Error ? err.message : String(err) });
|
|
396
435
|
}
|
|
397
436
|
}
|
|
398
437
|
/** Terminal outcome shared by table row 8 and respawn exhaustion. */
|
|
@@ -402,7 +441,7 @@ export class DaemonFleet {
|
|
|
402
441
|
event: opts.event,
|
|
403
442
|
...(isSafeNodeId(nodeId) ? { node_id: nodeId } : { fields: { affected_node_id: nodeId } }),
|
|
404
443
|
});
|
|
405
|
-
transition(nodeId, 'crash');
|
|
444
|
+
transition(nodeId, 'crash', { reason: opts.reason, outcome: { detail: opts.detail } });
|
|
406
445
|
try {
|
|
407
446
|
await (opts.surface ?? (() => surfaceBootFailure(meta)))();
|
|
408
447
|
}
|
|
@@ -9,6 +9,7 @@ import { reviveNode } from '../../core/runtime/revive.js';
|
|
|
9
9
|
import { hasNoNaturalCycle } from '../../core/runtime/revive-all.js';
|
|
10
10
|
import { ReviewOperationError } from '../../core/review/types.js';
|
|
11
11
|
import { formatCard } from '../../shared/generated-context.js';
|
|
12
|
+
import { isProfilePaused } from '../../core/profiles/manifest.js';
|
|
12
13
|
function deliveryUnroutable(nodeId, reason) {
|
|
13
14
|
throw new ReviewOperationError('delivery_unroutable', reason === 'origin_missing' ? 'message target was not found' : 'message target has no natural delivery cycle', { node_id: nodeId, reason });
|
|
14
15
|
}
|
|
@@ -33,7 +34,7 @@ export async function deliverNodeMessage(args) {
|
|
|
33
34
|
let target = getNode(args.node_id);
|
|
34
35
|
if (target === null)
|
|
35
36
|
deliveryUnroutable(args.node_id, 'origin_missing');
|
|
36
|
-
if (args.mode === 'interactive' && isBrokerLive(target)) {
|
|
37
|
+
if (args.mode === 'interactive' && !isProfilePaused(target.profile_id) && isBrokerLive(target)) {
|
|
37
38
|
try {
|
|
38
39
|
// Inbox delivery supplies the raw body as an entry in its outer inbox
|
|
39
40
|
// card; only live delivery needs this message's own envelope.
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { type OutcomeDeliveryRecord, type OutcomeDeliverySettlement } from '../../core/canvas/node-outcome-deliveries.js';
|
|
2
|
+
export declare const OUTCOME_DELIVERY_TIMEOUT_MS = 120000;
|
|
3
|
+
export declare const OUTCOME_DELIVERY_STDERR_TAIL_BYTES: number;
|
|
4
|
+
export interface OutcomeDeliveryProcessOutcome {
|
|
5
|
+
exitCode: number | null;
|
|
6
|
+
signal: string | null;
|
|
7
|
+
timedOut: boolean;
|
|
8
|
+
spawnError?: Error;
|
|
9
|
+
stderr: string;
|
|
10
|
+
}
|
|
11
|
+
/** The delivery target's exit code is its complete acknowledgment protocol. */
|
|
12
|
+
export declare function outcomeDeliverySettlementForOutcome(outcome: OutcomeDeliveryProcessOutcome, attempt: number, now: number): OutcomeDeliverySettlement;
|
|
13
|
+
/** Spawn and settle one already-claimed outcome delivery. */
|
|
14
|
+
export declare function deliverOutcome(delivery: OutcomeDeliveryRecord, claimOwner: string): Promise<void>;
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import { spawn } from 'node:child_process';
|
|
2
|
+
import { envHomeOverride } from '../../shared/env.js';
|
|
3
|
+
import { outcomeDeliveryBackoffMs, settleOutcomeDelivery, } from '../../core/canvas/node-outcome-deliveries.js';
|
|
4
|
+
import { killProcessGroup } from '../../core/canvas/pid.js';
|
|
5
|
+
import { buildOperationalEnvBase } from '../../core/runtime/spawn-env.js';
|
|
6
|
+
export const OUTCOME_DELIVERY_TIMEOUT_MS = 120_000;
|
|
7
|
+
const OUTCOME_DELIVERY_KILL_GRACE_MS = 2_000;
|
|
8
|
+
export const OUTCOME_DELIVERY_STDERR_TAIL_BYTES = 4 * 1024;
|
|
9
|
+
/** The delivery target's exit code is its complete acknowledgment protocol. */
|
|
10
|
+
export function outcomeDeliverySettlementForOutcome(outcome, attempt, now) {
|
|
11
|
+
if (outcome.spawnError !== undefined) {
|
|
12
|
+
return {
|
|
13
|
+
state: 'pending',
|
|
14
|
+
nextAttemptAt: now + outcomeDeliveryBackoffMs(attempt),
|
|
15
|
+
failure: failure('spawn_error', { message: outcome.spawnError.message, stderr: outcome.stderr }),
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
if (outcome.timedOut) {
|
|
19
|
+
return {
|
|
20
|
+
state: 'pending',
|
|
21
|
+
nextAttemptAt: now + outcomeDeliveryBackoffMs(attempt),
|
|
22
|
+
failure: failure('timeout', { ...(outcome.signal === null ? {} : { signal: outcome.signal }), stderr: outcome.stderr }),
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
if (outcome.signal !== null) {
|
|
26
|
+
return {
|
|
27
|
+
state: 'pending',
|
|
28
|
+
nextAttemptAt: now + outcomeDeliveryBackoffMs(attempt),
|
|
29
|
+
failure: failure('signal', { signal: outcome.signal, stderr: outcome.stderr }),
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
if (outcome.exitCode === 0)
|
|
33
|
+
return { state: 'accepted' };
|
|
34
|
+
if (outcome.exitCode === 78) {
|
|
35
|
+
return { state: 'permanent_failed', failure: failure('exit', { exitCode: 78, stderr: outcome.stderr }) };
|
|
36
|
+
}
|
|
37
|
+
return {
|
|
38
|
+
state: 'pending',
|
|
39
|
+
nextAttemptAt: now + outcomeDeliveryBackoffMs(attempt),
|
|
40
|
+
failure: failure('exit', { exitCode: outcome.exitCode ?? undefined, stderr: outcome.stderr }),
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
function failure(kind, fields) {
|
|
44
|
+
return { kind, ...fields };
|
|
45
|
+
}
|
|
46
|
+
function stderrTailCapture() {
|
|
47
|
+
let tail = Buffer.alloc(0);
|
|
48
|
+
return {
|
|
49
|
+
take(chunk) {
|
|
50
|
+
tail = chunk.length >= OUTCOME_DELIVERY_STDERR_TAIL_BYTES
|
|
51
|
+
? chunk.subarray(chunk.length - OUTCOME_DELIVERY_STDERR_TAIL_BYTES)
|
|
52
|
+
: Buffer.concat([tail, chunk]).subarray(Math.max(0, tail.length + chunk.length - OUTCOME_DELIVERY_STDERR_TAIL_BYTES));
|
|
53
|
+
},
|
|
54
|
+
decode() {
|
|
55
|
+
return tail.toString('utf8');
|
|
56
|
+
},
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
/** The stored document is immutable; attempt is its only per-delivery mutation. */
|
|
60
|
+
function documentForAttempt(document, attempt) {
|
|
61
|
+
const outcome = document;
|
|
62
|
+
const delivery = outcome.delivery;
|
|
63
|
+
return JSON.stringify({ ...outcome, delivery: { ...delivery, attempt } });
|
|
64
|
+
}
|
|
65
|
+
/** Spawn and settle one already-claimed outcome delivery. */
|
|
66
|
+
export async function deliverOutcome(delivery, claimOwner) {
|
|
67
|
+
const stderr = stderrTailCapture();
|
|
68
|
+
let document;
|
|
69
|
+
try {
|
|
70
|
+
document = documentForAttempt(delivery.document, delivery.attempt);
|
|
71
|
+
}
|
|
72
|
+
catch (error) {
|
|
73
|
+
settle(delivery, claimOwner, outcomeDeliverySettlementForOutcome({
|
|
74
|
+
exitCode: null,
|
|
75
|
+
signal: null,
|
|
76
|
+
timedOut: false,
|
|
77
|
+
spawnError: error,
|
|
78
|
+
stderr: stderr.decode(),
|
|
79
|
+
}, delivery.attempt, Date.now()));
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
let child;
|
|
83
|
+
try {
|
|
84
|
+
child = spawn(delivery.argv[0], delivery.argv.slice(1), {
|
|
85
|
+
cwd: delivery.cwd,
|
|
86
|
+
env: outcomeDeliveryEnv(delivery.cwd),
|
|
87
|
+
detached: true,
|
|
88
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
catch (error) {
|
|
92
|
+
settle(delivery, claimOwner, outcomeDeliverySettlementForOutcome({
|
|
93
|
+
exitCode: null,
|
|
94
|
+
signal: null,
|
|
95
|
+
timedOut: false,
|
|
96
|
+
spawnError: error,
|
|
97
|
+
stderr: stderr.decode(),
|
|
98
|
+
}, delivery.attempt, Date.now()));
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
await new Promise((resolve) => {
|
|
102
|
+
let settled = false;
|
|
103
|
+
let timedOut = false;
|
|
104
|
+
let killTimer;
|
|
105
|
+
let timeout;
|
|
106
|
+
const finish = (outcome) => {
|
|
107
|
+
if (settled)
|
|
108
|
+
return;
|
|
109
|
+
settled = true;
|
|
110
|
+
if (timeout !== undefined)
|
|
111
|
+
clearTimeout(timeout);
|
|
112
|
+
// A timed-out leader can exit while a SIGTERM-ignoring group member remains.
|
|
113
|
+
if (!outcome.timedOut && killTimer !== undefined)
|
|
114
|
+
clearTimeout(killTimer);
|
|
115
|
+
child.stdin?.destroy();
|
|
116
|
+
child.stdout?.destroy();
|
|
117
|
+
child.stderr?.destroy();
|
|
118
|
+
settle(delivery, claimOwner, outcomeDeliverySettlementForOutcome(outcome, delivery.attempt, Date.now()));
|
|
119
|
+
resolve();
|
|
120
|
+
};
|
|
121
|
+
child.stdout?.on('data', () => { }); // Drain output; stdout is never a control channel.
|
|
122
|
+
child.stderr?.on('data', (chunk) => stderr.take(chunk));
|
|
123
|
+
child.stdin?.on('error', () => { }); // The child may exit before consuming stdin; its exit code is authoritative.
|
|
124
|
+
child.on('error', (error) => {
|
|
125
|
+
finish({ exitCode: null, signal: null, timedOut: false, spawnError: error, stderr: stderr.decode() });
|
|
126
|
+
});
|
|
127
|
+
child.on('exit', (exitCode, signal) => {
|
|
128
|
+
finish({ exitCode, signal, timedOut, stderr: stderr.decode() });
|
|
129
|
+
});
|
|
130
|
+
const killWithEscalation = () => {
|
|
131
|
+
if (child.pid === undefined)
|
|
132
|
+
return;
|
|
133
|
+
killProcessGroup(child.pid);
|
|
134
|
+
killTimer = setTimeout(() => killProcessGroup(child.pid, 'SIGKILL'), OUTCOME_DELIVERY_KILL_GRACE_MS);
|
|
135
|
+
killTimer.unref?.();
|
|
136
|
+
};
|
|
137
|
+
timeout = setTimeout(() => {
|
|
138
|
+
timedOut = true;
|
|
139
|
+
killWithEscalation();
|
|
140
|
+
}, OUTCOME_DELIVERY_TIMEOUT_MS);
|
|
141
|
+
timeout.unref?.();
|
|
142
|
+
child.stdin?.end(document, 'utf8');
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
/** Outcome targets cross the same consent boundary as every other crouter spawn. */
|
|
146
|
+
function outcomeDeliveryEnv(cwd) {
|
|
147
|
+
const env = buildOperationalEnvBase({ targetCwd: cwd, targetProfileId: null });
|
|
148
|
+
const canvasHome = envHomeOverride();
|
|
149
|
+
if (canvasHome !== undefined && canvasHome !== '')
|
|
150
|
+
env['CRTR_HOME'] = canvasHome;
|
|
151
|
+
return env;
|
|
152
|
+
}
|
|
153
|
+
function settle(delivery, claimOwner, settlement) {
|
|
154
|
+
settleOutcomeDelivery(delivery.nodeId, delivery.attempt, claimOwner, settlement, Date.now());
|
|
155
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { NodeMeta } from '../core/canvas/types.js';
|
|
2
|
+
import type { PendingPark } from './park-pending.js';
|
|
3
|
+
/** A main-session write after parking was armed is durable evidence of live
|
|
4
|
+
* input even when the turn has since settled and the viewer detached. */
|
|
5
|
+
export declare function mainSessionActivitySince(sessionFile: string | null | undefined, startedAt: number): boolean;
|
|
6
|
+
/** One verdict shared by normal completion and grace expiry. A broker records
|
|
7
|
+
* accepted live input against the process-local marker before it drives Pi, so
|
|
8
|
+
* the verdict cannot race a viewer, live delivery, or durable wake. */
|
|
9
|
+
export declare function shouldParkAfterSummary(nodeId: string, node: Pick<NodeMeta, 'lifecycle' | 'pi_session_file'>, pending: PendingPark): boolean;
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { statSync } from 'node:fs';
|
|
2
|
+
import { emitEvent } from '../core/events/emit.js';
|
|
3
|
+
import { hasWakeMailSince } from '../core/feed/inbox.js';
|
|
4
|
+
import { isBusy } from '../core/runtime/busy.js';
|
|
5
|
+
import { parkEvent } from './reconcilers/live-obligation.js';
|
|
6
|
+
/** A main-session write after parking was armed is durable evidence of live
|
|
7
|
+
* input even when the turn has since settled and the viewer detached. */
|
|
8
|
+
export function mainSessionActivitySince(sessionFile, startedAt) {
|
|
9
|
+
if (sessionFile === null || sessionFile === undefined)
|
|
10
|
+
return false;
|
|
11
|
+
try {
|
|
12
|
+
return statSync(sessionFile).mtimeMs >= startedAt;
|
|
13
|
+
}
|
|
14
|
+
catch (error) {
|
|
15
|
+
if (error.code === 'ENOENT')
|
|
16
|
+
return false;
|
|
17
|
+
emitEvent({ level: 'warn', event: 'broker.park.session_activity_scan_failed', error });
|
|
18
|
+
return true;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
/** One verdict shared by normal completion and grace expiry. A broker records
|
|
22
|
+
* accepted live input against the process-local marker before it drives Pi, so
|
|
23
|
+
* the verdict cannot race a viewer, live delivery, or durable wake. */
|
|
24
|
+
export function shouldParkAfterSummary(nodeId, node, pending) {
|
|
25
|
+
return !pending.activity
|
|
26
|
+
&& !isBusy(nodeId)
|
|
27
|
+
&& !mainSessionActivitySince(node.pi_session_file, pending.startedAt)
|
|
28
|
+
&& !hasWakeMailSince(nodeId, pending.startedAt)
|
|
29
|
+
&& parkEvent(nodeId, node.lifecycle) === 'park';
|
|
30
|
+
}
|
|
@@ -1,13 +1,18 @@
|
|
|
1
|
+
export type PendingPark = {
|
|
2
|
+
startedAt: number;
|
|
3
|
+
activity: boolean;
|
|
4
|
+
};
|
|
1
5
|
/** Arm a pending park for `nodeId`. The caller delivers the parking prompt. */
|
|
2
6
|
export declare function beginPendingPark(nodeId: string, now: number): void;
|
|
7
|
+
/** Record main-engine input accepted while the isolated turn is in flight. */
|
|
8
|
+
export declare function recordPendingParkActivity(nodeId: string): boolean;
|
|
3
9
|
export declare function isPendingPark(nodeId: string): boolean;
|
|
4
10
|
/** When the pending park was armed, or undefined when none is armed. */
|
|
5
11
|
export declare function pendingParkStartedAt(nodeId: string): number | undefined;
|
|
6
|
-
/** Delete-and-report
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
|
|
10
|
-
export declare function consumePendingPark(nodeId: string): boolean;
|
|
12
|
+
/** Delete-and-report the pending marker. Both the supervision tick and the
|
|
13
|
+
* settle handler run on the daemon thread with no await between their check and
|
|
14
|
+
* this call, so exactly one of them ever enacts a given park. */
|
|
15
|
+
export declare function consumePendingPark(nodeId: string): PendingPark | undefined;
|
|
11
16
|
/** Every node with a parking turn in flight — the reconciler's sweep drops the
|
|
12
17
|
* ones whose broker has left the fleet. */
|
|
13
18
|
export declare function pendingParkIds(): string[];
|
|
@@ -15,25 +15,35 @@
|
|
|
15
15
|
// and nothing needs repair. A durable in-flight flag would instead need its own
|
|
16
16
|
// reconciliation on every daemon start to avoid stranding a node that never got
|
|
17
17
|
// its prompt.
|
|
18
|
-
/** node id → the
|
|
18
|
+
/** node id → the pending parking turn and any accepted main-engine input. */
|
|
19
19
|
const pending = new Map();
|
|
20
20
|
/** Arm a pending park for `nodeId`. The caller delivers the parking prompt. */
|
|
21
21
|
export function beginPendingPark(nodeId, now) {
|
|
22
|
-
pending.set(nodeId, now);
|
|
22
|
+
pending.set(nodeId, { startedAt: now, activity: false });
|
|
23
|
+
}
|
|
24
|
+
/** Record main-engine input accepted while the isolated turn is in flight. */
|
|
25
|
+
export function recordPendingParkActivity(nodeId) {
|
|
26
|
+
const marker = pending.get(nodeId);
|
|
27
|
+
if (marker === undefined)
|
|
28
|
+
return false;
|
|
29
|
+
marker.activity = true;
|
|
30
|
+
return true;
|
|
23
31
|
}
|
|
24
32
|
export function isPendingPark(nodeId) {
|
|
25
33
|
return pending.has(nodeId);
|
|
26
34
|
}
|
|
27
35
|
/** When the pending park was armed, or undefined when none is armed. */
|
|
28
36
|
export function pendingParkStartedAt(nodeId) {
|
|
29
|
-
return pending.get(nodeId);
|
|
37
|
+
return pending.get(nodeId)?.startedAt;
|
|
30
38
|
}
|
|
31
|
-
/** Delete-and-report
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
* them ever enacts a given park. */
|
|
39
|
+
/** Delete-and-report the pending marker. Both the supervision tick and the
|
|
40
|
+
* settle handler run on the daemon thread with no await between their check and
|
|
41
|
+
* this call, so exactly one of them ever enacts a given park. */
|
|
35
42
|
export function consumePendingPark(nodeId) {
|
|
36
|
-
|
|
43
|
+
const marker = pending.get(nodeId);
|
|
44
|
+
if (marker !== undefined)
|
|
45
|
+
pending.delete(nodeId);
|
|
46
|
+
return marker;
|
|
37
47
|
}
|
|
38
48
|
/** Every node with a parking turn in flight — the reconciler's sweep drops the
|
|
39
49
|
* ones whose broker has left the fleet. */
|