@north-light/crouter 0.3.281 → 0.3.283
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/client.d.ts +19 -3
- package/dist/api/client.js +34 -0
- package/dist/api/dto/broker-ops.d.ts +11 -0
- package/dist/api/dto/broker.d.ts +3 -8
- package/dist/api/dto/common.d.ts +1 -1
- package/dist/api/dto/messages.d.ts +2 -2
- package/dist/api/dto/node-outcomes.d.ts +80 -0
- package/dist/api/dto/node-outcomes.js +2 -0
- package/dist/api/dto/nodes.d.ts +9 -0
- package/dist/api/dto/profiles.d.ts +5 -0
- package/dist/api/dto/reports.d.ts +45 -0
- package/dist/api/index.d.ts +1 -0
- package/dist/api/index.js +1 -0
- package/dist/api/plugin-manifest-schema.d.ts +208 -0
- package/dist/api/plugin-manifest-schema.js +23 -0
- package/dist/api/routes.d.ts +7 -0
- package/dist/api/routes.js +7 -0
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js +24 -3
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +31 -4
- package/dist/cli.js +8 -2
- package/dist/clients/attach/overlays/graph.d.ts +5 -0
- package/dist/clients/attach/overlays/graph.js +65 -47
- package/dist/clients/attach/overlays/pickers.js +8 -9
- package/dist/clients/attach/render/card-presentation.js +1 -1
- package/dist/clients/attach/viewer.js +1286 -1260
- package/dist/commands/api-client.js +22 -23
- package/dist/commands/node/create.js +38 -3
- package/dist/commands/node/inspect.js +3 -7
- package/dist/commands/node/message.js +3 -3
- package/dist/commands/node/outcome.d.ts +2 -0
- package/dist/commands/node/outcome.js +134 -0
- package/dist/commands/node.js +2 -1
- package/dist/commands/pkg/browse/catalog.js +1 -1
- package/dist/commands/pkg/plugin-manage.d.ts +46 -1
- package/dist/commands/pkg/plugin-manage.js +170 -20
- package/dist/commands/profile/pause.js +17 -14
- package/dist/commands/push.js +102 -1
- package/dist/commands/sys/__tests__/setup-core.test.js +4 -1
- package/dist/commands/sys/panels/broker-limits-panel.js +7 -1
- package/dist/commands/sys/panels/models-panel.js +2 -2
- package/dist/commands/sys/settings-shell.d.ts +1 -1
- package/dist/commands/sys/settings-shell.js +6 -0
- package/dist/commands/sys/settings.js +3 -3
- package/dist/core/__tests__/fixtures/fake-engine.d.ts +2 -0
- package/dist/core/__tests__/fixtures/fake-engine.js +42 -4
- package/dist/core/__tests__/helpers/harness.d.ts +1 -0
- package/dist/core/__tests__/helpers/harness.js +9 -0
- package/dist/core/__tests__/integration/plugin-revalidate.test.d.ts +1 -0
- package/dist/core/__tests__/integration/plugin-revalidate.test.js +355 -0
- package/dist/core/__tests__/integration/profile-pause.test.d.ts +1 -0
- package/dist/core/__tests__/integration/profile-pause.test.js +115 -0
- package/dist/core/__tests__/node-outcome.test.d.ts +1 -0
- package/dist/core/__tests__/node-outcome.test.js +184 -0
- package/dist/core/__tests__/revive-parked-fresh.test.js +13 -20
- package/dist/core/__tests__/seam/broker-provider-retry.test.js +6 -2
- package/dist/core/__tests__/seam/dormancy-release.test.js +3 -3
- package/dist/core/__tests__/seam/node-outcome-delivery.test.d.ts +1 -0
- package/dist/core/__tests__/seam/node-outcome-delivery.test.js +153 -0
- package/dist/core/bash-jobs.d.ts +6 -0
- package/dist/core/bash-jobs.js +22 -0
- package/dist/core/canvas/canvas.d.ts +41 -3
- package/dist/core/canvas/canvas.js +153 -39
- package/dist/core/canvas/crons.js +2 -1
- package/dist/core/canvas/index.d.ts +1 -0
- package/dist/core/canvas/index.js +1 -0
- package/dist/core/canvas/migrations.js +47 -0
- package/dist/core/canvas/node-outcome-deliveries.d.ts +80 -0
- package/dist/core/canvas/node-outcome-deliveries.js +163 -0
- package/dist/core/canvas/pid.d.ts +7 -0
- package/dist/core/canvas/pid.js +17 -0
- package/dist/core/canvas/render-source.js +2 -2
- package/dist/core/canvas/types.d.ts +42 -1
- package/dist/core/canvas/types.js +2 -1
- package/dist/core/command-manifests/registry.d.ts +3 -1
- package/dist/core/command-manifests/schema.d.ts +2 -75
- package/dist/core/command-manifests/schema.js +5 -0
- package/dist/core/command-plugins/compose.js +1 -1
- package/dist/core/command-plugins/revalidate.d.ts +18 -0
- package/dist/core/command-plugins/revalidate.js +152 -0
- package/dist/core/command-plugins/transport/http-fetch.d.ts +29 -4
- package/dist/core/command-plugins/transport/http-fetch.js +18 -8
- package/dist/core/command-plugins/transport/http-invoke.js +7 -0
- package/dist/core/command.d.ts +8 -0
- package/dist/core/command.js +19 -3
- package/dist/core/config.d.ts +4 -1
- package/dist/core/config.js +25 -1
- package/dist/core/exclusive-lock.d.ts +12 -0
- package/dist/core/exclusive-lock.js +29 -0
- package/dist/core/feed/feed.d.ts +8 -0
- package/dist/core/feed/feed.js +25 -4
- package/dist/core/feed/inbox.d.ts +5 -0
- package/dist/core/feed/inbox.js +13 -0
- package/dist/core/installed-plugins.js +31 -0
- package/dist/core/keybindings/catalog.d.ts +1 -1
- package/dist/core/keybindings/catalog.js +3 -0
- package/dist/core/manifest-recovery.d.ts +15 -0
- package/dist/core/manifest-recovery.js +72 -0
- package/dist/core/manifest-stale.d.ts +18 -0
- package/dist/core/manifest-stale.js +39 -0
- package/dist/core/plugin-swap-lock.d.ts +9 -0
- package/dist/core/plugin-swap-lock.js +31 -0
- package/dist/core/profiles/manifest.d.ts +3 -1
- package/dist/core/profiles/manifest.js +6 -1
- package/dist/core/runtime/broker/daemon-ops.d.ts +3 -1
- package/dist/core/runtime/broker/daemon-ops.js +6 -0
- package/dist/core/runtime/broker/engine-drive.d.ts +2 -3
- package/dist/core/runtime/broker/engine-drive.js +74 -27
- package/dist/core/runtime/broker/frame-dispatch.js +1 -0
- package/dist/core/runtime/broker/read-ops.js +14 -0
- package/dist/core/runtime/broker/rebind.js +0 -7
- package/dist/core/runtime/broker-protocol.d.ts +1 -6
- package/dist/core/runtime/fleet.d.ts +7 -0
- package/dist/core/runtime/headless-pi.d.ts +13 -0
- package/dist/core/runtime/headless-pi.js +10 -2
- package/dist/core/runtime/lifecycle.d.ts +6 -1
- package/dist/core/runtime/lifecycle.js +50 -1
- package/dist/core/runtime/nodes.d.ts +5 -1
- package/dist/core/runtime/nodes.js +2 -1
- package/dist/core/runtime/outcome-document.d.ts +70 -0
- package/dist/core/runtime/outcome-document.js +97 -0
- package/dist/core/runtime/revive.js +16 -21
- package/dist/core/runtime/spawn.d.ts +5 -1
- package/dist/core/runtime/spawn.js +18 -7
- package/dist/core/runtime/stamp/channel.d.ts +3 -5
- package/dist/core/runtime/stamp/channel.js +3 -10
- package/dist/core/runtime/stop-guard.js +2 -2
- package/dist/core/runtime/structured-output.d.ts +24 -3
- package/dist/core/runtime/structured-output.js +42 -3
- package/dist/core/runtime/tmux-bindings.js +4 -0
- package/dist/core/runtime/warm-pool.d.ts +5 -2
- package/dist/core/runtime/warm-pool.js +5 -5
- package/dist/core/subscription-state.js +7 -5
- package/dist/core/substrate/render.js +10 -10
- package/dist/core/user-settings.d.ts +18 -1
- package/dist/core/user-settings.js +37 -0
- package/dist/daemon/__tests__/node-deadline-lane.test.d.ts +1 -0
- package/dist/daemon/__tests__/node-deadline-lane.test.js +76 -0
- package/dist/daemon/__tests__/node-outcome-birth-invariants.test.d.ts +1 -0
- package/dist/daemon/__tests__/node-outcome-birth-invariants.test.js +122 -0
- package/dist/daemon/__tests__/node-outcome-delivery.test.d.ts +1 -0
- package/dist/daemon/__tests__/node-outcome-delivery.test.js +86 -0
- package/dist/daemon/api/__tests__/broker-settle-park.test.js +86 -18
- package/dist/daemon/api/__tests__/profile-launch-gates.test.js +10 -1
- package/dist/daemon/api/__tests__/seam/node-outcome-api.test.d.ts +1 -0
- package/dist/daemon/api/__tests__/seam/node-outcome-api.test.js +159 -0
- package/dist/daemon/api/__tests__/submit-result.test.d.ts +1 -0
- package/dist/daemon/api/__tests__/submit-result.test.js +214 -0
- package/dist/daemon/api/handlers/broker-ops.js +53 -24
- package/dist/daemon/api/handlers/canvas.js +2 -2
- package/dist/daemon/api/handlers/messages.js +12 -2
- package/dist/daemon/api/handlers/node-outcomes.d.ts +2 -0
- package/dist/daemon/api/handlers/node-outcomes.js +136 -0
- package/dist/daemon/api/handlers/nodes.js +67 -1
- package/dist/daemon/api/handlers/profiles.js +53 -1
- package/dist/daemon/api/handlers/reports.js +197 -33
- package/dist/daemon/api/map.d.ts +7 -2
- package/dist/daemon/api/map.js +78 -4
- package/dist/daemon/api/server.js +2 -0
- package/dist/daemon/crtrd.js +12 -0
- package/dist/daemon/fleet.d.ts +4 -8
- package/dist/daemon/fleet.js +48 -9
- package/dist/daemon/messaging/node-message.js +2 -1
- package/dist/daemon/node-outcome/deliver-outcome.d.ts +14 -0
- package/dist/daemon/node-outcome/deliver-outcome.js +155 -0
- package/dist/daemon/park-activity.d.ts +9 -0
- package/dist/daemon/park-activity.js +30 -0
- package/dist/daemon/park-pending.d.ts +10 -5
- package/dist/daemon/park-pending.js +18 -8
- package/dist/daemon/reconcilers/bash-deadline.js +6 -0
- package/dist/daemon/reconcilers/broker-supervision.d.ts +3 -7
- package/dist/daemon/reconcilers/broker-supervision.js +29 -33
- package/dist/daemon/reconcilers/node-deadline.d.ts +5 -0
- package/dist/daemon/reconcilers/node-deadline.js +82 -0
- package/dist/daemon/reconcilers/node-lifecycle/freeze-lane.js +9 -1
- package/dist/daemon/reconcilers/node-lifecycle/tick.js +12 -3
- package/dist/daemon/reconcilers/node-outcome-delivery-lane.d.ts +10 -0
- package/dist/daemon/reconcilers/node-outcome-delivery-lane.js +41 -0
- package/dist/pi-extensions/__tests__/canvas-structured-output.test.js +66 -66
- package/dist/pi-extensions/canvas-structured-output.d.ts +9 -5
- package/dist/pi-extensions/canvas-structured-output.js +23 -297
- package/dist/shared/env.d.ts +3 -0
- package/dist/shared/env.js +6 -0
- package/dist/shared/generated-context.d.ts +3 -9
- package/dist/shared/generated-context.js +8 -16
- package/dist/types.d.ts +19 -0
- package/dist/types.js +1 -0
- package/package.json +9 -8
- package/runtime.lock.json +227 -383
|
@@ -93,16 +93,21 @@ function upsertRow(meta) {
|
|
|
93
93
|
pi_session_file=excluded.pi_session_file`)
|
|
94
94
|
.run(meta.node_id, meta.name, meta.description ?? null, meta.cycles ?? null, meta.kind, meta.mode, meta.lifecycle, meta.cwd, meta.host_kind ?? null, meta.profile_id ?? null, meta.parent ?? null, meta.created, typeof meta.pi_session_file === 'string' ? canonicalSessionFile(meta.pi_session_file) : null);
|
|
95
95
|
}
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
96
|
+
function insertArmedOutcomeDelivery(db, nodeId, delivery) {
|
|
97
|
+
const payloadJson = delivery.payload === undefined ? null : JSON.stringify(delivery.payload);
|
|
98
|
+
if (payloadJson === undefined)
|
|
99
|
+
throw new TypeError('outcome delivery payload must be JSON-serializable');
|
|
100
|
+
const now = Date.now();
|
|
101
|
+
db.prepare(`INSERT INTO node_outcome_deliveries
|
|
102
|
+
(node_id, state, action_name, argv_json, cwd, config_origin, payload_json, next_attempt_at, created_at, updated_at)
|
|
103
|
+
VALUES (?, 'armed', ?, ?, ?, ?, ?, 0, ?, ?)`).run(nodeId, delivery.actionName, JSON.stringify(delivery.argv), delivery.cwd, delivery.configOrigin, payloadJson, now, now);
|
|
104
|
+
}
|
|
105
|
+
function seedRow(meta, db = openDb()) {
|
|
106
|
+
db
|
|
102
107
|
.prepare(`INSERT INTO nodes
|
|
103
108
|
(node_id, name, description, cycles, kind, mode, lifecycle, cwd, host_kind, profile_id, parent, created, pi_session_file,
|
|
104
|
-
status, intent, waiting_for, final_report, finalized_at, pi_pid, "window", tmux_session, pane)
|
|
105
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
109
|
+
status, intent, waiting_for, final_report, finalized_at, deadline_at, pi_pid, "window", tmux_session, pane)
|
|
110
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
106
111
|
ON CONFLICT(node_id) DO UPDATE SET
|
|
107
112
|
name=excluded.name, description=excluded.description, cycles=excluded.cycles,
|
|
108
113
|
kind=excluded.kind, mode=excluded.mode,
|
|
@@ -113,7 +118,7 @@ function seedRow(meta) {
|
|
|
113
118
|
final_report=excluded.final_report, finalized_at=excluded.finalized_at,
|
|
114
119
|
pi_pid=excluded.pi_pid, "window"=excluded."window", tmux_session=excluded.tmux_session,
|
|
115
120
|
pane=excluded.pane`)
|
|
116
|
-
.run(meta.node_id, meta.name, meta.description ?? null, meta.cycles ?? null, meta.kind, meta.mode, meta.lifecycle, meta.cwd, meta.host_kind ?? null, meta.profile_id ?? null, meta.parent ?? null, meta.created, typeof meta.pi_session_file === 'string' ? canonicalSessionFile(meta.pi_session_file) : null, meta.status ?? 'active', meta.intent ?? null, meta.waiting_for ?? null, meta.final_report ?? null, meta.finalized_at ?? null, meta.pi_pid ?? null, meta.window ?? null, meta.tmux_session ?? null, meta.pane ?? null);
|
|
121
|
+
.run(meta.node_id, meta.name, meta.description ?? null, meta.cycles ?? null, meta.kind, meta.mode, meta.lifecycle, meta.cwd, meta.host_kind ?? null, meta.profile_id ?? null, meta.parent ?? null, meta.created, typeof meta.pi_session_file === 'string' ? canonicalSessionFile(meta.pi_session_file) : null, meta.status ?? 'active', meta.intent ?? null, meta.waiting_for ?? null, meta.final_report ?? null, meta.finalized_at ?? null, meta.deadline_at ?? null, meta.pi_pid ?? null, meta.window ?? null, meta.tmux_session ?? null, meta.pane ?? null);
|
|
117
122
|
}
|
|
118
123
|
function rowFrom(r) {
|
|
119
124
|
return {
|
|
@@ -140,6 +145,12 @@ function rowFrom(r) {
|
|
|
140
145
|
terminal_reason: r['terminal_reason'] ?? null,
|
|
141
146
|
final_report: r['final_report'] ?? null,
|
|
142
147
|
finalized_at: r['finalized_at'] ?? null,
|
|
148
|
+
outcome_kind: r['outcome_kind'] ?? null,
|
|
149
|
+
outcome_reason: r['outcome_reason'] ?? null,
|
|
150
|
+
outcome_at: r['outcome_at'] ?? null,
|
|
151
|
+
outcome_revision: r['outcome_revision'] ?? 0,
|
|
152
|
+
outcome_detail_json: r['outcome_detail_json'] ?? null,
|
|
153
|
+
deadline_at: r['deadline_at'] ?? null,
|
|
143
154
|
pi_pid: r['pi_pid'] ?? null,
|
|
144
155
|
pi_pid_identity: r['pi_pid_identity'] ?? null,
|
|
145
156
|
window: r['window'] ?? null,
|
|
@@ -151,7 +162,7 @@ function rowFrom(r) {
|
|
|
151
162
|
* no row exists yet (the hydration then falls back to whatever the meta held). */
|
|
152
163
|
function runtimeFromRow(nodeId) {
|
|
153
164
|
const r = openDb()
|
|
154
|
-
.prepare('SELECT status, intent, waiting_for, frozen_at, respawn_failures, respawn_not_before, launched_at, last_settled_at, terminal_reason, final_report, finalized_at, pi_pid, pi_pid_identity, "window", tmux_session, pane FROM nodes WHERE node_id = ?')
|
|
165
|
+
.prepare('SELECT status, intent, waiting_for, frozen_at, respawn_failures, respawn_not_before, launched_at, last_settled_at, terminal_reason, final_report, finalized_at, outcome_kind, outcome_reason, outcome_at, outcome_revision, outcome_detail_json, deadline_at, pi_pid, pi_pid_identity, "window", tmux_session, pane FROM nodes WHERE node_id = ?')
|
|
155
166
|
.get(nodeId);
|
|
156
167
|
if (r === undefined)
|
|
157
168
|
return null;
|
|
@@ -167,6 +178,12 @@ function runtimeFromRow(nodeId) {
|
|
|
167
178
|
terminal_reason: r['terminal_reason'] ?? null,
|
|
168
179
|
final_report: r['final_report'] ?? null,
|
|
169
180
|
finalized_at: r['finalized_at'] ?? null,
|
|
181
|
+
outcome_kind: r['outcome_kind'] ?? null,
|
|
182
|
+
outcome_reason: r['outcome_reason'] ?? null,
|
|
183
|
+
outcome_at: r['outcome_at'] ?? null,
|
|
184
|
+
outcome_revision: r['outcome_revision'] ?? 0,
|
|
185
|
+
outcome_detail_json: r['outcome_detail_json'] ?? null,
|
|
186
|
+
deadline_at: r['deadline_at'] ?? null,
|
|
170
187
|
pi_pid: r['pi_pid'] ?? null,
|
|
171
188
|
pi_pid_identity: r['pi_pid_identity'] ?? null,
|
|
172
189
|
window: r['window'] ?? null,
|
|
@@ -177,7 +194,7 @@ function runtimeFromRow(nodeId) {
|
|
|
177
194
|
// Nodes
|
|
178
195
|
/** Create a node: scaffold its dirs, persist identity to meta.json, and seed the
|
|
179
196
|
* row (identity + runtime from the incoming meta). Returns the hydrated view. */
|
|
180
|
-
export function createNode(meta) {
|
|
197
|
+
export function createNode(meta, opts = {}) {
|
|
181
198
|
ensureHome();
|
|
182
199
|
const scaffolded = !existsSync(nodeDir(meta.node_id));
|
|
183
200
|
ensureNodeDirs(meta.node_id);
|
|
@@ -191,7 +208,11 @@ export function createNode(meta) {
|
|
|
191
208
|
rmSync(nodeDir(meta.node_id), { recursive: true, force: true });
|
|
192
209
|
throw error;
|
|
193
210
|
}
|
|
194
|
-
|
|
211
|
+
withCanvasWrite((db) => {
|
|
212
|
+
seedRow(meta, db);
|
|
213
|
+
if (opts.outcomeDelivery !== undefined)
|
|
214
|
+
insertArmedOutcomeDelivery(db, meta.node_id, opts.outcomeDelivery);
|
|
215
|
+
});
|
|
195
216
|
return getNode(meta.node_id);
|
|
196
217
|
}
|
|
197
218
|
/** The canonical node record: durable identity (meta.json) ∪ authoritative
|
|
@@ -237,6 +258,35 @@ export function getRowByPane(pane) {
|
|
|
237
258
|
.get(pane);
|
|
238
259
|
return r ? rowFrom(r) : null;
|
|
239
260
|
}
|
|
261
|
+
/** Nodes whose persisted spawn deadline has elapsed and that remain live work. */
|
|
262
|
+
export function dueNodeDeadlines(nowIso) {
|
|
263
|
+
return openDb().prepare(`SELECT * FROM nodes
|
|
264
|
+
WHERE deadline_at IS NOT NULL AND outcome_at IS NULL
|
|
265
|
+
AND deadline_at <= ? AND status IN ('active','idle') AND kind != 'human'
|
|
266
|
+
ORDER BY deadline_at, created, node_id`).all(nowIso).map(rowFrom);
|
|
267
|
+
}
|
|
268
|
+
/** Write a claimed warm spare's deadline before it is handed to its caller. */
|
|
269
|
+
export function setNodeDeadline(nodeId, deadlineAt) {
|
|
270
|
+
return Number(openDb().prepare('UPDATE nodes SET deadline_at = ? WHERE node_id = ?').run(deadlineAt, nodeId).changes) === 1;
|
|
271
|
+
}
|
|
272
|
+
/** Shift active/idle profile deadlines by a completed profile pause duration. */
|
|
273
|
+
export function shiftProfileNodeDeadlines(profileId, pauseDurationMs) {
|
|
274
|
+
if (!Number.isFinite(pauseDurationMs) || pauseDurationMs <= 0)
|
|
275
|
+
return 0;
|
|
276
|
+
return withCanvasWrite((db) => {
|
|
277
|
+
const rows = db.prepare(`SELECT node_id, deadline_at FROM nodes
|
|
278
|
+
WHERE profile_id = ? AND deadline_at IS NOT NULL AND status IN ('active', 'idle')`).all(profileId);
|
|
279
|
+
const update = db.prepare('UPDATE nodes SET deadline_at = ? WHERE node_id = ? AND deadline_at = ?');
|
|
280
|
+
let shifted = 0;
|
|
281
|
+
for (const row of rows) {
|
|
282
|
+
const at = Date.parse(row.deadline_at);
|
|
283
|
+
if (Number.isNaN(at))
|
|
284
|
+
continue;
|
|
285
|
+
shifted += Number(update.run(new Date(at + pauseDurationMs).toISOString(), row.node_id, row.deadline_at).changes);
|
|
286
|
+
}
|
|
287
|
+
return shifted;
|
|
288
|
+
});
|
|
289
|
+
}
|
|
240
290
|
/** Merge an IDENTITY patch into a node's meta.json and re-index its identity
|
|
241
291
|
* columns. Identity has a single writer per node, so this read-modify-write is
|
|
242
292
|
* safe (the contended runtime fields were moved out — see the atomic setters
|
|
@@ -279,32 +329,84 @@ export function setIntent(nodeId, intent) {
|
|
|
279
329
|
* unchanged. Returns the row-changed count so the caller can distinguish "no
|
|
280
330
|
* such node", "gate failed", and "already applied" without a second query. */
|
|
281
331
|
export function transitionStatusIntent(nodeId, fields, opts = {}) {
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
if (
|
|
285
|
-
assignments
|
|
286
|
-
values
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
predicates
|
|
300
|
-
values.push(
|
|
332
|
+
// Keep the ordinary lifecycle write exactly as its pre-outcome form: one
|
|
333
|
+
// conditional UPDATE and no transaction boundary.
|
|
334
|
+
if (opts.outcome === undefined) {
|
|
335
|
+
const assignments = [];
|
|
336
|
+
const values = [];
|
|
337
|
+
if (Object.prototype.hasOwnProperty.call(fields, 'status')) {
|
|
338
|
+
assignments.push('status = ?');
|
|
339
|
+
values.push(fields.status ?? null);
|
|
340
|
+
}
|
|
341
|
+
if (Object.prototype.hasOwnProperty.call(fields, 'intent')) {
|
|
342
|
+
assignments.push('intent = ?');
|
|
343
|
+
values.push(fields.intent ?? null);
|
|
344
|
+
}
|
|
345
|
+
if (Object.prototype.hasOwnProperty.call(fields, 'terminal_reason')) {
|
|
346
|
+
assignments.push('terminal_reason = ?');
|
|
347
|
+
values.push(fields.terminal_reason ?? null);
|
|
348
|
+
}
|
|
349
|
+
const predicates = ['node_id = ?'];
|
|
350
|
+
values.push(nodeId);
|
|
351
|
+
if (opts.fromStatuses !== undefined) {
|
|
352
|
+
predicates.push(`status IN (${opts.fromStatuses.map(() => '?').join(', ')})`);
|
|
353
|
+
values.push(...opts.fromStatuses);
|
|
354
|
+
}
|
|
355
|
+
if (opts.requireNoFinalReport === true)
|
|
356
|
+
predicates.push('final_report IS NULL');
|
|
357
|
+
const result = openDb()
|
|
358
|
+
.prepare(`UPDATE nodes SET ${assignments.join(', ')} WHERE ${predicates.join(' AND ')}`)
|
|
359
|
+
.run(...values);
|
|
360
|
+
return Number(result.changes);
|
|
301
361
|
}
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
.
|
|
307
|
-
|
|
362
|
+
const outcome = opts.outcome;
|
|
363
|
+
return withCanvasWrite((db) => {
|
|
364
|
+
const assignments = [];
|
|
365
|
+
const values = [];
|
|
366
|
+
if (Object.prototype.hasOwnProperty.call(fields, 'status')) {
|
|
367
|
+
assignments.push('status = ?');
|
|
368
|
+
values.push(fields.status ?? null);
|
|
369
|
+
}
|
|
370
|
+
if (Object.prototype.hasOwnProperty.call(fields, 'intent')) {
|
|
371
|
+
assignments.push('intent = ?');
|
|
372
|
+
values.push(fields.intent ?? null);
|
|
373
|
+
}
|
|
374
|
+
if (Object.prototype.hasOwnProperty.call(fields, 'terminal_reason')) {
|
|
375
|
+
assignments.push('terminal_reason = ?');
|
|
376
|
+
values.push(fields.terminal_reason ?? null);
|
|
377
|
+
}
|
|
378
|
+
const outcomeValue = (value, column) => outcome.supersede === true ? '?' : `CASE WHEN outcome_at IS NULL THEN ? ELSE ${column} END`;
|
|
379
|
+
assignments.push(`outcome_kind = ${outcomeValue(outcome.kind, 'outcome_kind')}`, `outcome_reason = ${outcomeValue(outcome.reason, 'outcome_reason')}`, `outcome_detail_json = ${outcomeValue(outcome.detailJson, 'outcome_detail_json')}`, `outcome_revision = ${outcome.supersede === true ? 'outcome_revision + 1' : 'CASE WHEN outcome_at IS NULL THEN outcome_revision + 1 ELSE outcome_revision END'}`, `outcome_at = ${outcomeValue(outcome.at, 'outcome_at')}`);
|
|
380
|
+
values.push(outcome.kind, outcome.reason, outcome.detailJson, outcome.at);
|
|
381
|
+
const predicates = ['node_id = ?'];
|
|
382
|
+
values.push(nodeId);
|
|
383
|
+
if (opts.fromStatuses !== undefined) {
|
|
384
|
+
predicates.push(`status IN (${opts.fromStatuses.map(() => '?').join(', ')})`);
|
|
385
|
+
values.push(...opts.fromStatuses);
|
|
386
|
+
}
|
|
387
|
+
if (opts.requireNoFinalReport === true)
|
|
388
|
+
predicates.push('final_report IS NULL');
|
|
389
|
+
const result = db.prepare(`UPDATE nodes SET ${assignments.join(', ')} WHERE ${predicates.join(' AND ')}`).run(...values);
|
|
390
|
+
if (Number(result.changes) === 0)
|
|
391
|
+
return 0;
|
|
392
|
+
const latched = db.prepare('SELECT outcome_at FROM nodes WHERE node_id = ?').get(nodeId);
|
|
393
|
+
if (outcome.supersede === true || latched.outcome_at === outcome.at) {
|
|
394
|
+
const now = Date.now();
|
|
395
|
+
if (outcome.supersede === true) {
|
|
396
|
+
db.prepare(`UPDATE node_outcome_deliveries
|
|
397
|
+
SET state = 'pending', attempt = 0, document_json = ?, next_attempt_at = ?,
|
|
398
|
+
claimed_at = NULL, claim_owner = NULL, accepted_at = NULL, permanent_failed_at = NULL,
|
|
399
|
+
last_failure_json = NULL, updated_at = ?
|
|
400
|
+
WHERE node_id = ?`).run(outcome.documentJson ?? null, now, now, nodeId);
|
|
401
|
+
}
|
|
402
|
+
else {
|
|
403
|
+
db.prepare(`UPDATE node_outcome_deliveries
|
|
404
|
+
SET state = 'pending', next_attempt_at = ?, document_json = ?, updated_at = ?
|
|
405
|
+
WHERE node_id = ? AND state = 'armed' AND ? IS NOT NULL`).run(now, outcome.documentJson ?? null, now, nodeId, outcome.documentJson ?? null);
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
return Number(result.changes);
|
|
409
|
+
});
|
|
308
410
|
}
|
|
309
411
|
/** Read a node's committed-final latch pointer. Pure read, no side effect. */
|
|
310
412
|
export function getFinalReport(nodeId) {
|
|
@@ -479,6 +581,15 @@ export function listNodes(filter) {
|
|
|
479
581
|
.all(...params);
|
|
480
582
|
return rows.map(rowFrom);
|
|
481
583
|
}
|
|
584
|
+
/** Identity-only projection of every user-visible row: the broker's report
|
|
585
|
+
* resolver needs only "does this sender exist, and what is it called", and a
|
|
586
|
+
* three-column read stays cheap against a canvas of thousands of rows where a
|
|
587
|
+
* full `listNodes()` hydration does not. */
|
|
588
|
+
export function listNodeIdentities() {
|
|
589
|
+
return openDb()
|
|
590
|
+
.prepare(`SELECT node_id, name, created FROM nodes WHERE ${NOT_A_SPARE} ORDER BY created`)
|
|
591
|
+
.all();
|
|
592
|
+
}
|
|
482
593
|
// Warm pool — spare booted roots awaiting a `prefer_warm` claim (see
|
|
483
594
|
// `runtime/warm-pool.ts` for the recipe key and the mint/refill policy).
|
|
484
595
|
/** Mark an already-spawned node as an unclaimed spare for `recipeKey`. Hides it
|
|
@@ -553,7 +664,7 @@ function spareIsClaimable(row) {
|
|
|
553
664
|
* spare must not sort as an hour-old conversation. This is the one sanctioned
|
|
554
665
|
* write of `created` after birth (`upsertRow` deliberately never touches it);
|
|
555
666
|
* the caller mirrors the same stamp into meta.json. */
|
|
556
|
-
export function claimWarmSpare(recipeKey, created) {
|
|
667
|
+
export function claimWarmSpare(recipeKey, created, opts = {}) {
|
|
557
668
|
return withCanvasWrite((db) => {
|
|
558
669
|
const row = warmSpareRowsFor(db, recipeKey).find(spareIsClaimable);
|
|
559
670
|
if (row === undefined)
|
|
@@ -561,7 +672,10 @@ export function claimWarmSpare(recipeKey, created) {
|
|
|
561
672
|
const res = db.prepare('DELETE FROM warm_pool WHERE node_id = ?').run(row.node_id);
|
|
562
673
|
if (res.changes === 0)
|
|
563
674
|
return null;
|
|
564
|
-
db.prepare('UPDATE nodes SET created = ? WHERE node_id = ?')
|
|
675
|
+
db.prepare('UPDATE nodes SET created = ?, deadline_at = ? WHERE node_id = ?')
|
|
676
|
+
.run(created, opts.deadlineAt ?? null, row.node_id);
|
|
677
|
+
if (opts.outcomeDelivery !== undefined)
|
|
678
|
+
insertArmedOutcomeDelivery(db, row.node_id, opts.outcomeDelivery);
|
|
565
679
|
return row.node_id;
|
|
566
680
|
});
|
|
567
681
|
}
|
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
// serve the daemon-owned run executor in src/daemon/cron-run.ts.
|
|
22
22
|
import { openDb, withCanvasWrite } from './db.js';
|
|
23
23
|
import { capturePidIdentities, identitiesMatch, killProcessGroup } from './pid.js';
|
|
24
|
+
import { isProfilePaused } from '../profiles/manifest.js';
|
|
24
25
|
/** Ring bound for `cron_runs`: recordCronRun prunes each cron's history to
|
|
25
26
|
* its most recent this-many rows. */
|
|
26
27
|
export const CRON_RUN_RING = 20;
|
|
@@ -305,7 +306,7 @@ export function dueClockCrons(nowIso) {
|
|
|
305
306
|
.prepare(`SELECT * FROM crons
|
|
306
307
|
WHERE state = 'active' AND run_state = 'idle' AND fire_at <= ?
|
|
307
308
|
ORDER BY fire_at, created, cron_id`)
|
|
308
|
-
.all(nowIso).map(cronFrom);
|
|
309
|
+
.all(nowIso).map(cronFrom).filter((cron) => !isProfilePaused(cron.profile));
|
|
309
310
|
}
|
|
310
311
|
/** Atomically acquire the idle run lease: flips `run_state` to 'running' and
|
|
311
312
|
* records lease/pid metadata, but ONLY if the row is still idle (the
|
|
@@ -1360,6 +1360,52 @@ function addNodeLabelColumns(db) {
|
|
|
1360
1360
|
upd.run(description, cycles, id);
|
|
1361
1361
|
}
|
|
1362
1362
|
}
|
|
1363
|
+
/** v39 — durable terminal outcomes, spawn deadlines, and externally-owned
|
|
1364
|
+
* outcome-delivery attempts. The outcome columns live on `nodes` so the same
|
|
1365
|
+
* conditional terminal-status UPDATE can latch them atomically. */
|
|
1366
|
+
function addNodeOutcomeAndDeadline(db) {
|
|
1367
|
+
const columns = nodeColumns(db);
|
|
1368
|
+
addNodeColumnIfMissing(db, columns, 'outcome_kind', 'ALTER TABLE nodes ADD COLUMN outcome_kind TEXT;');
|
|
1369
|
+
addNodeColumnIfMissing(db, columns, 'outcome_reason', 'ALTER TABLE nodes ADD COLUMN outcome_reason TEXT;');
|
|
1370
|
+
addNodeColumnIfMissing(db, columns, 'outcome_at', 'ALTER TABLE nodes ADD COLUMN outcome_at TEXT;');
|
|
1371
|
+
addNodeColumnIfMissing(db, columns, 'outcome_revision', 'ALTER TABLE nodes ADD COLUMN outcome_revision INTEGER NOT NULL DEFAULT 0;');
|
|
1372
|
+
addNodeColumnIfMissing(db, columns, 'outcome_detail_json', 'ALTER TABLE nodes ADD COLUMN outcome_detail_json TEXT;');
|
|
1373
|
+
addNodeColumnIfMissing(db, columns, 'deadline_at', 'ALTER TABLE nodes ADD COLUMN deadline_at TEXT;');
|
|
1374
|
+
db.exec(`
|
|
1375
|
+
CREATE TABLE IF NOT EXISTS node_outcome_deliveries (
|
|
1376
|
+
node_id TEXT PRIMARY KEY,
|
|
1377
|
+
state TEXT NOT NULL,
|
|
1378
|
+
attempt INTEGER NOT NULL DEFAULT 0,
|
|
1379
|
+
action_name TEXT NOT NULL,
|
|
1380
|
+
argv_json TEXT NOT NULL,
|
|
1381
|
+
cwd TEXT NOT NULL,
|
|
1382
|
+
config_origin TEXT NOT NULL,
|
|
1383
|
+
payload_json TEXT,
|
|
1384
|
+
document_json TEXT,
|
|
1385
|
+
next_attempt_at INTEGER NOT NULL DEFAULT 0,
|
|
1386
|
+
claimed_at INTEGER,
|
|
1387
|
+
claim_owner TEXT,
|
|
1388
|
+
accepted_at INTEGER,
|
|
1389
|
+
permanent_failed_at INTEGER,
|
|
1390
|
+
last_failure_json TEXT,
|
|
1391
|
+
created_at INTEGER NOT NULL,
|
|
1392
|
+
updated_at INTEGER NOT NULL,
|
|
1393
|
+
FOREIGN KEY(node_id) REFERENCES nodes(node_id) ON DELETE CASCADE
|
|
1394
|
+
);
|
|
1395
|
+
CREATE INDEX IF NOT EXISTS idx_node_outcome_deliveries_due
|
|
1396
|
+
ON node_outcome_deliveries(state, next_attempt_at);
|
|
1397
|
+
CREATE INDEX IF NOT EXISTS idx_nodes_deadline ON nodes(deadline_at) WHERE deadline_at IS NOT NULL;
|
|
1398
|
+
|
|
1399
|
+
UPDATE nodes SET outcome_kind='result', outcome_reason='finalized',
|
|
1400
|
+
outcome_at=finalized_at, outcome_revision=1
|
|
1401
|
+
WHERE final_report IS NOT NULL AND outcome_at IS NULL;
|
|
1402
|
+
|
|
1403
|
+
UPDATE nodes SET outcome_kind='failure',
|
|
1404
|
+
outcome_reason=COALESCE(terminal_reason,'crashed'),
|
|
1405
|
+
outcome_at=COALESCE(finalized_at, created), outcome_revision=1
|
|
1406
|
+
WHERE final_report IS NULL AND status IN ('done','dead','canceled') AND outcome_at IS NULL;
|
|
1407
|
+
`);
|
|
1408
|
+
}
|
|
1363
1409
|
/** The journaled lane of the state-migration registry, adapted to this chain's
|
|
1364
1410
|
* step shape. Registry order is preserved, so these versions stay stable as
|
|
1365
1411
|
* long as the registry is append-only (same contract as the local list). */
|
|
@@ -1411,6 +1457,7 @@ export const MIGRATIONS = [
|
|
|
1411
1457
|
/* v36 */ makeNodeSessionFileUnique,
|
|
1412
1458
|
/* v37 */ repairNodeSessionFileUniqueIndex,
|
|
1413
1459
|
/* v38 */ addNodeLabelColumns,
|
|
1460
|
+
/* v39 */ addNodeOutcomeAndDeadline,
|
|
1414
1461
|
...JOURNALED_STATE_MIGRATIONS,
|
|
1415
1462
|
];
|
|
1416
1463
|
/** Migration indexes that manage their OWN transaction and therefore must not
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { type NodeOutcomeKind, type TerminalReason } from './types.js';
|
|
2
|
+
export type OutcomeDeliveryState = 'armed' | 'pending' | 'running' | 'accepted' | 'permanent_failed';
|
|
3
|
+
export interface OutcomeDeliveryFailure {
|
|
4
|
+
kind: 'exit' | 'signal' | 'timeout' | 'spawn_error';
|
|
5
|
+
exitCode?: number;
|
|
6
|
+
signal?: string;
|
|
7
|
+
message?: string;
|
|
8
|
+
stderr?: string;
|
|
9
|
+
}
|
|
10
|
+
/** Durable exponential retry delay for the attempt that just failed. */
|
|
11
|
+
export declare function outcomeDeliveryBackoffMs(attempt: number): number;
|
|
12
|
+
export interface OutcomeDeliveryRecord {
|
|
13
|
+
nodeId: string;
|
|
14
|
+
state: OutcomeDeliveryState;
|
|
15
|
+
attempt: number;
|
|
16
|
+
actionName: string;
|
|
17
|
+
argv: string[];
|
|
18
|
+
cwd: string;
|
|
19
|
+
configOrigin: string;
|
|
20
|
+
payload: unknown | null;
|
|
21
|
+
document: unknown | null;
|
|
22
|
+
nextAttemptAt: number;
|
|
23
|
+
claimedAt: number | null;
|
|
24
|
+
claimOwner: string | null;
|
|
25
|
+
acceptedAt: number | null;
|
|
26
|
+
permanentFailedAt: number | null;
|
|
27
|
+
lastFailure: OutcomeDeliveryFailure | null;
|
|
28
|
+
createdAt: number;
|
|
29
|
+
updatedAt: number;
|
|
30
|
+
}
|
|
31
|
+
export interface OutcomeDocumentSnapshot {
|
|
32
|
+
kind: NodeOutcomeKind;
|
|
33
|
+
reason: TerminalReason;
|
|
34
|
+
at: string;
|
|
35
|
+
revision: number;
|
|
36
|
+
detailJson: string | null;
|
|
37
|
+
}
|
|
38
|
+
/** Compose every delivery document from the hydrated identity and the immutable outcome latch. */
|
|
39
|
+
export declare function composeOutcomeDeliveryDocument(args: {
|
|
40
|
+
nodeId: string;
|
|
41
|
+
outcome: OutcomeDocumentSnapshot;
|
|
42
|
+
action: {
|
|
43
|
+
name: string;
|
|
44
|
+
payload: unknown;
|
|
45
|
+
};
|
|
46
|
+
finalReport?: string | null;
|
|
47
|
+
structuredResult?: unknown;
|
|
48
|
+
}): string;
|
|
49
|
+
/** Register a target atomically against the outcome latch. A still-armed target
|
|
50
|
+
* is replaceable; a settled registration is not. When the outcome already
|
|
51
|
+
* exists, the row is born pending with its complete immutable document. */
|
|
52
|
+
export declare function armOutcomeDelivery(row: {
|
|
53
|
+
nodeId: string;
|
|
54
|
+
actionName: string;
|
|
55
|
+
argv: string[];
|
|
56
|
+
cwd: string;
|
|
57
|
+
configOrigin: string;
|
|
58
|
+
payload?: unknown;
|
|
59
|
+
now?: number;
|
|
60
|
+
}): OutcomeDeliveryRecord | null;
|
|
61
|
+
export declare function readOutcomeDelivery(nodeId: string): OutcomeDeliveryRecord | null;
|
|
62
|
+
export declare function dueOutcomeDeliveries(now: number): OutcomeDeliveryRecord[];
|
|
63
|
+
export declare function claimOutcomeDelivery(nodeId: string, now: number, claimOwner: string): OutcomeDeliveryRecord | null;
|
|
64
|
+
export type OutcomeDeliverySettlement = {
|
|
65
|
+
state: 'accepted';
|
|
66
|
+
} | {
|
|
67
|
+
state: 'permanent_failed';
|
|
68
|
+
failure: OutcomeDeliveryFailure;
|
|
69
|
+
} | {
|
|
70
|
+
state: 'pending';
|
|
71
|
+
nextAttemptAt: number;
|
|
72
|
+
failure: OutcomeDeliveryFailure;
|
|
73
|
+
};
|
|
74
|
+
/** Settle only the still-owned running attempt, fencing stale delivery processes. */
|
|
75
|
+
export declare function settleOutcomeDelivery(nodeId: string, attempt: number, claimOwner: string, settlement: OutcomeDeliverySettlement, now: number): boolean;
|
|
76
|
+
export declare function recoverRunningOutcomeDeliveries(now: number): number;
|
|
77
|
+
/** Disarming is deliberately limited to registrations that have not started delivery. */
|
|
78
|
+
export declare function disarmOutcomeDelivery(nodeId: string): boolean;
|
|
79
|
+
/** A canonical final is the one allowed outcome supersede: discard any old claim and retry state. */
|
|
80
|
+
export declare function resetOutcomeDeliveryForSupersede(nodeId: string, documentJson: string | null, now: number): boolean;
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
// Durable delivery registration and attempt state for a node's terminal outcome.
|
|
2
|
+
import { openDb, withCanvasWrite } from './db.js';
|
|
3
|
+
import { getNode } from './canvas.js';
|
|
4
|
+
import { OUTCOME_PAYLOAD_MAX_BYTES } from './types.js';
|
|
5
|
+
import { composeNodeOutcomeDocument } from '../runtime/outcome-document.js';
|
|
6
|
+
/** Durable exponential retry delay for the attempt that just failed. */
|
|
7
|
+
export function outcomeDeliveryBackoffMs(attempt) {
|
|
8
|
+
return Math.min(5_000 * 2 ** (attempt - 1), 3_600_000);
|
|
9
|
+
}
|
|
10
|
+
function serializedPayload(payload) {
|
|
11
|
+
if (payload === undefined)
|
|
12
|
+
return null;
|
|
13
|
+
const json = JSON.stringify(payload);
|
|
14
|
+
if (json === undefined)
|
|
15
|
+
throw new TypeError('outcome delivery payload must be JSON-serializable');
|
|
16
|
+
if (Buffer.byteLength(json, 'utf8') > OUTCOME_PAYLOAD_MAX_BYTES) {
|
|
17
|
+
throw new RangeError(`outcome delivery payload exceeds ${OUTCOME_PAYLOAD_MAX_BYTES} bytes`);
|
|
18
|
+
}
|
|
19
|
+
return json;
|
|
20
|
+
}
|
|
21
|
+
function outcomeDeliveryFrom(row) {
|
|
22
|
+
return {
|
|
23
|
+
nodeId: row.node_id,
|
|
24
|
+
state: row.state,
|
|
25
|
+
attempt: row.attempt,
|
|
26
|
+
actionName: row.action_name,
|
|
27
|
+
argv: JSON.parse(row.argv_json),
|
|
28
|
+
cwd: row.cwd,
|
|
29
|
+
configOrigin: row.config_origin,
|
|
30
|
+
payload: row.payload_json === null ? null : JSON.parse(row.payload_json),
|
|
31
|
+
document: row.document_json === null ? null : JSON.parse(row.document_json),
|
|
32
|
+
nextAttemptAt: row.next_attempt_at,
|
|
33
|
+
claimedAt: row.claimed_at,
|
|
34
|
+
claimOwner: row.claim_owner,
|
|
35
|
+
acceptedAt: row.accepted_at,
|
|
36
|
+
permanentFailedAt: row.permanent_failed_at,
|
|
37
|
+
lastFailure: row.last_failure_json === null ? null : JSON.parse(row.last_failure_json),
|
|
38
|
+
createdAt: row.created_at,
|
|
39
|
+
updatedAt: row.updated_at,
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
/** The latch, rather than a later mutable row, defines the terminal view in a delivery document. */
|
|
43
|
+
function nodeAtOutcome(node, outcome) {
|
|
44
|
+
let status;
|
|
45
|
+
let intent;
|
|
46
|
+
if (outcome.kind === 'result' || outcome.reason === 'finished' || outcome.reason === 'declined') {
|
|
47
|
+
status = 'done';
|
|
48
|
+
intent = 'done';
|
|
49
|
+
}
|
|
50
|
+
else if (outcome.reason === 'parked') {
|
|
51
|
+
status = 'done';
|
|
52
|
+
intent = 'parked';
|
|
53
|
+
}
|
|
54
|
+
else if (outcome.reason === 'closed' || outcome.reason === 'retired' || outcome.reason === 'stranded' || outcome.reason === 'deadline_exceeded') {
|
|
55
|
+
status = 'canceled';
|
|
56
|
+
intent = null;
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
status = 'dead';
|
|
60
|
+
intent = null;
|
|
61
|
+
}
|
|
62
|
+
return { ...node, status, intent, terminal_reason: outcome.reason };
|
|
63
|
+
}
|
|
64
|
+
/** Compose every delivery document from the hydrated identity and the immutable outcome latch. */
|
|
65
|
+
export function composeOutcomeDeliveryDocument(args) {
|
|
66
|
+
const node = getNode(args.nodeId);
|
|
67
|
+
if (node === null)
|
|
68
|
+
throw new Error(`unknown node ${args.nodeId}`);
|
|
69
|
+
return JSON.stringify(composeNodeOutcomeDocument({
|
|
70
|
+
node: nodeAtOutcome(node, args.outcome),
|
|
71
|
+
outcome: args.outcome,
|
|
72
|
+
action: args.action,
|
|
73
|
+
...(args.finalReport === undefined ? {} : { finalReport: args.finalReport }),
|
|
74
|
+
...(args.structuredResult === undefined ? {} : { structuredResult: args.structuredResult }),
|
|
75
|
+
}));
|
|
76
|
+
}
|
|
77
|
+
/** Register a target atomically against the outcome latch. A still-armed target
|
|
78
|
+
* is replaceable; a settled registration is not. When the outcome already
|
|
79
|
+
* exists, the row is born pending with its complete immutable document. */
|
|
80
|
+
export function armOutcomeDelivery(row) {
|
|
81
|
+
const now = row.now ?? Date.now();
|
|
82
|
+
const payloadJson = serializedPayload(row.payload);
|
|
83
|
+
return withCanvasWrite((db) => {
|
|
84
|
+
const node = db.prepare('SELECT * FROM nodes WHERE node_id = ?').get(row.nodeId);
|
|
85
|
+
if (node === undefined)
|
|
86
|
+
throw new Error(`unknown node ${row.nodeId}`);
|
|
87
|
+
const settled = node.outcome_at !== null;
|
|
88
|
+
if (settled && (node.outcome_kind === null || node.outcome_reason === null)) {
|
|
89
|
+
throw new Error(`outcome latch is incomplete for node ${row.nodeId}`);
|
|
90
|
+
}
|
|
91
|
+
const documentJson = !settled
|
|
92
|
+
? null
|
|
93
|
+
: composeOutcomeDeliveryDocument({
|
|
94
|
+
nodeId: row.nodeId,
|
|
95
|
+
outcome: {
|
|
96
|
+
kind: node.outcome_kind,
|
|
97
|
+
reason: node.outcome_reason,
|
|
98
|
+
at: node.outcome_at,
|
|
99
|
+
revision: node.outcome_revision,
|
|
100
|
+
detailJson: node.outcome_detail_json,
|
|
101
|
+
},
|
|
102
|
+
action: { name: row.actionName, payload: row.payload ?? null },
|
|
103
|
+
finalReport: node.final_report,
|
|
104
|
+
});
|
|
105
|
+
const state = settled ? 'pending' : 'armed';
|
|
106
|
+
const inserted = db.prepare(`INSERT INTO node_outcome_deliveries
|
|
107
|
+
(node_id, state, action_name, argv_json, cwd, config_origin, payload_json, document_json, next_attempt_at, created_at, updated_at)
|
|
108
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
109
|
+
ON CONFLICT(node_id) DO UPDATE SET
|
|
110
|
+
state = excluded.state, action_name = excluded.action_name, argv_json = excluded.argv_json,
|
|
111
|
+
cwd = excluded.cwd, config_origin = excluded.config_origin, payload_json = excluded.payload_json,
|
|
112
|
+
document_json = excluded.document_json, next_attempt_at = excluded.next_attempt_at,
|
|
113
|
+
updated_at = excluded.updated_at
|
|
114
|
+
WHERE node_outcome_deliveries.state = 'armed'`).run(row.nodeId, state, row.actionName, JSON.stringify(row.argv), row.cwd, row.configOrigin, payloadJson, documentJson, settled ? now : 0, now, now);
|
|
115
|
+
if (Number(inserted.changes) === 0)
|
|
116
|
+
return null;
|
|
117
|
+
return outcomeDeliveryFrom(db.prepare('SELECT * FROM node_outcome_deliveries WHERE node_id = ?').get(row.nodeId));
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
export function readOutcomeDelivery(nodeId) {
|
|
121
|
+
const row = openDb().prepare('SELECT * FROM node_outcome_deliveries WHERE node_id = ?').get(nodeId);
|
|
122
|
+
return row === undefined ? null : outcomeDeliveryFrom(row);
|
|
123
|
+
}
|
|
124
|
+
export function dueOutcomeDeliveries(now) {
|
|
125
|
+
return openDb().prepare(`SELECT * FROM node_outcome_deliveries
|
|
126
|
+
WHERE state = 'pending' AND next_attempt_at <= ?
|
|
127
|
+
ORDER BY next_attempt_at, created_at, node_id`).all(now).map(outcomeDeliveryFrom);
|
|
128
|
+
}
|
|
129
|
+
export function claimOutcomeDelivery(nodeId, now, claimOwner) {
|
|
130
|
+
return withCanvasWrite((db) => {
|
|
131
|
+
const claimed = db.prepare(`UPDATE node_outcome_deliveries
|
|
132
|
+
SET state = 'running', attempt = attempt + 1, claimed_at = ?, claim_owner = ?, updated_at = ?
|
|
133
|
+
WHERE node_id = ? AND state = 'pending' AND next_attempt_at <= ?`).run(now, claimOwner, now, nodeId, now);
|
|
134
|
+
if (Number(claimed.changes) === 0)
|
|
135
|
+
return null;
|
|
136
|
+
return outcomeDeliveryFrom(db.prepare('SELECT * FROM node_outcome_deliveries WHERE node_id = ?').get(nodeId));
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
/** Settle only the still-owned running attempt, fencing stale delivery processes. */
|
|
140
|
+
export function settleOutcomeDelivery(nodeId, attempt, claimOwner, settlement, now) {
|
|
141
|
+
const failure = settlement.state === 'accepted' ? null : JSON.stringify(settlement.failure);
|
|
142
|
+
return withCanvasWrite((db) => Number(db.prepare(`UPDATE node_outcome_deliveries
|
|
143
|
+
SET state = ?, next_attempt_at = ?, claimed_at = NULL, claim_owner = NULL,
|
|
144
|
+
accepted_at = ?, permanent_failed_at = ?, last_failure_json = ?, updated_at = ?
|
|
145
|
+
WHERE node_id = ? AND state = 'running' AND attempt = ? AND claim_owner = ?`).run(settlement.state, settlement.state === 'pending' ? settlement.nextAttemptAt : now, settlement.state === 'accepted' ? now : null, settlement.state === 'permanent_failed' ? now : null, failure, now, nodeId, attempt, claimOwner).changes) === 1);
|
|
146
|
+
}
|
|
147
|
+
export function recoverRunningOutcomeDeliveries(now) {
|
|
148
|
+
return withCanvasWrite((db) => Number(db.prepare(`UPDATE node_outcome_deliveries
|
|
149
|
+
SET state = 'pending', next_attempt_at = ?, claimed_at = NULL, claim_owner = NULL, updated_at = ?
|
|
150
|
+
WHERE state = 'running'`).run(now, now).changes));
|
|
151
|
+
}
|
|
152
|
+
/** Disarming is deliberately limited to registrations that have not started delivery. */
|
|
153
|
+
export function disarmOutcomeDelivery(nodeId) {
|
|
154
|
+
return withCanvasWrite((db) => Number(db.prepare("DELETE FROM node_outcome_deliveries WHERE node_id = ? AND state = 'armed'").run(nodeId).changes) === 1);
|
|
155
|
+
}
|
|
156
|
+
/** A canonical final is the one allowed outcome supersede: discard any old claim and retry state. */
|
|
157
|
+
export function resetOutcomeDeliveryForSupersede(nodeId, documentJson, now) {
|
|
158
|
+
return withCanvasWrite((db) => Number(db.prepare(`UPDATE node_outcome_deliveries
|
|
159
|
+
SET state = 'pending', attempt = 0, document_json = ?, next_attempt_at = ?,
|
|
160
|
+
claimed_at = NULL, claim_owner = NULL, accepted_at = NULL, permanent_failed_at = NULL,
|
|
161
|
+
last_failure_json = NULL, updated_at = ?
|
|
162
|
+
WHERE node_id = ?`).run(documentJson, now, now, nodeId).changes) === 1);
|
|
163
|
+
}
|
|
@@ -4,6 +4,13 @@
|
|
|
4
4
|
* zombies and processes reaped between the two probes. Only an unavailable
|
|
5
5
|
* `ps` probe fails open to alive. A null/undefined pid reads dead. */
|
|
6
6
|
export declare function isPidAlive(pid: number | null | undefined): boolean;
|
|
7
|
+
/** Existence-only liveness for READ-ONLY display projections: the signal-0
|
|
8
|
+
* probe alone, no `ps`. A zombie reads alive here, which a dashboard column
|
|
9
|
+
* tolerates and which no lifecycle decision may rely on — revive, teardown,
|
|
10
|
+
* and every other authority stays on `isPidAlive`. Exists because the
|
|
11
|
+
* dashboard snapshot runs this per live row per request; a `ps` spawn each
|
|
12
|
+
* time was a third of daemon CPU on a large canvas. */
|
|
13
|
+
export declare function isPidPresent(pid: number | null | undefined): boolean;
|
|
7
14
|
/** THE single precision-aware identity compare, used EVERYWHERE two
|
|
8
15
|
* `composeIdentity` fingerprints are compared (this module's own
|
|
9
16
|
* `recordedPidLiveness`, `captureTeardownSnapshot`'s reuse check, and
|
package/dist/core/canvas/pid.js
CHANGED
|
@@ -46,6 +46,23 @@ export function isPidAlive(pid) {
|
|
|
46
46
|
const state = psProcessState(pid);
|
|
47
47
|
return state !== 'zombie' && state !== 'gone';
|
|
48
48
|
}
|
|
49
|
+
/** Existence-only liveness for READ-ONLY display projections: the signal-0
|
|
50
|
+
* probe alone, no `ps`. A zombie reads alive here, which a dashboard column
|
|
51
|
+
* tolerates and which no lifecycle decision may rely on — revive, teardown,
|
|
52
|
+
* and every other authority stays on `isPidAlive`. Exists because the
|
|
53
|
+
* dashboard snapshot runs this per live row per request; a `ps` spawn each
|
|
54
|
+
* time was a third of daemon CPU on a large canvas. */
|
|
55
|
+
export function isPidPresent(pid) {
|
|
56
|
+
if (pid == null)
|
|
57
|
+
return false;
|
|
58
|
+
try {
|
|
59
|
+
process.kill(pid, 0);
|
|
60
|
+
return true;
|
|
61
|
+
}
|
|
62
|
+
catch (e) {
|
|
63
|
+
return e.code === 'EPERM';
|
|
64
|
+
}
|
|
65
|
+
}
|
|
49
66
|
/** Matches the kernel `boot_id` UUID shape (`/proc/sys/kernel/random/boot_id`),
|
|
50
67
|
* used to tell a NEW-format identity base (a per-boot UUID) apart from a
|
|
51
68
|
* LEGACY one (a `ps lstart` wall-clock timestamp). A boot_id is a lowercase
|