@intx/workflow-host 0.2.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +77 -14
- package/dist/adapters/mail-part-store.d.ts +46 -0
- package/dist/adapters/mail-part-store.js +251 -0
- package/dist/adapters/repo-store.d.ts +22 -1
- package/dist/adapters/repo-store.js +56 -65
- package/dist/adapters/spawn-child.d.ts +109 -44
- package/dist/adapters/spawn-child.js +77 -81
- package/dist/adapters/step-invoker.d.ts +52 -2
- package/dist/adapters/step-invoker.js +284 -37
- package/dist/adapters/substrate-mailbox-store.d.ts +80 -0
- package/dist/adapters/substrate-mailbox-store.js +404 -0
- package/dist/child/child-mailbox-reader.d.ts +10 -0
- package/dist/child/child-mailbox-reader.js +23 -0
- package/dist/child/credential-cell.d.ts +8 -0
- package/dist/child/credential-cell.js +66 -0
- package/dist/child/env-bootstrap.d.ts +20 -6
- package/dist/child/env-bootstrap.js +9 -1
- package/dist/child/from-process-env.d.ts +12 -0
- package/dist/child/from-process-env.js +6 -0
- package/dist/child/index.d.ts +6 -2
- package/dist/child/index.js +4 -1
- package/dist/child/mailbox-mutation-bridge.d.ts +61 -0
- package/dist/child/mailbox-mutation-bridge.js +101 -0
- package/dist/child/mailbox-watch-registry.d.ts +17 -0
- package/dist/child/mailbox-watch-registry.js +61 -0
- package/dist/child/outbound-mail-bridge.d.ts +3 -2
- package/dist/child/outbound-mail-bridge.js +20 -32
- package/dist/child/parked-correlations.d.ts +42 -0
- package/dist/child/parked-correlations.js +80 -0
- package/dist/child/pending-request.d.ts +89 -0
- package/dist/child/pending-request.js +80 -0
- package/dist/child/proxy-repo-store.d.ts +3 -2
- package/dist/child/proxy-repo-store.js +2 -0
- package/dist/child/run-child.d.ts +170 -14
- package/dist/child/run-child.js +569 -155
- package/dist/child/self-discovery.d.ts +10 -0
- package/dist/child/self-discovery.js +25 -1
- package/dist/child/substrate-write-bridge.d.ts +3 -2
- package/dist/child/substrate-write-bridge.js +21 -38
- package/dist/child/supervisor-backed-transport.d.ts +52 -6
- package/dist/child/supervisor-backed-transport.js +205 -62
- package/dist/child/verified-definition-loader.d.ts +33 -0
- package/dist/child/verified-definition-loader.js +43 -0
- package/dist/child/warm-agent-cache.d.ts +44 -4
- package/dist/child/warm-agent-cache.js +41 -10
- package/dist/index.d.ts +6 -4
- package/dist/index.js +6 -4
- package/dist/ipc/control-channel.d.ts +151 -2
- package/dist/ipc/control-channel.js +222 -29
- package/dist/ipc/event-channel.d.ts +32 -1
- package/dist/ipc/index.d.ts +1 -1
- package/dist/ipc/index.js +1 -1
- package/dist/mail-bus/hub-transport-adapter.d.ts +12 -7
- package/dist/mail-bus/hub-transport-adapter.js +9 -5
- package/dist/run-body-then-cleanup.d.ts +17 -0
- package/dist/run-body-then-cleanup.js +38 -0
- package/dist/seams/scheduler.d.ts +16 -6
- package/dist/seams/scheduler.js +87 -97
- package/dist/supervisor/cancel-signing.d.ts +2 -2
- package/dist/supervisor/cancel-signing.js +4 -8
- package/dist/supervisor/credentials.d.ts +28 -15
- package/dist/supervisor/credentials.js +7 -7
- package/dist/supervisor/dispatch-attribution.js +1 -1
- package/dist/supervisor/drain-timeout.d.ts +2 -2
- package/dist/supervisor/drain-timeout.js +1 -1
- package/dist/supervisor/index.d.ts +3 -3
- package/dist/supervisor/index.js +2 -2
- package/dist/supervisor/recycle.d.ts +10 -3
- package/dist/supervisor/recycle.js +18 -7
- package/dist/supervisor/run-event-compaction.d.ts +5 -5
- package/dist/supervisor/run-event-compaction.js +14 -19
- package/dist/supervisor/run-event-recovery.d.ts +34 -0
- package/dist/supervisor/run-event-recovery.js +45 -0
- package/dist/supervisor/spawn-env.d.ts +2 -2
- package/dist/supervisor/spawn-env.js +1 -1
- package/dist/supervisor/supervisor.d.ts +106 -26
- package/dist/supervisor/supervisor.js +1903 -414
- package/dist/supervisor/terminal-commit.d.ts +36 -0
- package/dist/supervisor/terminal-commit.js +130 -0
- package/dist/supervisor/types.d.ts +180 -23
- package/dist/testing/change-notifier.d.ts +12 -0
- package/dist/testing/change-notifier.js +63 -0
- package/dist/testing/index.d.ts +8 -0
- package/dist/testing/index.js +16 -0
- package/dist/testing/log-capture.d.ts +52 -0
- package/dist/testing/log-capture.js +124 -0
- package/dist/testing/mail-bus.d.ts +22 -0
- package/dist/testing/mail-bus.js +78 -0
- package/dist/testing/memory-streams.d.ts +43 -0
- package/dist/testing/memory-streams.js +211 -0
- package/dist/testing/spawn-observer.d.ts +12 -0
- package/dist/testing/spawn-observer.js +36 -0
- package/dist/testing/stub-repo-store.d.ts +10 -0
- package/dist/testing/stub-repo-store.js +39 -0
- package/dist/testing/supervisor-reaper.d.ts +24 -0
- package/dist/testing/supervisor-reaper.js +49 -0
- package/dist/testing/upstream-frames.d.ts +47 -0
- package/dist/testing/upstream-frames.js +94 -0
- package/dist/workflow-definition-loader.d.ts +187 -0
- package/dist/workflow-definition-loader.js +422 -0
- package/package.json +18 -11
package/dist/seams/scheduler.js
CHANGED
|
@@ -26,17 +26,19 @@
|
|
|
26
26
|
//
|
|
27
27
|
// The scheduler reads workflow-event blobs at the canonical layout
|
|
28
28
|
// `runs/<runId>/events/<seq>.json` and writes a fresh blob at the
|
|
29
|
-
// next-seq slot for each `TimerFired` commit. The blob envelope
|
|
30
|
-
//
|
|
31
|
-
//
|
|
32
|
-
// `
|
|
33
|
-
//
|
|
34
|
-
//
|
|
35
|
-
//
|
|
36
|
-
// `
|
|
37
|
-
//
|
|
29
|
+
// next-seq slot for each `TimerFired` commit. The blob envelope is
|
|
30
|
+
// FLAT -- `{ seq, type, ...eventFields }` -- matching the runtime
|
|
31
|
+
// repo-store's `workflowEventToOnDisk` and the workflow-run kind
|
|
32
|
+
// handler's `EventEnvelope` contract: `seq` is the integer that also
|
|
33
|
+
// appears in the filename, `type` is the `subscribeKind` discriminator
|
|
34
|
+
// the scheduler narrows on, and the timer fields (`timerId`, `fireAt`,
|
|
35
|
+
// ...) sit alongside them, NOT under a nested `data` object. The kind
|
|
36
|
+
// handler's `validatePush` enforces that every event blob's body `seq`
|
|
37
|
+
// matches the filename's seq, so the scheduler mints the next seq
|
|
38
|
+
// inside the `writeTreePreservingPrefix` merge step and writes both
|
|
39
|
+
// into the envelope and the filename.
|
|
38
40
|
import { type } from "arktype";
|
|
39
|
-
import { subscribeKind } from "@intx/hub-sessions/substrate";
|
|
41
|
+
import { requireEventSeq, subscribeKind, WORKFLOW_RUN_EVENTS_DIR, WORKFLOW_RUN_RUNS_PREFIX, } from "@intx/hub-sessions/substrate";
|
|
40
42
|
/**
|
|
41
43
|
* Substrate-shape envelope for the workflow-event blob committed to
|
|
42
44
|
* `runs/<runId>/events/<seq>.json`. The validator covers the two
|
|
@@ -49,14 +51,27 @@ import { subscribeKind } from "@intx/hub-sessions/substrate";
|
|
|
49
51
|
export const TimerEventEnvelope = type({
|
|
50
52
|
seq: "number >= 0",
|
|
51
53
|
type: "'TimerSet' | 'TimerFired'",
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
54
|
+
// FLAT on-disk shape, matching the runtime repo-store's `workflowEventToOnDisk`
|
|
55
|
+
// ({ seq, type, ...eventFields }) and the substrate's enforced `EventEnvelope`
|
|
56
|
+
// contract -- NOT a nested `data` object. The scheduler was the lone component
|
|
57
|
+
// writing/reading a nested `data` envelope; nothing else parsed it, so a
|
|
58
|
+
// deployed timer never round-tripped until this was aligned.
|
|
59
|
+
timerId: "string",
|
|
60
|
+
"fireAt?": "string",
|
|
61
|
+
"stepId?": "string | null",
|
|
62
|
+
"cron?": "string | null",
|
|
63
|
+
"+": "ignore",
|
|
58
64
|
});
|
|
59
65
|
export function createWorkflowHostScheduler(opts) {
|
|
66
|
+
// Production arms the global timer; a caller supplying its own can fire a
|
|
67
|
+
// queued timer on demand instead of waiting out its delay.
|
|
68
|
+
const schedule = opts.scheduleTimeout ??
|
|
69
|
+
((handler, ms) => {
|
|
70
|
+
const handle = setTimeout(handler, ms);
|
|
71
|
+
return () => {
|
|
72
|
+
clearTimeout(handle);
|
|
73
|
+
};
|
|
74
|
+
});
|
|
60
75
|
const queues = new Map();
|
|
61
76
|
const liveSubscriptions = [];
|
|
62
77
|
let started = false;
|
|
@@ -81,7 +96,7 @@ export function createWorkflowHostScheduler(opts) {
|
|
|
81
96
|
if (queues.has(key))
|
|
82
97
|
return; // idempotent
|
|
83
98
|
const delayMs = Math.max(0, fireAtMs - opts.clock().getTime());
|
|
84
|
-
const
|
|
99
|
+
const cancelTimeout = schedule(() => {
|
|
85
100
|
void fireTimer(runId, timerId).catch((cause) => {
|
|
86
101
|
// The scheduler's commit failed. Surface as unhandled so
|
|
87
102
|
// operators see it; the runtime body's awaiter will hang
|
|
@@ -91,7 +106,7 @@ export function createWorkflowHostScheduler(opts) {
|
|
|
91
106
|
: new Error(`scheduler ${String(repoId.id)}/${runId}/${timerId} commit failed: ${String(cause)}`);
|
|
92
107
|
});
|
|
93
108
|
}, delayMs);
|
|
94
|
-
queues.set(key, { runId, timerId, fireAtMs,
|
|
109
|
+
queues.set(key, { runId, timerId, fireAtMs, cancelTimeout, cron });
|
|
95
110
|
}
|
|
96
111
|
function startLiveSubscription(repoId) {
|
|
97
112
|
const abort = new AbortController();
|
|
@@ -106,21 +121,21 @@ export function createWorkflowHostScheduler(opts) {
|
|
|
106
121
|
break;
|
|
107
122
|
if (entry.event.type !== "TimerSet")
|
|
108
123
|
continue;
|
|
109
|
-
const fireAt = entry.event.
|
|
124
|
+
const fireAt = entry.event.fireAt;
|
|
110
125
|
if (fireAt === undefined) {
|
|
111
|
-
throw new Error(`scheduler live ingest: TimerSet in ${String(repoId.id)} run ${entry.runId} timer ${entry.event.
|
|
126
|
+
throw new Error(`scheduler live ingest: TimerSet in ${String(repoId.id)} run ${entry.runId} timer ${entry.event.timerId} missing fireAt`);
|
|
112
127
|
}
|
|
113
128
|
const fireAtMs = Date.parse(fireAt);
|
|
114
129
|
if (Number.isNaN(fireAtMs)) {
|
|
115
|
-
throw new Error(`scheduler live ingest: TimerSet in ${String(repoId.id)} run ${entry.runId} timer ${entry.event.
|
|
130
|
+
throw new Error(`scheduler live ingest: TimerSet in ${String(repoId.id)} run ${entry.runId} timer ${entry.event.timerId} fireAt unparseable: ${fireAt}`);
|
|
116
131
|
}
|
|
117
|
-
const cron = entry.event.
|
|
132
|
+
const cron = entry.event.cron !== undefined && entry.event.cron !== null;
|
|
118
133
|
if (cron && fireAtMs < opts.clock().getTime()) {
|
|
119
134
|
// Same missed-cron-tick spec as recovery: a cron TimerSet
|
|
120
135
|
// whose fireAt is in the past on arrival is dropped.
|
|
121
136
|
continue;
|
|
122
137
|
}
|
|
123
|
-
enqueue(repoId, entry.runId, entry.event.
|
|
138
|
+
enqueue(repoId, entry.runId, entry.event.timerId, fireAtMs, cron);
|
|
124
139
|
}
|
|
125
140
|
})();
|
|
126
141
|
liveSubscriptions.push({ abort, done });
|
|
@@ -133,24 +148,24 @@ export function createWorkflowHostScheduler(opts) {
|
|
|
133
148
|
const unfired = new Map();
|
|
134
149
|
for (const e of events) {
|
|
135
150
|
if (e.envelope.type === "TimerSet") {
|
|
136
|
-
const fireAt = e.envelope.
|
|
151
|
+
const fireAt = e.envelope.fireAt;
|
|
137
152
|
if (fireAt === undefined) {
|
|
138
|
-
throw new Error(`scheduler recovery: TimerSet in ${String(repoId.id)} run ${e.runId} timer ${e.envelope.
|
|
153
|
+
throw new Error(`scheduler recovery: TimerSet in ${String(repoId.id)} run ${e.runId} timer ${e.envelope.timerId} missing fireAt`);
|
|
139
154
|
}
|
|
140
155
|
const fireAtMs = Date.parse(fireAt);
|
|
141
156
|
if (Number.isNaN(fireAtMs)) {
|
|
142
|
-
throw new Error(`scheduler recovery: TimerSet in ${String(repoId.id)} run ${e.runId} timer ${e.envelope.
|
|
157
|
+
throw new Error(`scheduler recovery: TimerSet in ${String(repoId.id)} run ${e.runId} timer ${e.envelope.timerId} fireAt unparseable: ${fireAt}`);
|
|
143
158
|
}
|
|
144
|
-
const cron = e.envelope.
|
|
145
|
-
unfired.set(`${e.runId} ${e.envelope.
|
|
159
|
+
const cron = e.envelope.cron !== undefined && e.envelope.cron !== null;
|
|
160
|
+
unfired.set(`${e.runId} ${e.envelope.timerId}`, {
|
|
146
161
|
runId: e.runId,
|
|
147
|
-
timerId: e.envelope.
|
|
162
|
+
timerId: e.envelope.timerId,
|
|
148
163
|
fireAtMs,
|
|
149
164
|
cron,
|
|
150
165
|
});
|
|
151
166
|
}
|
|
152
167
|
else {
|
|
153
|
-
unfired.delete(`${e.runId} ${e.envelope.
|
|
168
|
+
unfired.delete(`${e.runId} ${e.envelope.timerId}`);
|
|
154
169
|
}
|
|
155
170
|
}
|
|
156
171
|
const now = opts.clock().getTime();
|
|
@@ -196,7 +211,7 @@ export function createWorkflowHostScheduler(opts) {
|
|
|
196
211
|
return;
|
|
197
212
|
stopped = true;
|
|
198
213
|
for (const t of queues.values())
|
|
199
|
-
|
|
214
|
+
t.cancelTimeout();
|
|
200
215
|
queues.clear();
|
|
201
216
|
for (const sub of liveSubscriptions.splice(0)) {
|
|
202
217
|
sub.abort.abort();
|
|
@@ -210,7 +225,7 @@ export function createWorkflowHostScheduler(opts) {
|
|
|
210
225
|
const entry = queues.get(key);
|
|
211
226
|
if (entry === undefined)
|
|
212
227
|
return;
|
|
213
|
-
|
|
228
|
+
entry.cancelTimeout();
|
|
214
229
|
queues.delete(key);
|
|
215
230
|
},
|
|
216
231
|
queuedTimers() {
|
|
@@ -224,14 +239,15 @@ export function createWorkflowHostScheduler(opts) {
|
|
|
224
239
|
}
|
|
225
240
|
/**
|
|
226
241
|
* Read every timer-event blob across every run under the given
|
|
227
|
-
* workflow-run repo.
|
|
228
|
-
*
|
|
229
|
-
*
|
|
230
|
-
*
|
|
231
|
-
*
|
|
232
|
-
*
|
|
233
|
-
*
|
|
234
|
-
* current ref tip
|
|
242
|
+
* workflow-run repo. Recovery reconstructs its ledger from committed
|
|
243
|
+
* state, so `enumerateEventBlobs` reads the git object store at the
|
|
244
|
+
* events ref tip via `openCommittedReads` rather than the materialized
|
|
245
|
+
* working tree, whose non-atomic post-commit checkout can lag the
|
|
246
|
+
* committed tree on a contended filesystem and hide an already-committed
|
|
247
|
+
* `TimerFired`. `subscribeKind` is a diff-shaped iterator over new
|
|
248
|
+
* commits, not a "list everything at HEAD" primitive, so the startup
|
|
249
|
+
* ledger needs this path-aware enumeration of the current ref tip rather
|
|
250
|
+
* than a tail subscription.
|
|
235
251
|
*/
|
|
236
252
|
async function readAllEvents(opts, repoId) {
|
|
237
253
|
const entries = [];
|
|
@@ -261,53 +277,36 @@ async function readAllEvents(opts, repoId) {
|
|
|
261
277
|
* has none.
|
|
262
278
|
*/
|
|
263
279
|
async function enumerateEventBlobs(opts, repoId) {
|
|
264
|
-
const
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
const runsDir = path.join(dir, "runs");
|
|
280
|
+
const reads = await opts.repoStore.openCommittedReads(opts.principal, repoId, opts.ref);
|
|
281
|
+
if (reads === null)
|
|
282
|
+
return [];
|
|
268
283
|
const out = [];
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
throw cause;
|
|
277
|
-
}
|
|
278
|
-
for (const runId of runEntries) {
|
|
279
|
-
const eventsDir = path.join(runsDir, runId, "events");
|
|
280
|
-
let blobs;
|
|
281
|
-
try {
|
|
282
|
-
blobs = await fs.readdir(eventsDir);
|
|
283
|
-
}
|
|
284
|
-
catch (cause) {
|
|
285
|
-
if (isErrnoNotFound(cause))
|
|
286
|
-
continue;
|
|
287
|
-
throw cause;
|
|
288
|
-
}
|
|
284
|
+
const runEntries = await reads.listDir(WORKFLOW_RUN_RUNS_PREFIX);
|
|
285
|
+
for (const runEntry of runEntries) {
|
|
286
|
+
if (runEntry.type !== "tree")
|
|
287
|
+
continue;
|
|
288
|
+
const runId = runEntry.name;
|
|
289
|
+
const eventsDir = `${WORKFLOW_RUN_RUNS_PREFIX}/${runId}/${WORKFLOW_RUN_EVENTS_DIR}`;
|
|
290
|
+
const blobs = await reads.listDir(eventsDir);
|
|
289
291
|
for (const blob of blobs) {
|
|
290
|
-
if (
|
|
292
|
+
if (blob.type !== "blob")
|
|
291
293
|
continue;
|
|
292
|
-
|
|
294
|
+
// Assert a legal <seq>.json name; an illegal one under the events
|
|
295
|
+
// prefix is corruption the recovery walk must not silently drop.
|
|
296
|
+
requireEventSeq(blob.name, `${eventsDir}/${blob.name}`);
|
|
297
|
+
const raw = await reads.readBlobByOid(blob.oid);
|
|
293
298
|
let parsed;
|
|
294
299
|
try {
|
|
295
|
-
parsed = JSON.parse(raw);
|
|
300
|
+
parsed = JSON.parse(new TextDecoder().decode(raw));
|
|
296
301
|
}
|
|
297
302
|
catch (cause) {
|
|
298
|
-
throw new Error(`scheduler recovery: cannot parse ${String(repoId.id)}/${runId}
|
|
303
|
+
throw new Error(`scheduler recovery: cannot parse ${String(repoId.id)}/${runId}/${WORKFLOW_RUN_EVENTS_DIR}/${blob.name}: ${String(cause)}`);
|
|
299
304
|
}
|
|
300
305
|
out.push({ runId, payload: parsed });
|
|
301
306
|
}
|
|
302
307
|
}
|
|
303
308
|
return out;
|
|
304
309
|
}
|
|
305
|
-
function isErrnoNotFound(cause) {
|
|
306
|
-
if (cause === null || typeof cause !== "object")
|
|
307
|
-
return false;
|
|
308
|
-
const code = cause.code;
|
|
309
|
-
return code === "ENOENT";
|
|
310
|
-
}
|
|
311
310
|
/**
|
|
312
311
|
* Commit a `TimerFired` event blob to the workflow-run repo. The
|
|
313
312
|
* commit goes through `writeTreePreservingPrefix` so concurrent
|
|
@@ -321,7 +320,7 @@ async function commitTimerFired(opts, runId, timerId) {
|
|
|
321
320
|
if (owningRepoId === undefined) {
|
|
322
321
|
throw new Error(`scheduler commit: cannot find deployment owning run ${runId}`);
|
|
323
322
|
}
|
|
324
|
-
const prefix =
|
|
323
|
+
const prefix = `${WORKFLOW_RUN_RUNS_PREFIX}/${runId}/${WORKFLOW_RUN_EVENTS_DIR}/`;
|
|
325
324
|
await opts.repoStore.writeTreePreservingPrefix(opts.principal, owningRepoId, opts.ref, {
|
|
326
325
|
preservePrefix: prefix,
|
|
327
326
|
merge: async (existing) => {
|
|
@@ -329,13 +328,7 @@ async function commitTimerFired(opts, runId, timerId) {
|
|
|
329
328
|
let alreadyFired = false;
|
|
330
329
|
for (const [filepath, contents] of existing) {
|
|
331
330
|
const name = filepath.slice(prefix.length);
|
|
332
|
-
const
|
|
333
|
-
if (match === null)
|
|
334
|
-
continue;
|
|
335
|
-
const seqStr = match[1];
|
|
336
|
-
if (seqStr === undefined)
|
|
337
|
-
continue;
|
|
338
|
-
const seq = Number.parseInt(seqStr, 10);
|
|
331
|
+
const seq = requireEventSeq(name, `${prefix}${name}`);
|
|
339
332
|
if (seq > maxSeq)
|
|
340
333
|
maxSeq = seq;
|
|
341
334
|
try {
|
|
@@ -360,7 +353,7 @@ async function commitTimerFired(opts, runId, timerId) {
|
|
|
360
353
|
out[`${prefix}${String(nextSeq)}.json`] = JSON.stringify({
|
|
361
354
|
seq: nextSeq,
|
|
362
355
|
type: "TimerFired",
|
|
363
|
-
|
|
356
|
+
timerId,
|
|
364
357
|
});
|
|
365
358
|
return out;
|
|
366
359
|
},
|
|
@@ -373,25 +366,22 @@ function isMatchingTimerFired(parsed, timerId) {
|
|
|
373
366
|
const obj = parsed;
|
|
374
367
|
if (obj.type !== "TimerFired")
|
|
375
368
|
return false;
|
|
376
|
-
|
|
377
|
-
return false;
|
|
378
|
-
return obj.data.timerId === timerId;
|
|
369
|
+
return obj.timerId === timerId;
|
|
379
370
|
}
|
|
380
371
|
async function findOwningDeployment(opts, runId) {
|
|
381
372
|
const repoIds = await opts.listActiveDeployments();
|
|
382
|
-
const fs = await import("node:fs/promises");
|
|
383
|
-
const path = await import("node:path");
|
|
384
373
|
for (const repoId of repoIds) {
|
|
385
|
-
const
|
|
386
|
-
|
|
387
|
-
|
|
374
|
+
const reads = await opts.repoStore.openCommittedReads(opts.principal, repoId, opts.ref);
|
|
375
|
+
if (reads === null)
|
|
376
|
+
continue;
|
|
377
|
+
// The committed events subtree exists only when it holds at least
|
|
378
|
+
// one blob (git does not track empty trees), so a non-empty listing
|
|
379
|
+
// is exactly "this deployment owns the run". Reading committed state
|
|
380
|
+
// rather than the working tree keeps attribution correct when the
|
|
381
|
+
// checkout lags the object store.
|
|
382
|
+
const events = await reads.listDir(`${WORKFLOW_RUN_RUNS_PREFIX}/${runId}/${WORKFLOW_RUN_EVENTS_DIR}`);
|
|
383
|
+
if (events.length > 0)
|
|
388
384
|
return repoId;
|
|
389
|
-
}
|
|
390
|
-
catch (cause) {
|
|
391
|
-
if (isErrnoNotFound(cause))
|
|
392
|
-
continue;
|
|
393
|
-
throw cause;
|
|
394
|
-
}
|
|
395
385
|
}
|
|
396
386
|
return undefined;
|
|
397
387
|
}
|
|
@@ -16,8 +16,8 @@ export type CommitCancelRequestedOpts = {
|
|
|
16
16
|
repoId: RepoId;
|
|
17
17
|
/** Events ref the workflow-run repo writes to. */
|
|
18
18
|
ref: string;
|
|
19
|
-
/**
|
|
20
|
-
|
|
19
|
+
/** Anchor run id used to construct the supervisor principal. */
|
|
20
|
+
anchorRunId: string;
|
|
21
21
|
/** Run id whose event log receives the CancelRequested entry. */
|
|
22
22
|
runId: string;
|
|
23
23
|
/** Cancellation origin from the Q3 map. */
|
|
@@ -24,6 +24,7 @@
|
|
|
24
24
|
// `supervisor`-kind principal), which is the cross-check the
|
|
25
25
|
// supervisor's runtime-side signing keeps coherent.
|
|
26
26
|
import { type } from "arktype";
|
|
27
|
+
import { parseEventSeq } from "@intx/hub-sessions/substrate";
|
|
27
28
|
import { hexEncode } from "@intx/types";
|
|
28
29
|
/**
|
|
29
30
|
* Path inside the workflow-run repo each `CancelRequested` event
|
|
@@ -40,7 +41,6 @@ const EVENTS_DIR = "events";
|
|
|
40
41
|
* push principal and enforces the principal-vs-origin map.
|
|
41
42
|
*/
|
|
42
43
|
export const SUPERVISOR_PRINCIPAL_KIND = "supervisor";
|
|
43
|
-
const EVENT_FILENAME_RE = /^(0|[1-9][0-9]*)\.json$/;
|
|
44
44
|
const OnDiskEnvelope = type({
|
|
45
45
|
seq: "number >= 0",
|
|
46
46
|
type: "string",
|
|
@@ -76,7 +76,7 @@ export async function commitCancelRequested(opts) {
|
|
|
76
76
|
const prefix = `${RUNS_PREFIX}/${opts.runId}/${EVENTS_DIR}/`;
|
|
77
77
|
const principal = {
|
|
78
78
|
kind: SUPERVISOR_PRINCIPAL_KIND,
|
|
79
|
-
|
|
79
|
+
anchorRunId: opts.anchorRunId,
|
|
80
80
|
};
|
|
81
81
|
let resolved = null;
|
|
82
82
|
const { commitSha } = await opts.substrate.writeTreePreservingPrefix(principal, opts.repoId, opts.ref, {
|
|
@@ -85,13 +85,9 @@ export async function commitCancelRequested(opts) {
|
|
|
85
85
|
let maxSeq = -1;
|
|
86
86
|
for (const filepath of existing.keys()) {
|
|
87
87
|
const name = filepath.slice(prefix.length);
|
|
88
|
-
const
|
|
89
|
-
if (
|
|
88
|
+
const seq = parseEventSeq(name);
|
|
89
|
+
if (seq === null)
|
|
90
90
|
continue;
|
|
91
|
-
const seqStr = match[1];
|
|
92
|
-
if (seqStr === undefined)
|
|
93
|
-
continue;
|
|
94
|
-
const seq = Number.parseInt(seqStr, 10);
|
|
95
91
|
if (seq > maxSeq)
|
|
96
92
|
maxSeq = seq;
|
|
97
93
|
}
|
|
@@ -14,7 +14,11 @@ export declare const STEP_GRANTS_PATH = "state/grants.json";
|
|
|
14
14
|
*/
|
|
15
15
|
export declare const STEP_GRANTS_REF = "refs/heads/main";
|
|
16
16
|
export type CredentialsSnapshotStep = {
|
|
17
|
-
/**
|
|
17
|
+
/**
|
|
18
|
+
* Workflow step id: a `WorkflowDefinition.stepOrder` entry, or a step id
|
|
19
|
+
* from one of the definition's `loop` bodies (a loop body shares the
|
|
20
|
+
* enclosing definition's flat step-id namespace).
|
|
21
|
+
*/
|
|
18
22
|
stepId: string;
|
|
19
23
|
/** Mail address the step's agent presents to the bus. */
|
|
20
24
|
address: string;
|
|
@@ -24,29 +28,29 @@ export type CredentialsSnapshotStep = {
|
|
|
24
28
|
contentHash: string;
|
|
25
29
|
};
|
|
26
30
|
export type CredentialsSnapshot = {
|
|
27
|
-
/** Step-id keyed entries in
|
|
31
|
+
/** Step-id keyed entries in the caller's traversal order. */
|
|
28
32
|
steps: readonly CredentialsSnapshotStep[];
|
|
29
33
|
};
|
|
30
34
|
/**
|
|
31
35
|
* Caller-supplied derivation of the per-step mail address from the
|
|
32
|
-
*
|
|
36
|
+
* run id and step id. The supervisor cannot encode the
|
|
33
37
|
* deployment-domain inside library code; the wiring module supplies
|
|
34
38
|
* the strategy the host owns.
|
|
35
39
|
*/
|
|
36
40
|
export type DeriveStepAddress = (args: {
|
|
37
|
-
|
|
41
|
+
runId: string;
|
|
38
42
|
stepId: string;
|
|
39
43
|
}) => string;
|
|
40
44
|
/**
|
|
41
45
|
* Caller-supplied override of the per-step `agent-state` repo identity
|
|
42
46
|
* the supervisor reads grants from. Defaults to the
|
|
43
|
-
* `<
|
|
47
|
+
* `<runId>-<stepId>` convention (`defaultStepRepoId`); the
|
|
44
48
|
* single-step launched-agent deploy supplies a derivation that returns
|
|
45
49
|
* the legacy agent-state repo so the child reads grants from the same
|
|
46
50
|
* repo the legacy agent identity already keys.
|
|
47
51
|
*/
|
|
48
52
|
export type DeriveStepRepoId = (args: {
|
|
49
|
-
|
|
53
|
+
runId: string;
|
|
50
54
|
stepId: string;
|
|
51
55
|
}) => RepoId;
|
|
52
56
|
export type AssembleCredentialsSnapshotOpts = {
|
|
@@ -55,32 +59,40 @@ export type AssembleCredentialsSnapshotOpts = {
|
|
|
55
59
|
/** Principal presented for each step's read. */
|
|
56
60
|
principal: Principal;
|
|
57
61
|
/**
|
|
58
|
-
*
|
|
59
|
-
* passes a single entry; multi-step deployments pass every
|
|
60
|
-
* the order the workflow asset declared.
|
|
62
|
+
* Every step id the snapshot must carry an entry for. The trivial
|
|
63
|
+
* workflow passes a single entry; multi-step deployments pass every
|
|
64
|
+
* step in the order the workflow asset declared.
|
|
65
|
+
*
|
|
66
|
+
* This is the deployment's flat step-id namespace, which is WIDER than
|
|
67
|
+
* the definition's own `stepOrder`: a `loop` body runs in-process as a
|
|
68
|
+
* child run inheriting the parent's env, so a body step authorizes
|
|
69
|
+
* against this same snapshot under its own plain step id. The caller
|
|
70
|
+
* owns that widening -- it is the layer that holds the definition --
|
|
71
|
+
* and the snapshot is total over whatever it passes, because the
|
|
72
|
+
* child's authorize treats a missing entry as unrecoverable.
|
|
61
73
|
*/
|
|
62
74
|
stepOrder: readonly string[];
|
|
63
|
-
/**
|
|
64
|
-
|
|
75
|
+
/** Anchor run id used in agent-state repo identity and address derivation. */
|
|
76
|
+
anchorRunId: string;
|
|
65
77
|
/** Per-step mail-address derivation callback. */
|
|
66
78
|
deriveStepAddress: DeriveStepAddress;
|
|
67
79
|
/**
|
|
68
80
|
* Optional override for the `agent-state` repo's id. Callers that
|
|
69
|
-
* follow the documented convention (`<
|
|
81
|
+
* follow the documented convention (`<runId>-<stepId>`) can
|
|
70
82
|
* omit this; tests and bespoke layouts can supply their own.
|
|
71
83
|
*/
|
|
72
84
|
deriveStepRepoId?: DeriveStepRepoId;
|
|
73
85
|
};
|
|
74
86
|
/**
|
|
75
|
-
* Default mapping from `(
|
|
76
|
-
* repo id: `<
|
|
87
|
+
* Default mapping from `(runId, stepId)` to the agent-state
|
|
88
|
+
* repo id: `<runId>-<stepId>`, isolating each step's grants in
|
|
77
89
|
* its own repo. Applied to a one-step `stepOrder` it yields a single
|
|
78
90
|
* such repo. The single-step launched-agent deploy overrides this
|
|
79
91
|
* default (see `DeriveStepRepoId`) to reuse the legacy agent-state
|
|
80
92
|
* repo.
|
|
81
93
|
*/
|
|
82
94
|
export declare function defaultStepRepoId(args: {
|
|
83
|
-
|
|
95
|
+
runId: string;
|
|
84
96
|
stepId: string;
|
|
85
97
|
}): RepoId;
|
|
86
98
|
/**
|
|
@@ -99,3 +111,4 @@ export declare function hashGrants(grants: readonly unknown[]): Promise<string>;
|
|
|
99
111
|
* arrives after a fresher one.
|
|
100
112
|
*/
|
|
101
113
|
export declare function assembleCredentialsSnapshot(opts: AssembleCredentialsSnapshotOpts): Promise<CredentialsSnapshot>;
|
|
114
|
+
export declare function isErrnoNotFound(cause: unknown): boolean;
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
// substrate's git layout.
|
|
21
21
|
//
|
|
22
22
|
// Per-step address derivation (Q6.4 discovery decision):
|
|
23
|
-
// - Multi-step deployments use `<
|
|
23
|
+
// - Multi-step deployments use `<runId>-<stepId>@<domain>`.
|
|
24
24
|
// - Trivial (single-step) deployments use the deployment's own
|
|
25
25
|
// mail address as the sole step's address.
|
|
26
26
|
// The derivation is supplied by the caller as a `deriveStepAddress`
|
|
@@ -51,8 +51,8 @@ const StepGrantsFile = type({
|
|
|
51
51
|
grants: "unknown[]",
|
|
52
52
|
}).onUndeclaredKey("ignore");
|
|
53
53
|
/**
|
|
54
|
-
* Default mapping from `(
|
|
55
|
-
* repo id: `<
|
|
54
|
+
* Default mapping from `(runId, stepId)` to the agent-state
|
|
55
|
+
* repo id: `<runId>-<stepId>`, isolating each step's grants in
|
|
56
56
|
* its own repo. Applied to a one-step `stepOrder` it yields a single
|
|
57
57
|
* such repo. The single-step launched-agent deploy overrides this
|
|
58
58
|
* default (see `DeriveStepRepoId`) to reuse the legacy agent-state
|
|
@@ -61,7 +61,7 @@ const StepGrantsFile = type({
|
|
|
61
61
|
export function defaultStepRepoId(args) {
|
|
62
62
|
return {
|
|
63
63
|
kind: "agent-state",
|
|
64
|
-
id: `${args.
|
|
64
|
+
id: `${args.runId}-${args.stepId}`,
|
|
65
65
|
};
|
|
66
66
|
}
|
|
67
67
|
/**
|
|
@@ -128,12 +128,12 @@ export async function assembleCredentialsSnapshot(opts) {
|
|
|
128
128
|
const steps = [];
|
|
129
129
|
for (const stepId of opts.stepOrder) {
|
|
130
130
|
const repoId = deriveRepoId({
|
|
131
|
-
|
|
131
|
+
runId: opts.anchorRunId,
|
|
132
132
|
stepId,
|
|
133
133
|
});
|
|
134
134
|
const grants = await readStepGrants(opts, repoId);
|
|
135
135
|
const address = opts.deriveStepAddress({
|
|
136
|
-
|
|
136
|
+
runId: opts.anchorRunId,
|
|
137
137
|
stepId,
|
|
138
138
|
});
|
|
139
139
|
steps.push({
|
|
@@ -145,7 +145,7 @@ export async function assembleCredentialsSnapshot(opts) {
|
|
|
145
145
|
}
|
|
146
146
|
return { steps };
|
|
147
147
|
}
|
|
148
|
-
function isErrnoNotFound(cause) {
|
|
148
|
+
export function isErrnoNotFound(cause) {
|
|
149
149
|
if (cause === null || typeof cause !== "object")
|
|
150
150
|
return false;
|
|
151
151
|
const code = cause.code;
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
import { spawnSync } from "node:child_process";
|
|
22
22
|
import fs from "node:fs";
|
|
23
23
|
import path from "node:path";
|
|
24
|
-
import { countLooseObjects, gitBytes } from "@intx/storage-isogit";
|
|
24
|
+
import { countLooseObjects, gitBytes } from "@intx/storage-isogit/node";
|
|
25
25
|
const RUNS_DIR = "runs";
|
|
26
26
|
const ADDRESSES_DIR = "addresses";
|
|
27
27
|
const CONSUMED_DIR = "consumed";
|
|
@@ -16,8 +16,8 @@ export type DrainTimeoutOpts = {
|
|
|
16
16
|
repoId: RepoId;
|
|
17
17
|
/** Workflow-run repo ref the supervisor commits events to. */
|
|
18
18
|
ref: string;
|
|
19
|
-
/**
|
|
20
|
-
|
|
19
|
+
/** Anchor run id baked into the supervisor's signing principal. */
|
|
20
|
+
anchorRunId: string;
|
|
21
21
|
/** Run id the drain is being escalated against. */
|
|
22
22
|
runId: string;
|
|
23
23
|
/**
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
export { createWorkflowSupervisor,
|
|
2
|
-
export { assembleCredentialsSnapshot, defaultStepRepoId, hashGrants, STEP_GRANTS_PATH, STEP_GRANTS_REF, type AssembleCredentialsSnapshotOpts, type CredentialsSnapshot, type CredentialsSnapshotStep, type DeriveStepAddress, type DeriveStepRepoId, } from "./credentials.js";
|
|
1
|
+
export { createWorkflowSupervisor, type CancelCommitInfo, type CancelRequestOpts, type DeliverCredentialsOpts, type DeliverSignalOpts, type DeliverSourcesOpts, type DrainOpts, type RecycleOpts, type SpawnOpts, type SpawnResult, type WorkflowSupervisor, } from "./supervisor.js";
|
|
2
|
+
export { assembleCredentialsSnapshot, isErrnoNotFound, defaultStepRepoId, hashGrants, STEP_GRANTS_PATH, STEP_GRANTS_REF, type AssembleCredentialsSnapshotOpts, type CredentialsSnapshot, type CredentialsSnapshotStep, type DeriveStepAddress, type DeriveStepRepoId, } from "./credentials.js";
|
|
3
3
|
export { commitCancelRequested, SUPERVISOR_PRINCIPAL_KIND, type CommitCancelRequestedOpts, type CommitCancelRequestedResult, } from "./cancel-signing.js";
|
|
4
4
|
export { createDrainTimeoutAccumulator, DEFAULT_DRAIN_TIMEOUT_MS, type DrainTimeoutAccumulator, type DrainTimeoutAccumulatorFactory, type DrainTimeoutOpts, } from "./drain-timeout.js";
|
|
5
5
|
export { DEFAULT_KILL_TIMEOUT_MS } from "./child-termination.js";
|
|
6
6
|
export { createRecyclePolicy, triggerRecycle, DEFAULT_POLICY_INTERVAL_MS, MAX_BUFFERED_MAIL, type ChildWiring, type RecycleAttempt, type RecycleContext, type RecycleOrigin, type RecyclePolicy, type RecyclePolicyBounds, type RecyclePolicyOpts, type TriggerRecycleOpts, } from "./recycle.js";
|
|
7
|
-
export type { DeriveMailAuditRef, DispatchStructuralCounters, DispatchSubstrateLeg, DispatchTimingMark, InboxPrimitives, MailAuditRef, MailBusBindings, PrincipalSigner, SignedPayload, SubprocessHandle, SubprocessSpawner, TerminalEventSource, TerminalRunEvent, WorkflowSupervisorBindings, WorkflowSupervisorPrincipalKind, } from "./types.js";
|
|
7
|
+
export type { DeriveMailAuditRef, DispatchStructuralCounters, DispatchSubstrateLeg, DispatchTimingMark, InboxPrimitives, MailAuditRef, MailBusBindings, PrincipalSigner, SignedPayload, SubprocessHandle, SubprocessSpawner, SuspensionRegistration, TerminalEventSource, TerminalRunEvent, WorkflowSupervisorBindings, WorkflowSupervisorPrincipalKind, } from "./types.js";
|
package/dist/supervisor/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
export { createWorkflowSupervisor,
|
|
2
|
-
export { assembleCredentialsSnapshot, defaultStepRepoId, hashGrants, STEP_GRANTS_PATH, STEP_GRANTS_REF, } from "./credentials.js";
|
|
1
|
+
export { createWorkflowSupervisor, } from "./supervisor.js";
|
|
2
|
+
export { assembleCredentialsSnapshot, isErrnoNotFound, defaultStepRepoId, hashGrants, STEP_GRANTS_PATH, STEP_GRANTS_REF, } from "./credentials.js";
|
|
3
3
|
export { commitCancelRequested, SUPERVISOR_PRINCIPAL_KIND, } from "./cancel-signing.js";
|
|
4
4
|
export { createDrainTimeoutAccumulator, DEFAULT_DRAIN_TIMEOUT_MS, } from "./drain-timeout.js";
|
|
5
5
|
export { DEFAULT_KILL_TIMEOUT_MS } from "./child-termination.js";
|
|
@@ -18,9 +18,12 @@ export declare const DEFAULT_POLICY_INTERVAL_MS = 60000;
|
|
|
18
18
|
/**
|
|
19
19
|
* Origin tag the recycle path stamps onto its log messages so an
|
|
20
20
|
* operator scanning logs can distinguish operator-initiated,
|
|
21
|
-
* policy-initiated,
|
|
21
|
+
* policy-initiated, self-initiated, and crash-respawn origins at a
|
|
22
|
+
* glance. The `crash` origin drives the same respawn sequence after an
|
|
23
|
+
* unexpected child exit; see the six-step header note about steps 1-2
|
|
24
|
+
* degrading to no-ops for it.
|
|
22
25
|
*/
|
|
23
|
-
export type RecycleOrigin = "operator" | "policy" | "self";
|
|
26
|
+
export type RecycleOrigin = "operator" | "policy" | "self" | "crash";
|
|
24
27
|
export interface RecycleAttempt {
|
|
25
28
|
/** Origin the recycle was initiated from. */
|
|
26
29
|
readonly origin: RecycleOrigin;
|
|
@@ -53,7 +56,11 @@ export interface ChildWiring {
|
|
|
53
56
|
export interface RecycleContext {
|
|
54
57
|
/** The supervisor's full bindings, reused on respawn for credentials and spawn. */
|
|
55
58
|
readonly bindings: WorkflowSupervisorBindings;
|
|
56
|
-
/**
|
|
59
|
+
/**
|
|
60
|
+
* Every step id in this deployment's flat step-id namespace -- the
|
|
61
|
+
* definition's own `stepOrder` plus the step ids of every `loop` body it
|
|
62
|
+
* carries -- for credentials re-assembly.
|
|
63
|
+
*/
|
|
57
64
|
readonly stepOrder: readonly string[];
|
|
58
65
|
/** Definition hash carried on respawn env (unchanged across recycle). */
|
|
59
66
|
readonly definitionHash: string;
|