@intx/workflow-host 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -10
- package/dist/adapters/repo-store.d.ts +22 -1
- package/dist/adapters/repo-store.js +53 -53
- package/dist/adapters/spawn-child.d.ts +71 -42
- package/dist/adapters/spawn-child.js +83 -77
- package/dist/adapters/step-invoker.js +84 -7
- package/dist/child/env-bootstrap.d.ts +20 -6
- package/dist/child/env-bootstrap.js +9 -1
- package/dist/child/index.d.ts +2 -1
- package/dist/child/parked-correlations.d.ts +42 -0
- package/dist/child/parked-correlations.js +80 -0
- package/dist/child/proxy-repo-store.d.ts +3 -2
- package/dist/child/proxy-repo-store.js +2 -0
- package/dist/child/run-child.d.ts +107 -13
- package/dist/child/run-child.js +290 -108
- package/dist/child/self-discovery.d.ts +10 -0
- package/dist/child/self-discovery.js +25 -1
- package/dist/child/verified-definition-loader.d.ts +33 -0
- package/dist/child/verified-definition-loader.js +43 -0
- package/dist/conversation-text.d.ts +23 -0
- package/dist/conversation-text.js +56 -0
- package/dist/index.d.ts +4 -3
- package/dist/index.js +3 -2
- package/dist/ipc/control-channel.d.ts +58 -0
- package/dist/ipc/control-channel.js +94 -1
- package/dist/ipc/event-channel.d.ts +32 -1
- package/dist/mail-bus/hub-transport-adapter.d.ts +12 -7
- package/dist/mail-bus/hub-transport-adapter.js +9 -5
- package/dist/seams/scheduler.d.ts +4 -6
- package/dist/seams/scheduler.js +74 -93
- package/dist/supervisor/cancel-signing.d.ts +2 -2
- package/dist/supervisor/cancel-signing.js +1 -1
- package/dist/supervisor/credentials.d.ts +11 -10
- package/dist/supervisor/credentials.js +7 -7
- package/dist/supervisor/dispatch-attribution.js +1 -1
- package/dist/supervisor/drain-timeout.d.ts +2 -2
- package/dist/supervisor/drain-timeout.js +1 -1
- package/dist/supervisor/index.d.ts +3 -3
- package/dist/supervisor/index.js +2 -2
- package/dist/supervisor/recycle.d.ts +5 -2
- package/dist/supervisor/recycle.js +18 -7
- package/dist/supervisor/run-event-compaction.d.ts +5 -5
- package/dist/supervisor/run-event-compaction.js +5 -5
- package/dist/supervisor/spawn-env.d.ts +2 -2
- package/dist/supervisor/spawn-env.js +1 -1
- package/dist/supervisor/supervisor.d.ts +82 -25
- package/dist/supervisor/supervisor.js +1313 -410
- package/dist/supervisor/terminal-commit.d.ts +36 -0
- package/dist/supervisor/terminal-commit.js +134 -0
- package/dist/supervisor/types.d.ts +150 -23
- package/dist/workflow-definition-loader.d.ts +131 -0
- package/dist/workflow-definition-loader.js +316 -0
- package/package.json +12 -11
package/dist/seams/scheduler.js
CHANGED
|
@@ -26,17 +26,19 @@
|
|
|
26
26
|
//
|
|
27
27
|
// The scheduler reads workflow-event blobs at the canonical layout
|
|
28
28
|
// `runs/<runId>/events/<seq>.json` and writes a fresh blob at the
|
|
29
|
-
// next-seq slot for each `TimerFired` commit. The blob envelope
|
|
30
|
-
//
|
|
31
|
-
//
|
|
32
|
-
// `
|
|
33
|
-
//
|
|
34
|
-
//
|
|
35
|
-
//
|
|
36
|
-
// `
|
|
37
|
-
//
|
|
29
|
+
// next-seq slot for each `TimerFired` commit. The blob envelope is
|
|
30
|
+
// FLAT -- `{ seq, type, ...eventFields }` -- matching the runtime
|
|
31
|
+
// repo-store's `workflowEventToOnDisk` and the workflow-run kind
|
|
32
|
+
// handler's `EventEnvelope` contract: `seq` is the integer that also
|
|
33
|
+
// appears in the filename, `type` is the `subscribeKind` discriminator
|
|
34
|
+
// the scheduler narrows on, and the timer fields (`timerId`, `fireAt`,
|
|
35
|
+
// ...) sit alongside them, NOT under a nested `data` object. The kind
|
|
36
|
+
// handler's `validatePush` enforces that every event blob's body `seq`
|
|
37
|
+
// matches the filename's seq, so the scheduler mints the next seq
|
|
38
|
+
// inside the `writeTreePreservingPrefix` merge step and writes both
|
|
39
|
+
// into the envelope and the filename.
|
|
38
40
|
import { type } from "arktype";
|
|
39
|
-
import { subscribeKind } from "@intx/hub-sessions/substrate";
|
|
41
|
+
import { requireEventSeq, subscribeKind, WORKFLOW_RUN_EVENTS_DIR, WORKFLOW_RUN_RUNS_PREFIX, } from "@intx/hub-sessions/substrate";
|
|
40
42
|
/**
|
|
41
43
|
* Substrate-shape envelope for the workflow-event blob committed to
|
|
42
44
|
* `runs/<runId>/events/<seq>.json`. The validator covers the two
|
|
@@ -49,12 +51,16 @@ import { subscribeKind } from "@intx/hub-sessions/substrate";
|
|
|
49
51
|
export const TimerEventEnvelope = type({
|
|
50
52
|
seq: "number >= 0",
|
|
51
53
|
type: "'TimerSet' | 'TimerFired'",
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
54
|
+
// FLAT on-disk shape, matching the runtime repo-store's `workflowEventToOnDisk`
|
|
55
|
+
// ({ seq, type, ...eventFields }) and the substrate's enforced `EventEnvelope`
|
|
56
|
+
// contract -- NOT a nested `data` object. The scheduler was the lone component
|
|
57
|
+
// writing/reading a nested `data` envelope; nothing else parsed it, so a
|
|
58
|
+
// deployed timer never round-tripped until this was aligned.
|
|
59
|
+
timerId: "string",
|
|
60
|
+
"fireAt?": "string",
|
|
61
|
+
"stepId?": "string | null",
|
|
62
|
+
"cron?": "string | null",
|
|
63
|
+
"+": "ignore",
|
|
58
64
|
});
|
|
59
65
|
export function createWorkflowHostScheduler(opts) {
|
|
60
66
|
const queues = new Map();
|
|
@@ -106,21 +112,21 @@ export function createWorkflowHostScheduler(opts) {
|
|
|
106
112
|
break;
|
|
107
113
|
if (entry.event.type !== "TimerSet")
|
|
108
114
|
continue;
|
|
109
|
-
const fireAt = entry.event.
|
|
115
|
+
const fireAt = entry.event.fireAt;
|
|
110
116
|
if (fireAt === undefined) {
|
|
111
|
-
throw new Error(`scheduler live ingest: TimerSet in ${String(repoId.id)} run ${entry.runId} timer ${entry.event.
|
|
117
|
+
throw new Error(`scheduler live ingest: TimerSet in ${String(repoId.id)} run ${entry.runId} timer ${entry.event.timerId} missing fireAt`);
|
|
112
118
|
}
|
|
113
119
|
const fireAtMs = Date.parse(fireAt);
|
|
114
120
|
if (Number.isNaN(fireAtMs)) {
|
|
115
|
-
throw new Error(`scheduler live ingest: TimerSet in ${String(repoId.id)} run ${entry.runId} timer ${entry.event.
|
|
121
|
+
throw new Error(`scheduler live ingest: TimerSet in ${String(repoId.id)} run ${entry.runId} timer ${entry.event.timerId} fireAt unparseable: ${fireAt}`);
|
|
116
122
|
}
|
|
117
|
-
const cron = entry.event.
|
|
123
|
+
const cron = entry.event.cron !== undefined && entry.event.cron !== null;
|
|
118
124
|
if (cron && fireAtMs < opts.clock().getTime()) {
|
|
119
125
|
// Same missed-cron-tick spec as recovery: a cron TimerSet
|
|
120
126
|
// whose fireAt is in the past on arrival is dropped.
|
|
121
127
|
continue;
|
|
122
128
|
}
|
|
123
|
-
enqueue(repoId, entry.runId, entry.event.
|
|
129
|
+
enqueue(repoId, entry.runId, entry.event.timerId, fireAtMs, cron);
|
|
124
130
|
}
|
|
125
131
|
})();
|
|
126
132
|
liveSubscriptions.push({ abort, done });
|
|
@@ -133,24 +139,24 @@ export function createWorkflowHostScheduler(opts) {
|
|
|
133
139
|
const unfired = new Map();
|
|
134
140
|
for (const e of events) {
|
|
135
141
|
if (e.envelope.type === "TimerSet") {
|
|
136
|
-
const fireAt = e.envelope.
|
|
142
|
+
const fireAt = e.envelope.fireAt;
|
|
137
143
|
if (fireAt === undefined) {
|
|
138
|
-
throw new Error(`scheduler recovery: TimerSet in ${String(repoId.id)} run ${e.runId} timer ${e.envelope.
|
|
144
|
+
throw new Error(`scheduler recovery: TimerSet in ${String(repoId.id)} run ${e.runId} timer ${e.envelope.timerId} missing fireAt`);
|
|
139
145
|
}
|
|
140
146
|
const fireAtMs = Date.parse(fireAt);
|
|
141
147
|
if (Number.isNaN(fireAtMs)) {
|
|
142
|
-
throw new Error(`scheduler recovery: TimerSet in ${String(repoId.id)} run ${e.runId} timer ${e.envelope.
|
|
148
|
+
throw new Error(`scheduler recovery: TimerSet in ${String(repoId.id)} run ${e.runId} timer ${e.envelope.timerId} fireAt unparseable: ${fireAt}`);
|
|
143
149
|
}
|
|
144
|
-
const cron = e.envelope.
|
|
145
|
-
unfired.set(`${e.runId} ${e.envelope.
|
|
150
|
+
const cron = e.envelope.cron !== undefined && e.envelope.cron !== null;
|
|
151
|
+
unfired.set(`${e.runId} ${e.envelope.timerId}`, {
|
|
146
152
|
runId: e.runId,
|
|
147
|
-
timerId: e.envelope.
|
|
153
|
+
timerId: e.envelope.timerId,
|
|
148
154
|
fireAtMs,
|
|
149
155
|
cron,
|
|
150
156
|
});
|
|
151
157
|
}
|
|
152
158
|
else {
|
|
153
|
-
unfired.delete(`${e.runId} ${e.envelope.
|
|
159
|
+
unfired.delete(`${e.runId} ${e.envelope.timerId}`);
|
|
154
160
|
}
|
|
155
161
|
}
|
|
156
162
|
const now = opts.clock().getTime();
|
|
@@ -224,14 +230,15 @@ export function createWorkflowHostScheduler(opts) {
|
|
|
224
230
|
}
|
|
225
231
|
/**
|
|
226
232
|
* Read every timer-event blob across every run under the given
|
|
227
|
-
* workflow-run repo.
|
|
228
|
-
*
|
|
229
|
-
*
|
|
230
|
-
*
|
|
231
|
-
*
|
|
232
|
-
*
|
|
233
|
-
*
|
|
234
|
-
* current ref tip
|
|
233
|
+
* workflow-run repo. Recovery reconstructs its ledger from committed
|
|
234
|
+
* state, so `enumerateEventBlobs` reads the git object store at the
|
|
235
|
+
* events ref tip via `openCommittedReads` rather than the materialized
|
|
236
|
+
* working tree, whose non-atomic post-commit checkout can lag the
|
|
237
|
+
* committed tree on a contended filesystem and hide an already-committed
|
|
238
|
+
* `TimerFired`. `subscribeKind` is a diff-shaped iterator over new
|
|
239
|
+
* commits, not a "list everything at HEAD" primitive, so the startup
|
|
240
|
+
* ledger needs this path-aware enumeration of the current ref tip rather
|
|
241
|
+
* than a tail subscription.
|
|
235
242
|
*/
|
|
236
243
|
async function readAllEvents(opts, repoId) {
|
|
237
244
|
const entries = [];
|
|
@@ -261,53 +268,36 @@ async function readAllEvents(opts, repoId) {
|
|
|
261
268
|
* has none.
|
|
262
269
|
*/
|
|
263
270
|
async function enumerateEventBlobs(opts, repoId) {
|
|
264
|
-
const
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
const runsDir = path.join(dir, "runs");
|
|
271
|
+
const reads = await opts.repoStore.openCommittedReads(opts.principal, repoId, opts.ref);
|
|
272
|
+
if (reads === null)
|
|
273
|
+
return [];
|
|
268
274
|
const out = [];
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
throw cause;
|
|
277
|
-
}
|
|
278
|
-
for (const runId of runEntries) {
|
|
279
|
-
const eventsDir = path.join(runsDir, runId, "events");
|
|
280
|
-
let blobs;
|
|
281
|
-
try {
|
|
282
|
-
blobs = await fs.readdir(eventsDir);
|
|
283
|
-
}
|
|
284
|
-
catch (cause) {
|
|
285
|
-
if (isErrnoNotFound(cause))
|
|
286
|
-
continue;
|
|
287
|
-
throw cause;
|
|
288
|
-
}
|
|
275
|
+
const runEntries = await reads.listDir(WORKFLOW_RUN_RUNS_PREFIX);
|
|
276
|
+
for (const runEntry of runEntries) {
|
|
277
|
+
if (runEntry.type !== "tree")
|
|
278
|
+
continue;
|
|
279
|
+
const runId = runEntry.name;
|
|
280
|
+
const eventsDir = `${WORKFLOW_RUN_RUNS_PREFIX}/${runId}/${WORKFLOW_RUN_EVENTS_DIR}`;
|
|
281
|
+
const blobs = await reads.listDir(eventsDir);
|
|
289
282
|
for (const blob of blobs) {
|
|
290
|
-
if (
|
|
283
|
+
if (blob.type !== "blob")
|
|
291
284
|
continue;
|
|
292
|
-
|
|
285
|
+
// Assert a legal <seq>.json name; an illegal one under the events
|
|
286
|
+
// prefix is corruption the recovery walk must not silently drop.
|
|
287
|
+
requireEventSeq(blob.name, `${eventsDir}/${blob.name}`);
|
|
288
|
+
const raw = await reads.readBlobByOid(blob.oid);
|
|
293
289
|
let parsed;
|
|
294
290
|
try {
|
|
295
|
-
parsed = JSON.parse(raw);
|
|
291
|
+
parsed = JSON.parse(new TextDecoder().decode(raw));
|
|
296
292
|
}
|
|
297
293
|
catch (cause) {
|
|
298
|
-
throw new Error(`scheduler recovery: cannot parse ${String(repoId.id)}/${runId}
|
|
294
|
+
throw new Error(`scheduler recovery: cannot parse ${String(repoId.id)}/${runId}/${WORKFLOW_RUN_EVENTS_DIR}/${blob.name}: ${String(cause)}`);
|
|
299
295
|
}
|
|
300
296
|
out.push({ runId, payload: parsed });
|
|
301
297
|
}
|
|
302
298
|
}
|
|
303
299
|
return out;
|
|
304
300
|
}
|
|
305
|
-
function isErrnoNotFound(cause) {
|
|
306
|
-
if (cause === null || typeof cause !== "object")
|
|
307
|
-
return false;
|
|
308
|
-
const code = cause.code;
|
|
309
|
-
return code === "ENOENT";
|
|
310
|
-
}
|
|
311
301
|
/**
|
|
312
302
|
* Commit a `TimerFired` event blob to the workflow-run repo. The
|
|
313
303
|
* commit goes through `writeTreePreservingPrefix` so concurrent
|
|
@@ -321,7 +311,7 @@ async function commitTimerFired(opts, runId, timerId) {
|
|
|
321
311
|
if (owningRepoId === undefined) {
|
|
322
312
|
throw new Error(`scheduler commit: cannot find deployment owning run ${runId}`);
|
|
323
313
|
}
|
|
324
|
-
const prefix =
|
|
314
|
+
const prefix = `${WORKFLOW_RUN_RUNS_PREFIX}/${runId}/${WORKFLOW_RUN_EVENTS_DIR}/`;
|
|
325
315
|
await opts.repoStore.writeTreePreservingPrefix(opts.principal, owningRepoId, opts.ref, {
|
|
326
316
|
preservePrefix: prefix,
|
|
327
317
|
merge: async (existing) => {
|
|
@@ -329,13 +319,7 @@ async function commitTimerFired(opts, runId, timerId) {
|
|
|
329
319
|
let alreadyFired = false;
|
|
330
320
|
for (const [filepath, contents] of existing) {
|
|
331
321
|
const name = filepath.slice(prefix.length);
|
|
332
|
-
const
|
|
333
|
-
if (match === null)
|
|
334
|
-
continue;
|
|
335
|
-
const seqStr = match[1];
|
|
336
|
-
if (seqStr === undefined)
|
|
337
|
-
continue;
|
|
338
|
-
const seq = Number.parseInt(seqStr, 10);
|
|
322
|
+
const seq = requireEventSeq(name, `${prefix}${name}`);
|
|
339
323
|
if (seq > maxSeq)
|
|
340
324
|
maxSeq = seq;
|
|
341
325
|
try {
|
|
@@ -360,7 +344,7 @@ async function commitTimerFired(opts, runId, timerId) {
|
|
|
360
344
|
out[`${prefix}${String(nextSeq)}.json`] = JSON.stringify({
|
|
361
345
|
seq: nextSeq,
|
|
362
346
|
type: "TimerFired",
|
|
363
|
-
|
|
347
|
+
timerId,
|
|
364
348
|
});
|
|
365
349
|
return out;
|
|
366
350
|
},
|
|
@@ -373,25 +357,22 @@ function isMatchingTimerFired(parsed, timerId) {
|
|
|
373
357
|
const obj = parsed;
|
|
374
358
|
if (obj.type !== "TimerFired")
|
|
375
359
|
return false;
|
|
376
|
-
|
|
377
|
-
return false;
|
|
378
|
-
return obj.data.timerId === timerId;
|
|
360
|
+
return obj.timerId === timerId;
|
|
379
361
|
}
|
|
380
362
|
async function findOwningDeployment(opts, runId) {
|
|
381
363
|
const repoIds = await opts.listActiveDeployments();
|
|
382
|
-
const fs = await import("node:fs/promises");
|
|
383
|
-
const path = await import("node:path");
|
|
384
364
|
for (const repoId of repoIds) {
|
|
385
|
-
const
|
|
386
|
-
|
|
387
|
-
|
|
365
|
+
const reads = await opts.repoStore.openCommittedReads(opts.principal, repoId, opts.ref);
|
|
366
|
+
if (reads === null)
|
|
367
|
+
continue;
|
|
368
|
+
// The committed events subtree exists only when it holds at least
|
|
369
|
+
// one blob (git does not track empty trees), so a non-empty listing
|
|
370
|
+
// is exactly "this deployment owns the run". Reading committed state
|
|
371
|
+
// rather than the working tree keeps attribution correct when the
|
|
372
|
+
// checkout lags the object store.
|
|
373
|
+
const events = await reads.listDir(`${WORKFLOW_RUN_RUNS_PREFIX}/${runId}/${WORKFLOW_RUN_EVENTS_DIR}`);
|
|
374
|
+
if (events.length > 0)
|
|
388
375
|
return repoId;
|
|
389
|
-
}
|
|
390
|
-
catch (cause) {
|
|
391
|
-
if (isErrnoNotFound(cause))
|
|
392
|
-
continue;
|
|
393
|
-
throw cause;
|
|
394
|
-
}
|
|
395
376
|
}
|
|
396
377
|
return undefined;
|
|
397
378
|
}
|
|
@@ -16,8 +16,8 @@ export type CommitCancelRequestedOpts = {
|
|
|
16
16
|
repoId: RepoId;
|
|
17
17
|
/** Events ref the workflow-run repo writes to. */
|
|
18
18
|
ref: string;
|
|
19
|
-
/**
|
|
20
|
-
|
|
19
|
+
/** Anchor run id used to construct the supervisor principal. */
|
|
20
|
+
anchorRunId: string;
|
|
21
21
|
/** Run id whose event log receives the CancelRequested entry. */
|
|
22
22
|
runId: string;
|
|
23
23
|
/** Cancellation origin from the Q3 map. */
|
|
@@ -76,7 +76,7 @@ export async function commitCancelRequested(opts) {
|
|
|
76
76
|
const prefix = `${RUNS_PREFIX}/${opts.runId}/${EVENTS_DIR}/`;
|
|
77
77
|
const principal = {
|
|
78
78
|
kind: SUPERVISOR_PRINCIPAL_KIND,
|
|
79
|
-
|
|
79
|
+
anchorRunId: opts.anchorRunId,
|
|
80
80
|
};
|
|
81
81
|
let resolved = null;
|
|
82
82
|
const { commitSha } = await opts.substrate.writeTreePreservingPrefix(principal, opts.repoId, opts.ref, {
|
|
@@ -29,24 +29,24 @@ export type CredentialsSnapshot = {
|
|
|
29
29
|
};
|
|
30
30
|
/**
|
|
31
31
|
* Caller-supplied derivation of the per-step mail address from the
|
|
32
|
-
*
|
|
32
|
+
* run id and step id. The supervisor cannot encode the
|
|
33
33
|
* deployment-domain inside library code; the wiring module supplies
|
|
34
34
|
* the strategy the host owns.
|
|
35
35
|
*/
|
|
36
36
|
export type DeriveStepAddress = (args: {
|
|
37
|
-
|
|
37
|
+
runId: string;
|
|
38
38
|
stepId: string;
|
|
39
39
|
}) => string;
|
|
40
40
|
/**
|
|
41
41
|
* Caller-supplied override of the per-step `agent-state` repo identity
|
|
42
42
|
* the supervisor reads grants from. Defaults to the
|
|
43
|
-
* `<
|
|
43
|
+
* `<runId>-<stepId>` convention (`defaultStepRepoId`); the
|
|
44
44
|
* single-step launched-agent deploy supplies a derivation that returns
|
|
45
45
|
* the legacy agent-state repo so the child reads grants from the same
|
|
46
46
|
* repo the legacy agent identity already keys.
|
|
47
47
|
*/
|
|
48
48
|
export type DeriveStepRepoId = (args: {
|
|
49
|
-
|
|
49
|
+
runId: string;
|
|
50
50
|
stepId: string;
|
|
51
51
|
}) => RepoId;
|
|
52
52
|
export type AssembleCredentialsSnapshotOpts = {
|
|
@@ -60,27 +60,27 @@ export type AssembleCredentialsSnapshotOpts = {
|
|
|
60
60
|
* the order the workflow asset declared.
|
|
61
61
|
*/
|
|
62
62
|
stepOrder: readonly string[];
|
|
63
|
-
/**
|
|
64
|
-
|
|
63
|
+
/** Anchor run id used in agent-state repo identity and address derivation. */
|
|
64
|
+
anchorRunId: string;
|
|
65
65
|
/** Per-step mail-address derivation callback. */
|
|
66
66
|
deriveStepAddress: DeriveStepAddress;
|
|
67
67
|
/**
|
|
68
68
|
* Optional override for the `agent-state` repo's id. Callers that
|
|
69
|
-
* follow the documented convention (`<
|
|
69
|
+
* follow the documented convention (`<runId>-<stepId>`) can
|
|
70
70
|
* omit this; tests and bespoke layouts can supply their own.
|
|
71
71
|
*/
|
|
72
72
|
deriveStepRepoId?: DeriveStepRepoId;
|
|
73
73
|
};
|
|
74
74
|
/**
|
|
75
|
-
* Default mapping from `(
|
|
76
|
-
* repo id: `<
|
|
75
|
+
* Default mapping from `(runId, stepId)` to the agent-state
|
|
76
|
+
* repo id: `<runId>-<stepId>`, isolating each step's grants in
|
|
77
77
|
* its own repo. Applied to a one-step `stepOrder` it yields a single
|
|
78
78
|
* such repo. The single-step launched-agent deploy overrides this
|
|
79
79
|
* default (see `DeriveStepRepoId`) to reuse the legacy agent-state
|
|
80
80
|
* repo.
|
|
81
81
|
*/
|
|
82
82
|
export declare function defaultStepRepoId(args: {
|
|
83
|
-
|
|
83
|
+
runId: string;
|
|
84
84
|
stepId: string;
|
|
85
85
|
}): RepoId;
|
|
86
86
|
/**
|
|
@@ -99,3 +99,4 @@ export declare function hashGrants(grants: readonly unknown[]): Promise<string>;
|
|
|
99
99
|
* arrives after a fresher one.
|
|
100
100
|
*/
|
|
101
101
|
export declare function assembleCredentialsSnapshot(opts: AssembleCredentialsSnapshotOpts): Promise<CredentialsSnapshot>;
|
|
102
|
+
export declare function isErrnoNotFound(cause: unknown): boolean;
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
// substrate's git layout.
|
|
21
21
|
//
|
|
22
22
|
// Per-step address derivation (Q6.4 discovery decision):
|
|
23
|
-
// - Multi-step deployments use `<
|
|
23
|
+
// - Multi-step deployments use `<runId>-<stepId>@<domain>`.
|
|
24
24
|
// - Trivial (single-step) deployments use the deployment's own
|
|
25
25
|
// mail address as the sole step's address.
|
|
26
26
|
// The derivation is supplied by the caller as a `deriveStepAddress`
|
|
@@ -51,8 +51,8 @@ const StepGrantsFile = type({
|
|
|
51
51
|
grants: "unknown[]",
|
|
52
52
|
}).onUndeclaredKey("ignore");
|
|
53
53
|
/**
|
|
54
|
-
* Default mapping from `(
|
|
55
|
-
* repo id: `<
|
|
54
|
+
* Default mapping from `(runId, stepId)` to the agent-state
|
|
55
|
+
* repo id: `<runId>-<stepId>`, isolating each step's grants in
|
|
56
56
|
* its own repo. Applied to a one-step `stepOrder` it yields a single
|
|
57
57
|
* such repo. The single-step launched-agent deploy overrides this
|
|
58
58
|
* default (see `DeriveStepRepoId`) to reuse the legacy agent-state
|
|
@@ -61,7 +61,7 @@ const StepGrantsFile = type({
|
|
|
61
61
|
export function defaultStepRepoId(args) {
|
|
62
62
|
return {
|
|
63
63
|
kind: "agent-state",
|
|
64
|
-
id: `${args.
|
|
64
|
+
id: `${args.runId}-${args.stepId}`,
|
|
65
65
|
};
|
|
66
66
|
}
|
|
67
67
|
/**
|
|
@@ -128,12 +128,12 @@ export async function assembleCredentialsSnapshot(opts) {
|
|
|
128
128
|
const steps = [];
|
|
129
129
|
for (const stepId of opts.stepOrder) {
|
|
130
130
|
const repoId = deriveRepoId({
|
|
131
|
-
|
|
131
|
+
runId: opts.anchorRunId,
|
|
132
132
|
stepId,
|
|
133
133
|
});
|
|
134
134
|
const grants = await readStepGrants(opts, repoId);
|
|
135
135
|
const address = opts.deriveStepAddress({
|
|
136
|
-
|
|
136
|
+
runId: opts.anchorRunId,
|
|
137
137
|
stepId,
|
|
138
138
|
});
|
|
139
139
|
steps.push({
|
|
@@ -145,7 +145,7 @@ export async function assembleCredentialsSnapshot(opts) {
|
|
|
145
145
|
}
|
|
146
146
|
return { steps };
|
|
147
147
|
}
|
|
148
|
-
function isErrnoNotFound(cause) {
|
|
148
|
+
export function isErrnoNotFound(cause) {
|
|
149
149
|
if (cause === null || typeof cause !== "object")
|
|
150
150
|
return false;
|
|
151
151
|
const code = cause.code;
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
import { spawnSync } from "node:child_process";
|
|
22
22
|
import fs from "node:fs";
|
|
23
23
|
import path from "node:path";
|
|
24
|
-
import { countLooseObjects, gitBytes } from "@intx/storage-isogit";
|
|
24
|
+
import { countLooseObjects, gitBytes } from "@intx/storage-isogit/node";
|
|
25
25
|
const RUNS_DIR = "runs";
|
|
26
26
|
const ADDRESSES_DIR = "addresses";
|
|
27
27
|
const CONSUMED_DIR = "consumed";
|
|
@@ -16,8 +16,8 @@ export type DrainTimeoutOpts = {
|
|
|
16
16
|
repoId: RepoId;
|
|
17
17
|
/** Workflow-run repo ref the supervisor commits events to. */
|
|
18
18
|
ref: string;
|
|
19
|
-
/**
|
|
20
|
-
|
|
19
|
+
/** Anchor run id baked into the supervisor's signing principal. */
|
|
20
|
+
anchorRunId: string;
|
|
21
21
|
/** Run id the drain is being escalated against. */
|
|
22
22
|
runId: string;
|
|
23
23
|
/**
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
export { createWorkflowSupervisor,
|
|
2
|
-
export { assembleCredentialsSnapshot, defaultStepRepoId, hashGrants, STEP_GRANTS_PATH, STEP_GRANTS_REF, type AssembleCredentialsSnapshotOpts, type CredentialsSnapshot, type CredentialsSnapshotStep, type DeriveStepAddress, type DeriveStepRepoId, } from "./credentials.js";
|
|
1
|
+
export { createWorkflowSupervisor, type CancelCommitInfo, type CancelRequestOpts, type DeliverCredentialsOpts, type DeliverSignalOpts, type DeliverSourcesOpts, type DrainOpts, type RecycleOpts, type SpawnOpts, type SpawnResult, type WorkflowSupervisor, } from "./supervisor.js";
|
|
2
|
+
export { assembleCredentialsSnapshot, isErrnoNotFound, defaultStepRepoId, hashGrants, STEP_GRANTS_PATH, STEP_GRANTS_REF, type AssembleCredentialsSnapshotOpts, type CredentialsSnapshot, type CredentialsSnapshotStep, type DeriveStepAddress, type DeriveStepRepoId, } from "./credentials.js";
|
|
3
3
|
export { commitCancelRequested, SUPERVISOR_PRINCIPAL_KIND, type CommitCancelRequestedOpts, type CommitCancelRequestedResult, } from "./cancel-signing.js";
|
|
4
4
|
export { createDrainTimeoutAccumulator, DEFAULT_DRAIN_TIMEOUT_MS, type DrainTimeoutAccumulator, type DrainTimeoutAccumulatorFactory, type DrainTimeoutOpts, } from "./drain-timeout.js";
|
|
5
5
|
export { DEFAULT_KILL_TIMEOUT_MS } from "./child-termination.js";
|
|
6
6
|
export { createRecyclePolicy, triggerRecycle, DEFAULT_POLICY_INTERVAL_MS, MAX_BUFFERED_MAIL, type ChildWiring, type RecycleAttempt, type RecycleContext, type RecycleOrigin, type RecyclePolicy, type RecyclePolicyBounds, type RecyclePolicyOpts, type TriggerRecycleOpts, } from "./recycle.js";
|
|
7
|
-
export type { DeriveMailAuditRef, DispatchStructuralCounters, DispatchSubstrateLeg, DispatchTimingMark, InboxPrimitives, MailAuditRef, MailBusBindings, PrincipalSigner, SignedPayload, SubprocessHandle, SubprocessSpawner, TerminalEventSource, TerminalRunEvent, WorkflowSupervisorBindings, WorkflowSupervisorPrincipalKind, } from "./types.js";
|
|
7
|
+
export type { DeriveMailAuditRef, DispatchStructuralCounters, DispatchSubstrateLeg, DispatchTimingMark, InboxPrimitives, MailAuditRef, MailBusBindings, PrincipalSigner, SignedPayload, SubprocessHandle, SubprocessSpawner, SuspensionRegistration, TerminalEventSource, TerminalRunEvent, WorkflowSupervisorBindings, WorkflowSupervisorPrincipalKind, } from "./types.js";
|
package/dist/supervisor/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
export { createWorkflowSupervisor,
|
|
2
|
-
export { assembleCredentialsSnapshot, defaultStepRepoId, hashGrants, STEP_GRANTS_PATH, STEP_GRANTS_REF, } from "./credentials.js";
|
|
1
|
+
export { createWorkflowSupervisor, } from "./supervisor.js";
|
|
2
|
+
export { assembleCredentialsSnapshot, isErrnoNotFound, defaultStepRepoId, hashGrants, STEP_GRANTS_PATH, STEP_GRANTS_REF, } from "./credentials.js";
|
|
3
3
|
export { commitCancelRequested, SUPERVISOR_PRINCIPAL_KIND, } from "./cancel-signing.js";
|
|
4
4
|
export { createDrainTimeoutAccumulator, DEFAULT_DRAIN_TIMEOUT_MS, } from "./drain-timeout.js";
|
|
5
5
|
export { DEFAULT_KILL_TIMEOUT_MS } from "./child-termination.js";
|
|
@@ -18,9 +18,12 @@ export declare const DEFAULT_POLICY_INTERVAL_MS = 60000;
|
|
|
18
18
|
/**
|
|
19
19
|
* Origin tag the recycle path stamps onto its log messages so an
|
|
20
20
|
* operator scanning logs can distinguish operator-initiated,
|
|
21
|
-
* policy-initiated,
|
|
21
|
+
* policy-initiated, self-initiated, and crash-respawn origins at a
|
|
22
|
+
* glance. The `crash` origin drives the same respawn sequence after an
|
|
23
|
+
* unexpected child exit; see the six-step header note about steps 1-2
|
|
24
|
+
* degrading to no-ops for it.
|
|
22
25
|
*/
|
|
23
|
-
export type RecycleOrigin = "operator" | "policy" | "self";
|
|
26
|
+
export type RecycleOrigin = "operator" | "policy" | "self" | "crash";
|
|
24
27
|
export interface RecycleAttempt {
|
|
25
28
|
/** Origin the recycle was initiated from. */
|
|
26
29
|
readonly origin: RecycleOrigin;
|
|
@@ -4,8 +4,9 @@
|
|
|
4
4
|
//
|
|
5
5
|
// Recycle is the supervisor's "same deploy tree, fresh process" path.
|
|
6
6
|
// It tears the existing workflow-process child down and stands a new
|
|
7
|
-
// one up against the SAME deploy tree (same
|
|
8
|
-
// per-step credential repos). It is STRICTLY ORTHOGONAL
|
|
7
|
+
// one up against the SAME deploy tree (same materialized source
|
|
8
|
+
// closure, same per-step credential repos). It is STRICTLY ORTHOGONAL
|
|
9
|
+
// TO REDEPLOY:
|
|
9
10
|
//
|
|
10
11
|
// - Recycle = same deploy tree, fresh process.
|
|
11
12
|
// - Redeploy = new deploy tree.
|
|
@@ -20,6 +21,15 @@
|
|
|
20
21
|
//
|
|
21
22
|
// Six-step sequence (locked):
|
|
22
23
|
//
|
|
24
|
+
// The `drain` step and the `SubprocessHandle` handed in as `current` are
|
|
25
|
+
// caller-parameterized. For the operator/policy/self recycle origins the
|
|
26
|
+
// child is live: `drain` sends the real drain control mail and `kill`
|
|
27
|
+
// terminates a running process. For the `crash` origin the child has
|
|
28
|
+
// already exited unexpectedly, so the caller supplies a no-op `drain`
|
|
29
|
+
// (there is nothing to drain) and the `kill` in step 2 lands on an
|
|
30
|
+
// already-dead handle as a cheap no-op. Steps 3-6 are identical for every
|
|
31
|
+
// origin.
|
|
32
|
+
//
|
|
23
33
|
// 1. `drain` -- send the existing drain control mail. Wait for
|
|
24
34
|
// in-flight runs to drain per each step's `drainBehavior`.
|
|
25
35
|
// `drainTimeout` escalation applies normally; the drain-timeout
|
|
@@ -153,8 +163,8 @@ export async function triggerRecycle(ctx, opts) {
|
|
|
153
163
|
// Step 3: respawn. Fresh channelId, fresh HMAC key, fresh Ed25519
|
|
154
164
|
// IPC keypair. Per-step credentials are re-read so a grants update
|
|
155
165
|
// that landed since the original spawn is reflected in the new
|
|
156
|
-
// child's snapshot. The deploy tree (
|
|
157
|
-
// workflow-asset repo) is UNCHANGED.
|
|
166
|
+
// child's snapshot. The deploy tree (the materialized source closure,
|
|
167
|
+
// the workflow-asset repo, the agent-state repos) is UNCHANGED.
|
|
158
168
|
const channelId = generateChannelId();
|
|
159
169
|
const hmacKey = generateHmacKey();
|
|
160
170
|
const ipcKeypair = await (ctx.bindings.ipcKeyPairFactory ?? generateKeyPair)();
|
|
@@ -164,7 +174,7 @@ export async function triggerRecycle(ctx, opts) {
|
|
|
164
174
|
channelId,
|
|
165
175
|
hmacKey,
|
|
166
176
|
hostPublicKey: ipcKeypair.publicKey,
|
|
167
|
-
|
|
177
|
+
anchorRunId: ctx.bindings.anchorRunId,
|
|
168
178
|
deploymentMailAddress: ctx.bindings.deploymentMailAddress,
|
|
169
179
|
stepCount: ctx.bindings.stepCount,
|
|
170
180
|
definitionHash: ctx.definitionHash,
|
|
@@ -210,7 +220,8 @@ export async function triggerRecycle(ctx, opts) {
|
|
|
210
220
|
// the previous child's lifetime is picked up here -- the recycle
|
|
211
221
|
// doubles as the supervisor's grant-refresh path. The deploy tree
|
|
212
222
|
// is not consulted; this read is against the `agent-state` repos
|
|
213
|
-
// alone, whose contents are independent of
|
|
223
|
+
// alone, whose contents are independent of the materialized source
|
|
224
|
+
// closure.
|
|
214
225
|
//
|
|
215
226
|
// This is a substrate read that can reject -- a grants file that
|
|
216
227
|
// became malformed is precisely the recycle's grant-refresh path. The
|
|
@@ -227,7 +238,7 @@ export async function triggerRecycle(ctx, opts) {
|
|
|
227
238
|
repoStore: ctx.bindings.repoStore,
|
|
228
239
|
principal: ctx.bindings.readPrincipal,
|
|
229
240
|
stepOrder: ctx.stepOrder,
|
|
230
|
-
|
|
241
|
+
anchorRunId: ctx.bindings.anchorRunId,
|
|
231
242
|
deriveStepAddress: ctx.bindings.deriveStepAddress,
|
|
232
243
|
...(ctx.bindings.deriveStepRepoId !== undefined
|
|
233
244
|
? { deriveStepRepoId: ctx.bindings.deriveStepRepoId }
|
|
@@ -6,8 +6,8 @@ export type CompactRunEventsOpts = {
|
|
|
6
6
|
repoId: RepoId;
|
|
7
7
|
/** Events ref the workflow-run repo writes to. */
|
|
8
8
|
ref: string;
|
|
9
|
-
/**
|
|
10
|
-
|
|
9
|
+
/** Anchor run id used to construct the supervisor principal. */
|
|
10
|
+
anchorRunId: string;
|
|
11
11
|
/** Run to seal. */
|
|
12
12
|
runId: string;
|
|
13
13
|
};
|
|
@@ -19,9 +19,9 @@ export type CompactRunEventsOpts = {
|
|
|
19
19
|
*
|
|
20
20
|
* Idempotent and terminal-only: a run already sealed (no `events/` subtree)
|
|
21
21
|
* or one whose latest event is not terminal is left untouched, so the call
|
|
22
|
-
* is safe to repeat. The live caller
|
|
23
|
-
* run terminates; a bounded recovery sweep that would
|
|
24
|
-
*
|
|
22
|
+
* is safe to repeat. The live caller invokes it once per run, right after the
|
|
23
|
+
* run terminates; a bounded recovery sweep that would retry the fold for a run
|
|
24
|
+
* whose compaction a crash interrupted is not yet implemented.
|
|
25
25
|
*
|
|
26
26
|
* The combined file is the verbatim byte concatenation of the per-event
|
|
27
27
|
* blobs in seq order (`encodeCombinedEventLog`), the exact shape the
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
// and drops the per-event files. This shrinks the workflow-run repo's
|
|
6
6
|
// file count -- and every per-commit cost that scales with it --
|
|
7
7
|
// without losing any event. The fold writes under the substrate's
|
|
8
|
-
// per-repo lock as the `supervisor` principal, whose `
|
|
8
|
+
// per-repo lock as the `supervisor` principal, whose `anchorRunId`
|
|
9
9
|
// the workflow-run kind handler checks against `repoId.id`.
|
|
10
10
|
import { WORKFLOW_RUN_EVENTS_FILE, encodeCombinedEventLog, } from "@intx/hub-sessions/substrate";
|
|
11
11
|
import { SUPERVISOR_PRINCIPAL_KIND } from "./cancel-signing.js";
|
|
@@ -25,9 +25,9 @@ const TERMINAL_EVENT_TYPES = new Set([
|
|
|
25
25
|
*
|
|
26
26
|
* Idempotent and terminal-only: a run already sealed (no `events/` subtree)
|
|
27
27
|
* or one whose latest event is not terminal is left untouched, so the call
|
|
28
|
-
* is safe to repeat. The live caller
|
|
29
|
-
* run terminates; a bounded recovery sweep that would
|
|
30
|
-
*
|
|
28
|
+
* is safe to repeat. The live caller invokes it once per run, right after the
|
|
29
|
+
* run terminates; a bounded recovery sweep that would retry the fold for a run
|
|
30
|
+
* whose compaction a crash interrupted is not yet implemented.
|
|
31
31
|
*
|
|
32
32
|
* The combined file is the verbatim byte concatenation of the per-event
|
|
33
33
|
* blobs in seq order (`encodeCombinedEventLog`), the exact shape the
|
|
@@ -81,7 +81,7 @@ export async function compactRunEvents(opts) {
|
|
|
81
81
|
const combinedPath = `${RUNS_PREFIX}/${opts.runId}/${WORKFLOW_RUN_EVENTS_FILE}`;
|
|
82
82
|
const principal = {
|
|
83
83
|
kind: SUPERVISOR_PRINCIPAL_KIND,
|
|
84
|
-
|
|
84
|
+
anchorRunId: opts.anchorRunId,
|
|
85
85
|
};
|
|
86
86
|
let sealed = false;
|
|
87
87
|
await opts.substrate.writeTreePreservingPrefix(principal, opts.repoId, opts.ref, {
|
|
@@ -19,8 +19,8 @@ export interface ChildSpawnEnvParts {
|
|
|
19
19
|
hmacKey: Uint8Array;
|
|
20
20
|
/** Supervisor's Ed25519 public key for this spawn's control channel. */
|
|
21
21
|
hostPublicKey: Uint8Array;
|
|
22
|
-
/**
|
|
23
|
-
|
|
22
|
+
/** Anchor run id the supervisor manages. */
|
|
23
|
+
anchorRunId: string;
|
|
24
24
|
/** Mail address the deployment registered on the bus. */
|
|
25
25
|
deploymentMailAddress: string;
|
|
26
26
|
/** Step count of the deployed workflow (`stepOrder.length`). */
|
|
@@ -20,7 +20,7 @@ export function buildChildSpawnEnv(parts) {
|
|
|
20
20
|
IPC_CHANNEL_ID: parts.channelId,
|
|
21
21
|
IPC_HMAC_KEY: hexEncode(parts.hmacKey),
|
|
22
22
|
HOST_PUBKEY: hexEncode(parts.hostPublicKey),
|
|
23
|
-
DEPLOYMENT_ID: parts.
|
|
23
|
+
DEPLOYMENT_ID: parts.anchorRunId,
|
|
24
24
|
DEFINITION_HASH: parts.definitionHash,
|
|
25
25
|
MAILBOX_ADDRESS: parts.deploymentMailAddress,
|
|
26
26
|
STEP_COUNT: String(parts.stepCount),
|