@intx/workflow-host 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -4
- package/dist/adapters/mail-part-store.d.ts +46 -0
- package/dist/adapters/mail-part-store.js +251 -0
- package/dist/adapters/repo-store.js +5 -14
- package/dist/adapters/spawn-child.d.ts +42 -6
- package/dist/adapters/spawn-child.js +8 -18
- package/dist/adapters/step-invoker.d.ts +52 -2
- package/dist/adapters/step-invoker.js +230 -60
- package/dist/adapters/substrate-mailbox-store.d.ts +80 -0
- package/dist/adapters/substrate-mailbox-store.js +404 -0
- package/dist/child/child-mailbox-reader.d.ts +10 -0
- package/dist/child/child-mailbox-reader.js +23 -0
- package/dist/child/credential-cell.d.ts +8 -0
- package/dist/child/credential-cell.js +66 -0
- package/dist/child/from-process-env.d.ts +12 -0
- package/dist/child/from-process-env.js +6 -0
- package/dist/child/index.d.ts +4 -1
- package/dist/child/index.js +4 -1
- package/dist/child/mailbox-mutation-bridge.d.ts +61 -0
- package/dist/child/mailbox-mutation-bridge.js +101 -0
- package/dist/child/mailbox-watch-registry.d.ts +17 -0
- package/dist/child/mailbox-watch-registry.js +61 -0
- package/dist/child/outbound-mail-bridge.d.ts +3 -2
- package/dist/child/outbound-mail-bridge.js +20 -32
- package/dist/child/pending-request.d.ts +89 -0
- package/dist/child/pending-request.js +80 -0
- package/dist/child/run-child.d.ts +69 -7
- package/dist/child/run-child.js +307 -75
- package/dist/child/substrate-write-bridge.d.ts +3 -2
- package/dist/child/substrate-write-bridge.js +21 -38
- package/dist/child/supervisor-backed-transport.d.ts +52 -6
- package/dist/child/supervisor-backed-transport.js +205 -62
- package/dist/child/warm-agent-cache.d.ts +44 -4
- package/dist/child/warm-agent-cache.js +41 -10
- package/dist/index.d.ts +4 -3
- package/dist/index.js +4 -3
- package/dist/ipc/control-channel.d.ts +93 -2
- package/dist/ipc/control-channel.js +147 -47
- package/dist/ipc/index.d.ts +1 -1
- package/dist/ipc/index.js +1 -1
- package/dist/run-body-then-cleanup.d.ts +17 -0
- package/dist/run-body-then-cleanup.js +38 -0
- package/dist/seams/scheduler.d.ts +12 -0
- package/dist/seams/scheduler.js +13 -4
- package/dist/supervisor/cancel-signing.js +3 -7
- package/dist/supervisor/credentials.d.ts +17 -5
- package/dist/supervisor/recycle.d.ts +5 -1
- package/dist/supervisor/run-event-compaction.d.ts +2 -2
- package/dist/supervisor/run-event-compaction.js +11 -16
- package/dist/supervisor/run-event-recovery.d.ts +34 -0
- package/dist/supervisor/run-event-recovery.js +45 -0
- package/dist/supervisor/supervisor.d.ts +27 -4
- package/dist/supervisor/supervisor.js +644 -58
- package/dist/supervisor/terminal-commit.js +3 -7
- package/dist/supervisor/types.d.ts +30 -0
- package/dist/testing/change-notifier.d.ts +12 -0
- package/dist/testing/change-notifier.js +63 -0
- package/dist/testing/index.d.ts +8 -0
- package/dist/testing/index.js +16 -0
- package/dist/testing/log-capture.d.ts +52 -0
- package/dist/testing/log-capture.js +124 -0
- package/dist/testing/mail-bus.d.ts +22 -0
- package/dist/testing/mail-bus.js +78 -0
- package/dist/testing/memory-streams.d.ts +43 -0
- package/dist/testing/memory-streams.js +211 -0
- package/dist/testing/spawn-observer.d.ts +12 -0
- package/dist/testing/spawn-observer.js +36 -0
- package/dist/testing/stub-repo-store.d.ts +10 -0
- package/dist/testing/stub-repo-store.js +39 -0
- package/dist/testing/supervisor-reaper.d.ts +24 -0
- package/dist/testing/supervisor-reaper.js +49 -0
- package/dist/testing/upstream-frames.d.ts +47 -0
- package/dist/testing/upstream-frames.js +94 -0
- package/dist/workflow-definition-loader.d.ts +56 -0
- package/dist/workflow-definition-loader.js +106 -0
- package/package.json +17 -11
- package/dist/conversation-text.d.ts +0 -23
- package/dist/conversation-text.js +0 -56
|
@@ -0,0 +1,404 @@
|
|
|
1
|
+
// Workflow-run-substrate backing for the `@intx/mailbox` `MailboxStore`.
|
|
2
|
+
//
|
|
3
|
+
// The `MailboxStore` mutation surface is SYNCHRONOUS, but the workflow-run
|
|
4
|
+
// substrate is an async git store. This backing follows the shape of an IMAP
|
|
5
|
+
// client with a local cache: the async `createSubstrateMailboxStore` factory
|
|
6
|
+
// loads the committed `mailbox/INBOX/` METADATA (`index.json`) into an
|
|
7
|
+
// in-memory mirror on open, exposes the synchronous mutation surface over that
|
|
8
|
+
// mirror, and persists the current state back to the substrate through
|
|
9
|
+
// `flush`. Callers mutate synchronously (append / addFlags / removeFlags /
|
|
10
|
+
// remove) and `await flush()` at a boundary.
|
|
11
|
+
//
|
|
12
|
+
// The per-message raw RFC 2822 bytes are NOT loaded on open and are NOT held
|
|
13
|
+
// resident: `readRaw(uid)` reads a message's `<uid>.eml` blob on demand from
|
|
14
|
+
// the committed-read snapshot the open pinned. The only raw this backing keeps
|
|
15
|
+
// in memory is that of a message appended-but-not-yet-flushed; a successful
|
|
16
|
+
// `flush` drops it. So the resident footprint of a long-lived warm mailbox is
|
|
17
|
+
// bounded to metadata regardless of how much mail it has accumulated.
|
|
18
|
+
//
|
|
19
|
+
// On-disk layout, a top-level subtree of the workflow-run repo (one mailbox per
|
|
20
|
+
// deployment repo):
|
|
21
|
+
//
|
|
22
|
+
// mailbox/INBOX/index.json committed metadata: uidValidity, the
|
|
23
|
+
// uid/modseq counters, one entry per live
|
|
24
|
+
// message (uid, modseq, flags, pre-parsed
|
|
25
|
+
// envelope), and the expunged-uid tombstones
|
|
26
|
+
// QRESYNC answers `vanished` from.
|
|
27
|
+
// mailbox/INBOX/<uid>.eml the verbatim raw RFC 2822 bytes of each live
|
|
28
|
+
// message, so `fetchFull` verifies signatures
|
|
29
|
+
// byte-exactly. Write-once per uid.
|
|
30
|
+
//
|
|
31
|
+
// Reads resolve from the committed substrate (`openCommittedReads`), never the
|
|
32
|
+
// lagging working tree, matching the sibling `mail-part-store` reader. Writes
|
|
33
|
+
// go through `writeTreeDelta`: `index.json` changes on every mutation and is
|
|
34
|
+
// always put; each `<uid>.eml` is immutable, so a flush puts only the blobs
|
|
35
|
+
// appended since the last successful flush and deletes only those whose
|
|
36
|
+
// message was removed since then, and the substrate carries every untouched
|
|
37
|
+
// `.eml` forward by object id. This keeps a flush O(delta) rather than
|
|
38
|
+
// O(mailbox), so a long-lived warm conversational mailbox does not re-hash its
|
|
39
|
+
// whole history on each append or flag change. The committed subtree the delta
|
|
40
|
+
// leaves behind is byte-identical in shape to a full rewrite of the same live
|
|
41
|
+
// set. The kind handler's push-time validation of this subtree is owned
|
|
42
|
+
// separately by the hub replication layer; this module owns the on-disk shape
|
|
43
|
+
// it validates.
|
|
44
|
+
import { type } from "arktype";
|
|
45
|
+
/** Top-level subtree of the workflow-run repo that holds the mailbox. */
|
|
46
|
+
export const MAILBOX_PREFIX = "mailbox";
|
|
47
|
+
/** The single mailbox this backing persists, an IMAP INBOX. */
|
|
48
|
+
export const MAILBOX_INBOX_DIR = "INBOX";
|
|
49
|
+
/** Committed metadata blob name directly under `mailbox/INBOX/`. */
|
|
50
|
+
export const MAILBOX_INDEX_FILE = "index.json";
|
|
51
|
+
/** Suffix of a per-message raw-bytes blob (`<uid>.eml`). */
|
|
52
|
+
export const MAILBOX_EML_SUFFIX = ".eml";
|
|
53
|
+
/**
|
|
54
|
+
* The `mailbox/INBOX/` prefix, ending in `/` as
|
|
55
|
+
* `writeTreePreservingPrefix` requires. Every blob this backing writes is a
|
|
56
|
+
* direct child of it.
|
|
57
|
+
*/
|
|
58
|
+
export const MAILBOX_INBOX_PREFIX = `${MAILBOX_PREFIX}/${MAILBOX_INBOX_DIR}/`;
|
|
59
|
+
/** Relative directory path of the INBOX, for `CommittedReads.listDir`. */
|
|
60
|
+
const MAILBOX_INBOX_DIR_PATH = `${MAILBOX_PREFIX}/${MAILBOX_INBOX_DIR}`;
|
|
61
|
+
/** Current on-disk schema version of `index.json`. */
|
|
62
|
+
const INDEX_VERSION = 1;
|
|
63
|
+
const decoder = new TextDecoder();
|
|
64
|
+
const encoder = new TextEncoder();
|
|
65
|
+
/**
|
|
66
|
+
* On-disk envelope shape. Mirrors `StoredEnvelope` but serializes `date` as an
|
|
67
|
+
* ISO string and the three nullable header fields as `string | null` (JSON has
|
|
68
|
+
* no `undefined`); the loader maps `null` back to `undefined`.
|
|
69
|
+
*/
|
|
70
|
+
const StoredEnvelopeJson = type({
|
|
71
|
+
messageId: "string",
|
|
72
|
+
from: "string",
|
|
73
|
+
to: "string[]",
|
|
74
|
+
subject: "string",
|
|
75
|
+
date: "string",
|
|
76
|
+
inReplyTo: "string | null",
|
|
77
|
+
references: "string[]",
|
|
78
|
+
interchangeType: "string | null",
|
|
79
|
+
interchangeCorrelationId: "string | null",
|
|
80
|
+
});
|
|
81
|
+
/**
|
|
82
|
+
* On-disk `index.json` shape. Validated on every open: the committed tree is
|
|
83
|
+
* durable but external to this process, so it is parsed at the boundary rather
|
|
84
|
+
* than trusted. `expunged` records the uid and the modseq at which each
|
|
85
|
+
* message vanished so a QRESYNC `sync` can answer `vanished` since a client's
|
|
86
|
+
* known modseq.
|
|
87
|
+
*/
|
|
88
|
+
const MailboxIndexJson = type({
|
|
89
|
+
version: `${INDEX_VERSION}`,
|
|
90
|
+
uidValidity: "number >= 0",
|
|
91
|
+
uidNext: "number >= 1",
|
|
92
|
+
highestModSeq: "number >= 0",
|
|
93
|
+
messages: type({
|
|
94
|
+
uid: "number >= 1",
|
|
95
|
+
modseq: "number >= 1",
|
|
96
|
+
flags: "string[]",
|
|
97
|
+
envelope: StoredEnvelopeJson,
|
|
98
|
+
}).array(),
|
|
99
|
+
expunged: type({
|
|
100
|
+
uid: "number >= 1",
|
|
101
|
+
modseq: "number >= 1",
|
|
102
|
+
}).array(),
|
|
103
|
+
});
|
|
104
|
+
function serializeEnvelope(envelope) {
|
|
105
|
+
return {
|
|
106
|
+
messageId: envelope.messageId,
|
|
107
|
+
from: envelope.from,
|
|
108
|
+
to: envelope.to,
|
|
109
|
+
subject: envelope.subject,
|
|
110
|
+
date: envelope.date.toISOString(),
|
|
111
|
+
inReplyTo: envelope.inReplyTo ?? null,
|
|
112
|
+
references: envelope.references,
|
|
113
|
+
interchangeType: envelope.interchangeType ?? null,
|
|
114
|
+
interchangeCorrelationId: envelope.interchangeCorrelationId ?? null,
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
function deserializeEnvelope(raw) {
|
|
118
|
+
return {
|
|
119
|
+
messageId: raw.messageId,
|
|
120
|
+
from: raw.from,
|
|
121
|
+
to: raw.to,
|
|
122
|
+
subject: raw.subject,
|
|
123
|
+
date: new Date(raw.date),
|
|
124
|
+
inReplyTo: raw.inReplyTo === null ? undefined : raw.inReplyTo,
|
|
125
|
+
references: raw.references,
|
|
126
|
+
interchangeType: raw.interchangeType === null ? undefined : raw.interchangeType,
|
|
127
|
+
interchangeCorrelationId: raw.interchangeCorrelationId === null
|
|
128
|
+
? undefined
|
|
129
|
+
: raw.interchangeCorrelationId,
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
/** The `<uid>.eml` blob name for a message. */
|
|
133
|
+
function emlName(uid) {
|
|
134
|
+
return `${String(uid)}${MAILBOX_EML_SUFFIX}`;
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Load the committed `mailbox/INBOX/` METADATA into an in-memory state, or the
|
|
138
|
+
* empty state (a fresh `uidValidity`) when the repo, the ref, or the subtree
|
|
139
|
+
* does not yet exist. Only `index.json` is read; the per-message `<uid>.eml`
|
|
140
|
+
* blobs stay on disk and are read lazily by `readRaw`, so an open's resident
|
|
141
|
+
* footprint is bounded to metadata regardless of mailbox size. The committed
|
|
142
|
+
* read snapshot and the uid->oid map are retained so `readRaw` resolves a
|
|
143
|
+
* blob against the same pinned commit the open observed. Every read resolves
|
|
144
|
+
* against the committed object store, so an open observes committed state even
|
|
145
|
+
* when the working tree lags.
|
|
146
|
+
*/
|
|
147
|
+
async function loadCommittedState(opts) {
|
|
148
|
+
const empty = (reads) => ({
|
|
149
|
+
uidValidity: Date.now(),
|
|
150
|
+
uidNext: 1,
|
|
151
|
+
highestModSeq: 0,
|
|
152
|
+
messages: [],
|
|
153
|
+
expunged: [],
|
|
154
|
+
reads,
|
|
155
|
+
oidByUid: new Map(),
|
|
156
|
+
});
|
|
157
|
+
const reads = await opts.substrate.openCommittedReads(opts.principal, opts.repoId, opts.ref);
|
|
158
|
+
if (reads === null)
|
|
159
|
+
return empty(reads);
|
|
160
|
+
const entries = await reads.listDir(MAILBOX_INBOX_DIR_PATH);
|
|
161
|
+
const indexEntry = entries.find((e) => e.name === MAILBOX_INDEX_FILE && e.type === "blob");
|
|
162
|
+
if (indexEntry === undefined)
|
|
163
|
+
return empty(reads);
|
|
164
|
+
const indexBytes = await reads.readBlobByOid(indexEntry.oid);
|
|
165
|
+
let parsedJson;
|
|
166
|
+
try {
|
|
167
|
+
parsedJson = JSON.parse(decoder.decode(indexBytes));
|
|
168
|
+
}
|
|
169
|
+
catch (cause) {
|
|
170
|
+
throw new Error(`substrate mailbox store: ${MAILBOX_INBOX_PREFIX}${MAILBOX_INDEX_FILE} is not valid JSON`, { cause });
|
|
171
|
+
}
|
|
172
|
+
const index = MailboxIndexJson(parsedJson);
|
|
173
|
+
if (index instanceof type.errors) {
|
|
174
|
+
throw new Error(`substrate mailbox store: invalid ${MAILBOX_INBOX_PREFIX}${MAILBOX_INDEX_FILE}: ${index.summary}`);
|
|
175
|
+
}
|
|
176
|
+
const emlByName = new Map(entries
|
|
177
|
+
.filter((e) => e.type === "blob" && e.name.endsWith(MAILBOX_EML_SUFFIX))
|
|
178
|
+
.map((e) => [e.name, e.oid]));
|
|
179
|
+
const messages = [];
|
|
180
|
+
const oidByUid = new Map();
|
|
181
|
+
for (const entry of index.messages) {
|
|
182
|
+
const oid = emlByName.get(emlName(entry.uid));
|
|
183
|
+
if (oid === undefined) {
|
|
184
|
+
throw new Error(`substrate mailbox store: index references message uid ${String(entry.uid)} but ${MAILBOX_INBOX_PREFIX}${emlName(entry.uid)} is absent`);
|
|
185
|
+
}
|
|
186
|
+
// The blob's presence is asserted by its object id; its bytes are not read
|
|
187
|
+
// here -- `readRaw` reads them on demand.
|
|
188
|
+
oidByUid.set(entry.uid, oid);
|
|
189
|
+
messages.push({
|
|
190
|
+
uid: entry.uid,
|
|
191
|
+
modseq: entry.modseq,
|
|
192
|
+
flags: new Set(entry.flags),
|
|
193
|
+
envelope: deserializeEnvelope(entry.envelope),
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
return {
|
|
197
|
+
uidValidity: index.uidValidity,
|
|
198
|
+
uidNext: index.uidNext,
|
|
199
|
+
highestModSeq: index.highestModSeq,
|
|
200
|
+
messages,
|
|
201
|
+
expunged: index.expunged.map((e) => ({ uid: e.uid, modseq: e.modseq })),
|
|
202
|
+
reads,
|
|
203
|
+
oidByUid,
|
|
204
|
+
};
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* Create a workflow-run-substrate-backed `MailboxStore`. Loads the committed
|
|
208
|
+
* `mailbox/INBOX/` subtree into an in-memory mirror, then serves the
|
|
209
|
+
* synchronous `MailboxStore` surface over that mirror. Mutations stay in
|
|
210
|
+
* memory until `flush` persists them.
|
|
211
|
+
*/
|
|
212
|
+
export async function createSubstrateMailboxStore(opts) {
|
|
213
|
+
const state = await loadCommittedState(opts);
|
|
214
|
+
const messages = state.messages;
|
|
215
|
+
const expunged = state.expunged;
|
|
216
|
+
const uidValidity = state.uidValidity;
|
|
217
|
+
const reads = state.reads;
|
|
218
|
+
const oidByUid = state.oidByUid;
|
|
219
|
+
let uidCounter = state.uidNext;
|
|
220
|
+
// The next modseq to assign. `highestModSeq` is the largest assigned, so the
|
|
221
|
+
// next is one past it; a fresh mailbox (highestModSeq 0) starts at 1.
|
|
222
|
+
let modseqCounter = state.highestModSeq + 1;
|
|
223
|
+
let dirty = false;
|
|
224
|
+
// Delta tracking for `flush`. `index.json` changes on every mutation, so it
|
|
225
|
+
// is put unconditionally; each `<uid>.eml` is immutable and written once, so
|
|
226
|
+
// a flush need only put the blobs appended since the last successful flush
|
|
227
|
+
// and delete the blobs whose message was removed since then. The removed set
|
|
228
|
+
// clears on a successful flush; a flush that throws leaves it intact so the
|
|
229
|
+
// next flush re-attempts the same delta.
|
|
230
|
+
//
|
|
231
|
+
// `pendingRawByUid` holds the raw bytes of appended-but-not-yet-flushed
|
|
232
|
+
// messages -- the only raw this backing keeps resident. It doubles as the
|
|
233
|
+
// "appended since flush" set: `flush` puts each entry's blob, then drops it
|
|
234
|
+
// so a flushed message's bytes leave memory and are read from disk on demand.
|
|
235
|
+
const pendingRawByUid = new Map();
|
|
236
|
+
const removedSinceFlush = new Set();
|
|
237
|
+
function find(uid) {
|
|
238
|
+
return messages.find((m) => m.uid === uid);
|
|
239
|
+
}
|
|
240
|
+
function require(uid) {
|
|
241
|
+
const msg = find(uid);
|
|
242
|
+
if (msg === undefined) {
|
|
243
|
+
throw new Error(`Message UID ${String(uid)} not found`);
|
|
244
|
+
}
|
|
245
|
+
return msg;
|
|
246
|
+
}
|
|
247
|
+
async function flush() {
|
|
248
|
+
if (!dirty)
|
|
249
|
+
return;
|
|
250
|
+
const index = {
|
|
251
|
+
version: INDEX_VERSION,
|
|
252
|
+
uidValidity,
|
|
253
|
+
uidNext: uidCounter,
|
|
254
|
+
highestModSeq: modseqCounter - 1,
|
|
255
|
+
messages: messages.map((m) => ({
|
|
256
|
+
uid: m.uid,
|
|
257
|
+
modseq: m.modseq,
|
|
258
|
+
flags: Array.from(m.flags),
|
|
259
|
+
envelope: serializeEnvelope(m.envelope),
|
|
260
|
+
})),
|
|
261
|
+
expunged: expunged.map((e) => ({ uid: e.uid, modseq: e.modseq })),
|
|
262
|
+
};
|
|
263
|
+
// `index.json` is put on every flush. Each `<uid>.eml` is immutable, so
|
|
264
|
+
// only the blobs appended since the last successful flush are put and only
|
|
265
|
+
// those whose message was removed are deleted; every other `.eml` is
|
|
266
|
+
// carried forward by object id, so the flush never re-hashes the mailbox's
|
|
267
|
+
// whole history.
|
|
268
|
+
const puts = {
|
|
269
|
+
[`${MAILBOX_INBOX_PREFIX}${MAILBOX_INDEX_FILE}`]: encoder.encode(JSON.stringify(index)),
|
|
270
|
+
};
|
|
271
|
+
const flushedUids = [];
|
|
272
|
+
for (const [uid, raw] of pendingRawByUid) {
|
|
273
|
+
puts[`${MAILBOX_INBOX_PREFIX}${emlName(uid)}`] = raw;
|
|
274
|
+
flushedUids.push(uid);
|
|
275
|
+
}
|
|
276
|
+
const deletes = Array.from(removedSinceFlush, (uid) => `${MAILBOX_INBOX_PREFIX}${emlName(uid)}`);
|
|
277
|
+
await opts.substrate.writeTreeDelta(opts.principal, opts.repoId, opts.ref, {
|
|
278
|
+
computeDelta: async () => ({ puts, deletes }),
|
|
279
|
+
changedPathPrefixes: new Set([MAILBOX_INBOX_PREFIX]),
|
|
280
|
+
message: `persist mailbox INBOX (${String(messages.length)} message(s))`,
|
|
281
|
+
});
|
|
282
|
+
// The appended blobs are now committed, so their raw leaves memory: a later
|
|
283
|
+
// `readRaw` reads them from disk. Their object ids are not recorded here
|
|
284
|
+
// (the pinned committed-read snapshot predates this commit), so `readRaw`
|
|
285
|
+
// resolves a post-open append only while its raw is still pending; the
|
|
286
|
+
// long-lived writer never reads its own appends back, and every reader
|
|
287
|
+
// opens a fresh snapshot that sees the committed blob.
|
|
288
|
+
for (const uid of flushedUids) {
|
|
289
|
+
pendingRawByUid.delete(uid);
|
|
290
|
+
}
|
|
291
|
+
removedSinceFlush.clear();
|
|
292
|
+
dirty = false;
|
|
293
|
+
}
|
|
294
|
+
function sync(known) {
|
|
295
|
+
const highestModSeq = modseqCounter - 1;
|
|
296
|
+
if (known.uidValidity !== uidValidity) {
|
|
297
|
+
return {
|
|
298
|
+
resync: true,
|
|
299
|
+
uidValidity,
|
|
300
|
+
uidNext: uidCounter,
|
|
301
|
+
highestModSeq,
|
|
302
|
+
messages: messages.slice(),
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
const changed = messages
|
|
306
|
+
.filter((m) => m.modseq > known.highestModSeq)
|
|
307
|
+
.sort((a, b) => a.uid - b.uid);
|
|
308
|
+
const vanished = expunged
|
|
309
|
+
.filter((e) => e.modseq > known.highestModSeq)
|
|
310
|
+
.map((e) => e.uid)
|
|
311
|
+
.sort((a, b) => a - b);
|
|
312
|
+
return {
|
|
313
|
+
resync: false,
|
|
314
|
+
uidValidity,
|
|
315
|
+
uidNext: uidCounter,
|
|
316
|
+
highestModSeq,
|
|
317
|
+
changed,
|
|
318
|
+
vanished,
|
|
319
|
+
};
|
|
320
|
+
}
|
|
321
|
+
return {
|
|
322
|
+
uidValidity,
|
|
323
|
+
get uidNext() {
|
|
324
|
+
return uidCounter;
|
|
325
|
+
},
|
|
326
|
+
get highestModSeq() {
|
|
327
|
+
return modseqCounter - 1;
|
|
328
|
+
},
|
|
329
|
+
get messages() {
|
|
330
|
+
return messages;
|
|
331
|
+
},
|
|
332
|
+
get pendingWrites() {
|
|
333
|
+
return dirty;
|
|
334
|
+
},
|
|
335
|
+
append(raw, envelope, flags) {
|
|
336
|
+
const uid = uidCounter++;
|
|
337
|
+
const modseq = modseqCounter++;
|
|
338
|
+
messages.push({ uid, modseq, flags: new Set(flags), envelope });
|
|
339
|
+
pendingRawByUid.set(uid, raw);
|
|
340
|
+
dirty = true;
|
|
341
|
+
return uid;
|
|
342
|
+
},
|
|
343
|
+
async readRaw(uid) {
|
|
344
|
+
if (find(uid) === undefined) {
|
|
345
|
+
throw new Error(`Message UID ${String(uid)} not found`);
|
|
346
|
+
}
|
|
347
|
+
// An appended-but-not-yet-flushed message keeps its raw in memory; a
|
|
348
|
+
// flushed or previously-committed message reads its blob from the pinned
|
|
349
|
+
// committed-read snapshot on demand.
|
|
350
|
+
const pending = pendingRawByUid.get(uid);
|
|
351
|
+
if (pending !== undefined)
|
|
352
|
+
return pending;
|
|
353
|
+
const oid = oidByUid.get(uid);
|
|
354
|
+
if (oid === undefined || reads === null) {
|
|
355
|
+
throw new Error(`substrate mailbox store: no committed blob for message uid ${String(uid)}; its raw bytes are not resolvable from this snapshot`);
|
|
356
|
+
}
|
|
357
|
+
return reads.readBlobByOid(oid);
|
|
358
|
+
},
|
|
359
|
+
find,
|
|
360
|
+
addFlags(uid, flags) {
|
|
361
|
+
const msg = require(uid);
|
|
362
|
+
for (const flag of flags) {
|
|
363
|
+
msg.flags.add(flag);
|
|
364
|
+
}
|
|
365
|
+
msg.modseq = modseqCounter++;
|
|
366
|
+
dirty = true;
|
|
367
|
+
return msg;
|
|
368
|
+
},
|
|
369
|
+
removeFlags(uid, flags) {
|
|
370
|
+
const msg = require(uid);
|
|
371
|
+
for (const flag of flags) {
|
|
372
|
+
msg.flags.delete(flag);
|
|
373
|
+
}
|
|
374
|
+
msg.modseq = modseqCounter++;
|
|
375
|
+
dirty = true;
|
|
376
|
+
return msg;
|
|
377
|
+
},
|
|
378
|
+
remove(uid) {
|
|
379
|
+
const idx = messages.findIndex((m) => m.uid === uid);
|
|
380
|
+
if (idx === -1) {
|
|
381
|
+
throw new Error(`Message UID ${String(uid)} not found`);
|
|
382
|
+
}
|
|
383
|
+
messages.splice(idx, 1);
|
|
384
|
+
// Record the expunge with a fresh modseq so a QRESYNC `sync` can report
|
|
385
|
+
// this uid as `vanished` to a client whose known modseq predates it. The
|
|
386
|
+
// in-memory reference backing does not advance modseq on remove; this
|
|
387
|
+
// backing does, because it must answer QRESYNC across reopens.
|
|
388
|
+
expunged.push({ uid, modseq: modseqCounter++ });
|
|
389
|
+
// A message appended and removed within the same flush window was never
|
|
390
|
+
// committed, so its `.eml` must be neither put nor deleted: drop its
|
|
391
|
+
// pending raw. Otherwise the blob is already committed and the next flush
|
|
392
|
+
// deletes it.
|
|
393
|
+
if (pendingRawByUid.has(uid)) {
|
|
394
|
+
pendingRawByUid.delete(uid);
|
|
395
|
+
}
|
|
396
|
+
else {
|
|
397
|
+
removedSinceFlush.add(uid);
|
|
398
|
+
}
|
|
399
|
+
dirty = true;
|
|
400
|
+
},
|
|
401
|
+
flush,
|
|
402
|
+
sync,
|
|
403
|
+
};
|
|
404
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { type SubstrateMailboxStore, type SubstrateMailboxStoreOpts } from "../adapters/substrate-mailbox-store.js";
|
|
2
|
+
export interface ChildMailboxReader {
|
|
3
|
+
/**
|
|
4
|
+
* Open a fresh committed snapshot of the deployment's substrate INBOX. Each
|
|
5
|
+
* call re-reads the committed `mailbox/INBOX/` subtree, so a snapshot opened
|
|
6
|
+
* after a `mailbox.notify` observes the newly committed message.
|
|
7
|
+
*/
|
|
8
|
+
open(): Promise<SubstrateMailboxStore>;
|
|
9
|
+
}
|
|
10
|
+
export declare function createChildMailboxReader(opts: SubstrateMailboxStoreOpts): ChildMailboxReader;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
// Child-side substrate mailbox reader (INBOUND half of mailbox ownership,
|
|
2
|
+
// design §3b).
|
|
3
|
+
//
|
|
4
|
+
// The supervisor commits an arrived message to the deployment's workflow-run
|
|
5
|
+
// substrate mailbox (`mailbox/INBOX/`), then fires a `mailbox.notify` control
|
|
6
|
+
// frame. A step agent's supervisor-backed transport answers the IMAP read
|
|
7
|
+
// surface (`search`, `fetchHeaders`, `fetchFull`, ...) by opening a fresh
|
|
8
|
+
// committed snapshot of that mailbox through this reader. `open` re-reads the
|
|
9
|
+
// committed subtree on every call, so a snapshot taken after a `mailbox.notify`
|
|
10
|
+
// observes the message the supervisor just committed.
|
|
11
|
+
//
|
|
12
|
+
// Modeled on the sibling `createMailPartReader` wiring: the same per-deployment
|
|
13
|
+
// substrate handles (substrate, repoId, principal, workflow-run ref), bound
|
|
14
|
+
// once so the transport reads without re-threading them. `createSubstrateMailboxStore`
|
|
15
|
+
// loads committed state on open and is async, so `open` returns a promise.
|
|
16
|
+
import { createSubstrateMailboxStore, } from "../adapters/substrate-mailbox-store.js";
|
|
17
|
+
export function createChildMailboxReader(opts) {
|
|
18
|
+
return {
|
|
19
|
+
open() {
|
|
20
|
+
return createSubstrateMailboxStore(opts);
|
|
21
|
+
},
|
|
22
|
+
};
|
|
23
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { CredentialDelivery } from "@intx/types/sidecar";
|
|
2
|
+
/**
|
|
3
|
+
* Apply a `credentials-updated` frame to the current cell and return the next
|
|
4
|
+
* cell. Pure: it reads neither argument's identity back out, so the caller can
|
|
5
|
+
* assign the result to the live ref in one atomic whole-object swap and a
|
|
6
|
+
* concurrent reader never observes a torn cell.
|
|
7
|
+
*/
|
|
8
|
+
export declare function mergeCredentialDelivery(current: CredentialDelivery | null, delivery: CredentialDelivery, revoke: readonly string[] | undefined): CredentialDelivery;
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
// Merge semantics for the child's in-memory credential-material cell.
|
|
2
|
+
//
|
|
3
|
+
// The cell is fed by several INDEPENDENTLY-SCOPED producers: the deploy frame
|
|
4
|
+
// (tool bindings plus the run's inference materials, frozen at deploy time and
|
|
5
|
+
// re-asserted on every spawn and pre-trigger barrier), an inference rotation
|
|
6
|
+
// (materials only, no binding), and a tool-grant push (tool bindings plus their
|
|
7
|
+
// materials). Each producer carries only its own slice of the cell. A wholesale
|
|
8
|
+
// swap would therefore let one producer evict another's credentials -- an
|
|
9
|
+
// inference rotation would drop every tool binding, a barrier re-push would drop
|
|
10
|
+
// a live-pushed tool credential. So a `credentials-updated` frame MERGES:
|
|
11
|
+
// materials upsert by `credentialId`, bindings upsert by `(consumer, handle)`.
|
|
12
|
+
//
|
|
13
|
+
// Because omission no longer means "gone", revocation is EXPLICIT: a frame's
|
|
14
|
+
// `revoke` list names the credentialIds to drop, and dropping one also drops
|
|
15
|
+
// every binding that references it. Revocation is applied before the upsert, so
|
|
16
|
+
// a frame that both revokes and re-adds the same credentialId resolves to the
|
|
17
|
+
// re-add. A material an upsert leaves unreferenced (a rebind of a handle to a
|
|
18
|
+
// different credential) is not garbage-collected -- inference materials never
|
|
19
|
+
// carry a binding, so "unreferenced" is not a drop signal; only an explicit
|
|
20
|
+
// `revoke` removes a material.
|
|
21
|
+
// A binding's identity is the (consumer, handle) pair: a handle string is only
|
|
22
|
+
// unique within a consumer, so two consumers can each bind their own handle of
|
|
23
|
+
// the same name. The U+0000 joiner keeps the pair injective where a printable
|
|
24
|
+
// joiner would not -- a handle is an arbitrary string that may contain a space
|
|
25
|
+
// or a colon, but by convention never a NUL.
|
|
26
|
+
function bindingKey(binding) {
|
|
27
|
+
return `${binding.consumer}\u0000${binding.handle}`;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Apply a `credentials-updated` frame to the current cell and return the next
|
|
31
|
+
* cell. Pure: it reads neither argument's identity back out, so the caller can
|
|
32
|
+
* assign the result to the live ref in one atomic whole-object swap and a
|
|
33
|
+
* concurrent reader never observes a torn cell.
|
|
34
|
+
*/
|
|
35
|
+
export function mergeCredentialDelivery(current, delivery, revoke) {
|
|
36
|
+
const materials = new Map();
|
|
37
|
+
const bindings = new Map();
|
|
38
|
+
if (current !== null) {
|
|
39
|
+
for (const material of current.materials) {
|
|
40
|
+
materials.set(material.credentialId, material);
|
|
41
|
+
}
|
|
42
|
+
for (const binding of current.bindings) {
|
|
43
|
+
bindings.set(bindingKey(binding), binding);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
if (revoke !== undefined) {
|
|
47
|
+
for (const credentialId of revoke) {
|
|
48
|
+
materials.delete(credentialId);
|
|
49
|
+
for (const [key, binding] of bindings) {
|
|
50
|
+
if (binding.credentialId === credentialId) {
|
|
51
|
+
bindings.delete(key);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
for (const material of delivery.materials) {
|
|
57
|
+
materials.set(material.credentialId, material);
|
|
58
|
+
}
|
|
59
|
+
for (const binding of delivery.bindings) {
|
|
60
|
+
bindings.set(bindingKey(binding), binding);
|
|
61
|
+
}
|
|
62
|
+
return {
|
|
63
|
+
bindings: [...bindings.values()],
|
|
64
|
+
materials: [...materials.values()],
|
|
65
|
+
};
|
|
66
|
+
}
|
|
@@ -2,6 +2,7 @@ import { type SpawnTimeEnv } from "./env-bootstrap.js";
|
|
|
2
2
|
import { type RunWorkflowChildBindings, type RunWorkflowChildResult } from "./run-child.js";
|
|
3
3
|
import { type ChildSubstrateWriteBridge } from "./substrate-write-bridge.js";
|
|
4
4
|
import { type ChildOutboundMailBridge } from "./outbound-mail-bridge.js";
|
|
5
|
+
import { type ChildMailboxMutationBridge } from "./mailbox-mutation-bridge.js";
|
|
5
6
|
import { type FrameWriter, type NdjsonReader, type NdjsonWriter } from "../ipc/index.js";
|
|
6
7
|
/**
|
|
7
8
|
* File descriptor the supervisor's `Bun.spawn` wires the
|
|
@@ -63,6 +64,17 @@ export interface SubstrateFactoryEnv {
|
|
|
63
64
|
* agent's signature without the child ever holding the agent's key.
|
|
64
65
|
*/
|
|
65
66
|
readonly outboundMailBridge: ChildOutboundMailBridge;
|
|
67
|
+
/**
|
|
68
|
+
* Child-side IPC bridge over the upstream control channel for the
|
|
69
|
+
* INBOUND half of mailbox ownership (§3b). The substrate factory uses
|
|
70
|
+
* this to construct the supervisor-backed `MessageTransport`'s write
|
|
71
|
+
* surface: `setFlags` / `clearFlags` / `expunge` call `bridge.submit`,
|
|
72
|
+
* which emits a `mailbox.mutate.request` upstream frame and resolves
|
|
73
|
+
* once the supervisor's matching `mailbox.mutate.response` lands. The
|
|
74
|
+
* supervisor -- the sole mailbox writer -- applies the mutation to its
|
|
75
|
+
* owned store, so the child never flushes the run ref itself.
|
|
76
|
+
*/
|
|
77
|
+
readonly mailboxMutationBridge: ChildMailboxMutationBridge;
|
|
66
78
|
}
|
|
67
79
|
/**
|
|
68
80
|
* Substrate-factory callback the host supplies to
|
|
@@ -21,6 +21,7 @@ import { parseSpawnTimeEnv } from "./env-bootstrap.js";
|
|
|
21
21
|
import { runWorkflowChild, } from "./run-child.js";
|
|
22
22
|
import { createChildSubstrateWriteBridge, } from "./substrate-write-bridge.js";
|
|
23
23
|
import { createChildOutboundMailBridge, } from "./outbound-mail-bridge.js";
|
|
24
|
+
import { createChildMailboxMutationBridge, } from "./mailbox-mutation-bridge.js";
|
|
24
25
|
import { createControlChannelSender, } from "../ipc/index.js";
|
|
25
26
|
/**
|
|
26
27
|
* File descriptor the supervisor's `Bun.spawn` wires the
|
|
@@ -76,11 +77,15 @@ export async function runWorkflowChildFromProcessEnv(factory, opts = {}) {
|
|
|
76
77
|
const outboundMailBridge = createChildOutboundMailBridge({
|
|
77
78
|
upstreamSender,
|
|
78
79
|
});
|
|
80
|
+
const mailboxMutationBridge = createChildMailboxMutationBridge({
|
|
81
|
+
upstreamSender,
|
|
82
|
+
});
|
|
79
83
|
const bindings = await factory({
|
|
80
84
|
spawn,
|
|
81
85
|
substrateConfig,
|
|
82
86
|
substrateWriteBridge,
|
|
83
87
|
outboundMailBridge,
|
|
88
|
+
mailboxMutationBridge,
|
|
84
89
|
});
|
|
85
90
|
return runWorkflowChild({
|
|
86
91
|
env: spawn,
|
|
@@ -98,6 +103,7 @@ export async function runWorkflowChildFromProcessEnv(factory, opts = {}) {
|
|
|
98
103
|
upstreamSender,
|
|
99
104
|
substrateWriteBridge,
|
|
100
105
|
outboundMailBridge,
|
|
106
|
+
mailboxMutationBridge,
|
|
101
107
|
});
|
|
102
108
|
}
|
|
103
109
|
const SubstrateConfigValue = type("string > 0");
|
package/dist/child/index.d.ts
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
export { createCredentialsBackedAuthorize, hashGrants, runWorkflowChild, type ChildStepInvoker, type CredentialsSnapshotRef, type CredentialWiring, type DrainController, type GrantEvaluator, type RunWorkflowChildBindings, type RunWorkflowChildOpts, type RunWorkflowChildResult, type SourcesSnapshotRef, type SubstrateWriteResponseSink, } from "./run-child.js";
|
|
2
2
|
export { createChildSubstrateWriteBridge, type ChildSubstrateWriteBridge, type CreateChildSubstrateWriteBridgeOpts, type SubstrateWriteRequest, } from "./substrate-write-bridge.js";
|
|
3
3
|
export { createChildOutboundMailBridge, type ChildOutboundMailBridge, type CreateChildOutboundMailBridgeOpts, } from "./outbound-mail-bridge.js";
|
|
4
|
-
export {
|
|
4
|
+
export { createChildMailboxMutationBridge, type ChildMailboxMutationBridge, type CreateChildMailboxMutationBridgeOpts, type MailboxMutation, type MailboxMutationResult, } from "./mailbox-mutation-bridge.js";
|
|
5
|
+
export { createSupervisorBackedTransport, type SupervisorBackedTransportInbound, } from "./supervisor-backed-transport.js";
|
|
6
|
+
export { createMailboxWatchRegistry, type MailboxWatchRegistry, } from "./mailbox-watch-registry.js";
|
|
7
|
+
export { createChildMailboxReader, type ChildMailboxReader, } from "./child-mailbox-reader.js";
|
|
5
8
|
export { createProxyWorkflowRunRepoStore, type CreateProxyWorkflowRunRepoStoreOpts, } from "./proxy-repo-store.js";
|
|
6
9
|
export { parseSpawnTimeEnv, type SpawnTimeEnv } from "./env-bootstrap.js";
|
|
7
10
|
export { discoverInFlightRuns, type DiscoverRunsOpts, type DiscoveredRun, } from "./self-discovery.js";
|
package/dist/child/index.js
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
export { createCredentialsBackedAuthorize, hashGrants, runWorkflowChild, } from "./run-child.js";
|
|
2
2
|
export { createChildSubstrateWriteBridge, } from "./substrate-write-bridge.js";
|
|
3
3
|
export { createChildOutboundMailBridge, } from "./outbound-mail-bridge.js";
|
|
4
|
-
export {
|
|
4
|
+
export { createChildMailboxMutationBridge, } from "./mailbox-mutation-bridge.js";
|
|
5
|
+
export { createSupervisorBackedTransport, } from "./supervisor-backed-transport.js";
|
|
6
|
+
export { createMailboxWatchRegistry, } from "./mailbox-watch-registry.js";
|
|
7
|
+
export { createChildMailboxReader, } from "./child-mailbox-reader.js";
|
|
5
8
|
export { createProxyWorkflowRunRepoStore, } from "./proxy-repo-store.js";
|
|
6
9
|
export { parseSpawnTimeEnv } from "./env-bootstrap.js";
|
|
7
10
|
export { discoverInFlightRuns, } from "./self-discovery.js";
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import type { ControlChannelSender, ControlPayload } from "../ipc/control-channel.js";
|
|
2
|
+
/**
|
|
3
|
+
* A mailbox mutation the child asks the supervisor to apply. A flag
|
|
4
|
+
* mutation carries the target `uid` and the `flags` to add or remove; an
|
|
5
|
+
* `expunge` sweeps every `\Deleted` message in the mailbox and so carries
|
|
6
|
+
* neither.
|
|
7
|
+
*/
|
|
8
|
+
export type MailboxMutation = {
|
|
9
|
+
runId: string;
|
|
10
|
+
mailbox: string;
|
|
11
|
+
op: "addFlags" | "removeFlags";
|
|
12
|
+
uid: number;
|
|
13
|
+
flags: string[];
|
|
14
|
+
} | {
|
|
15
|
+
runId: string;
|
|
16
|
+
mailbox: string;
|
|
17
|
+
op: "expunge";
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* The supervisor's applied-mutation result. `expungedUids` is present
|
|
21
|
+
* only for an `expunge` and lists the uids the sweep removed, so the
|
|
22
|
+
* agent tool can report how many messages it consumed.
|
|
23
|
+
*/
|
|
24
|
+
export type MailboxMutationResult = {
|
|
25
|
+
expungedUids?: number[];
|
|
26
|
+
};
|
|
27
|
+
/**
|
|
28
|
+
* Bridge surface the child's supervisor-backed transport reaches into.
|
|
29
|
+
* `submit` sends a `mailbox.mutate.request` upstream and resolves once
|
|
30
|
+
* the supervisor's matching `mailbox.mutate.response` lands.
|
|
31
|
+
* `handleResult` is the receiver-side entry point the child's control
|
|
32
|
+
* loop invokes when the downstream `mailbox.mutate.response` frame
|
|
33
|
+
* arrives. `cancelAll` is the cleanup hook the control loop invokes on
|
|
34
|
+
* any exit path so a pending mutation does not leak an awaiter when the
|
|
35
|
+
* supervisor has torn the IPC down.
|
|
36
|
+
*/
|
|
37
|
+
export interface ChildMailboxMutationBridge {
|
|
38
|
+
submit(mutation: MailboxMutation): Promise<MailboxMutationResult>;
|
|
39
|
+
handleResult(data: Extract<ControlPayload, {
|
|
40
|
+
type: "mailbox.mutate.response";
|
|
41
|
+
}>["data"]): void;
|
|
42
|
+
cancelAll(reason: string): void;
|
|
43
|
+
readonly pendingCount: number;
|
|
44
|
+
}
|
|
45
|
+
export interface CreateChildMailboxMutationBridgeOpts {
|
|
46
|
+
upstreamSender: ControlChannelSender;
|
|
47
|
+
/**
|
|
48
|
+
* Optional `requestId` allocator. Production wires a per-instance
|
|
49
|
+
* monotonic counter plus a random suffix; tests inject a
|
|
50
|
+
* deterministic factory so the upstream frame's `requestId` is
|
|
51
|
+
* predictable.
|
|
52
|
+
*/
|
|
53
|
+
allocateRequestId?: () => string;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Construct the child-side mailbox-mutation bridge. Pending mutations
|
|
57
|
+
* live in the shared pending-request core keyed by `requestId`; the
|
|
58
|
+
* bridge resolves the awaiter when the supervisor's matching
|
|
59
|
+
* `mailbox.mutate.response` lands.
|
|
60
|
+
*/
|
|
61
|
+
export declare function createChildMailboxMutationBridge(opts: CreateChildMailboxMutationBridgeOpts): ChildMailboxMutationBridge;
|