@intx/hub-sessions 0.2.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -5
- package/dist/agent-repo.d.ts +23 -7
- package/dist/agent-repo.js +19 -6
- package/dist/agent-state-kind.js +18 -63
- package/dist/asset-service.d.ts +1 -20
- package/dist/asset-service.js +19 -97
- package/dist/committed-source-tree.d.ts +10 -0
- package/dist/committed-source-tree.js +35 -0
- package/dist/credential-push.d.ts +52 -7
- package/dist/credential-push.js +178 -22
- package/dist/event-collector-registry.d.ts +3 -2
- package/dist/event-collector-registry.js +42 -13
- package/dist/event-collector.d.ts +12 -2
- package/dist/event-collector.js +45 -4
- package/dist/hub-session-lookups.d.ts +125 -7
- package/dist/hub-session-lookups.js +541 -86
- package/dist/hub-session-orchestrator.d.ts +2 -3
- package/dist/hub-session-orchestrator.js +22 -56
- package/dist/index.d.ts +19 -9
- package/dist/index.js +16 -7
- package/dist/reconciliation-scheduler.d.ts +14 -0
- package/dist/reconciliation-scheduler.js +55 -0
- package/dist/repo-store/index.d.ts +2 -1
- package/dist/repo-store/index.js +1 -0
- package/dist/repo-store/store.d.ts +1 -1
- package/dist/repo-store/store.js +138 -1
- package/dist/repo-store/subscribe-kind.d.ts +6 -3
- package/dist/repo-store/subscribe-kind.js +42 -77
- package/dist/repo-store/types.d.ts +94 -6
- package/dist/repo-store/user-principal-gate.d.ts +26 -0
- package/dist/repo-store/user-principal-gate.js +78 -0
- package/dist/session-service.d.ts +252 -126
- package/dist/session-service.js +851 -624
- package/dist/sidecar-allocation/capability-policy.d.ts +27 -0
- package/dist/sidecar-allocation/capability-policy.js +124 -0
- package/dist/sidecar-allocation/contracts.d.ts +101 -0
- package/dist/sidecar-allocation/contracts.js +26 -0
- package/dist/sidecar-allocation/index.d.ts +5 -0
- package/dist/sidecar-allocation/index.js +4 -0
- package/dist/sidecar-allocation/operation.d.ts +10 -0
- package/dist/sidecar-allocation/operation.js +54 -0
- package/dist/sidecar-allocation/plugin-registry.d.ts +24 -0
- package/dist/sidecar-allocation/plugin-registry.js +61 -0
- package/dist/sidecar-allocation/reconciler.d.ts +54 -0
- package/dist/sidecar-allocation/reconciler.js +825 -0
- package/dist/skill-kind.js +12 -62
- package/dist/substrate.d.ts +3 -3
- package/dist/substrate.js +1 -1
- package/dist/workflow-allocation-service.d.ts +64 -0
- package/dist/workflow-allocation-service.js +554 -0
- package/dist/workflow-closure-resolution.d.ts +106 -0
- package/dist/workflow-closure-resolution.js +123 -0
- package/dist/workflow-definition-ensure.d.ts +24 -0
- package/dist/workflow-definition-ensure.js +75 -0
- package/dist/workflow-dispatch-service.d.ts +42 -0
- package/dist/workflow-dispatch-service.js +209 -0
- package/dist/workflow-dispatch-settlement.d.ts +29 -0
- package/dist/workflow-dispatch-settlement.js +140 -0
- package/dist/workflow-kind.d.ts +29 -1
- package/dist/workflow-kind.js +143 -139
- package/dist/workflow-probe-gate.d.ts +286 -0
- package/dist/workflow-probe-gate.js +382 -0
- package/dist/workflow-run-kind.d.ts +239 -32
- package/dist/workflow-run-kind.js +874 -188
- package/dist/workflow-run-reader.d.ts +1 -1
- package/dist/workflow-run-reader.js +3 -7
- package/dist/workflow-run-restore.d.ts +16 -0
- package/dist/workflow-run-restore.js +30 -0
- package/dist/workflow-source-closure.d.ts +35 -0
- package/dist/workflow-source-closure.js +342 -0
- package/dist/workflow-source-pins.d.ts +8 -0
- package/dist/workflow-source-pins.js +14 -0
- package/dist/ws/index.d.ts +3 -3
- package/dist/ws/index.js +2 -2
- package/dist/ws/pending-tracker.d.ts +93 -0
- package/dist/ws/pending-tracker.js +132 -0
- package/dist/ws/sidecar-events.d.ts +139 -37
- package/dist/ws/sidecar-events.js +2 -2
- package/dist/ws/sidecar-handler.d.ts +218 -60
- package/dist/ws/sidecar-handler.js +1713 -732
- package/dist/ws/sidecar-handler.test-helpers.d.ts +38 -0
- package/dist/ws/sidecar-handler.test-helpers.js +95 -0
- package/dist/ws/sidecar-token-authenticator.d.ts +3 -1
- package/dist/ws/sidecar-token-authenticator.js +78 -7
- package/package.json +14 -13
- package/dist/available-skills-stanza.d.ts +0 -21
- package/dist/available-skills-stanza.js +0 -32
|
@@ -151,11 +151,11 @@
|
|
|
151
151
|
// Authz:
|
|
152
152
|
// - `hub` principal: full access.
|
|
153
153
|
// - `workflow-process` principal: read/write its own deployment's
|
|
154
|
-
// event log. The principal carries `{
|
|
155
|
-
// this handler verifies `repoId.id ===
|
|
154
|
+
// event log. The principal carries `{ anchorRunId, runId? }`;
|
|
155
|
+
// this handler verifies `repoId.id === anchorRunId`.
|
|
156
156
|
// - `supervisor` principal: read/write its own deployment's event
|
|
157
|
-
// log. The principal carries `{
|
|
158
|
-
// verifies `repoId.id ===
|
|
157
|
+
// log. The principal carries `{ anchorRunId }`; this handler
|
|
158
|
+
// verifies `repoId.id === anchorRunId`.
|
|
159
159
|
// - `sidecar` principal: read-only (createPack, resolveRef) for
|
|
160
160
|
// resume.
|
|
161
161
|
// - `user` principal: gated by bearer-token claims and the route
|
|
@@ -165,19 +165,33 @@ import fs from "node:fs";
|
|
|
165
165
|
import git from "isomorphic-git";
|
|
166
166
|
import { type } from "arktype";
|
|
167
167
|
import { getLogger } from "@intx/log";
|
|
168
|
-
import {
|
|
169
|
-
import { UserPrincipal, } from "./repo-store/index.js";
|
|
168
|
+
import { authorizeUserPrincipal, } from "./repo-store/index.js";
|
|
170
169
|
import { WORKFLOW_RUN_EVENTS_FILE, splitCombinedEventLog, encodeCombinedEventLog, } from "./workflow-run-event-log.js";
|
|
171
170
|
const logger = getLogger(["hub-sessions", "workflow-run-kind"]);
|
|
172
171
|
export const WORKFLOW_RUN_GITIGNORE_PATH = ".gitignore";
|
|
173
172
|
export const WORKFLOW_RUN_RUNS_PREFIX = "runs";
|
|
174
173
|
export const WORKFLOW_RUN_EVENTS_DIR = "events";
|
|
175
174
|
export const WORKFLOW_RUN_BLOBS_DIR = "blobs";
|
|
175
|
+
export const WORKFLOW_RUN_GRANTS_FILE = "grants.json";
|
|
176
176
|
export const WORKFLOW_RUN_ADDRESSES_PREFIX = "addresses";
|
|
177
177
|
export const WORKFLOW_RUN_CONTROL_PREFIX = "control";
|
|
178
178
|
export const WORKFLOW_RUN_INBOX_DIR = "inbox";
|
|
179
179
|
export const WORKFLOW_RUN_PROCESSING_DIR = "processing";
|
|
180
180
|
export const WORKFLOW_RUN_CONSUMED_DIR = "consumed";
|
|
181
|
+
/**
|
|
182
|
+
* Per-run inbound mail-part subtree. Non-text inbound mail content
|
|
183
|
+
* (image/audio/video/document mail parts) is committed here as real
|
|
184
|
+
* files rather than inlined into the JSON event log, whose serialization
|
|
185
|
+
* boundary would corrupt binary bytes. The layout is
|
|
186
|
+
* `runs/<runId>/parts/<urlEncoded(messageId)>/<index>-<name>`: one
|
|
187
|
+
* directory per inbound message (so a long-lived run's successive turns
|
|
188
|
+
* never collide), and one file per mail part carrying its verbatim
|
|
189
|
+
* bytes. The workflow-host ingest writes the bytes and records a
|
|
190
|
+
* lightweight `{ name, contentType, ref }` reference into the run's
|
|
191
|
+
* trigger / signal payload; the step invoker reads the bytes back at
|
|
192
|
+
* `agent.send` time. Files are immutable once written, like `blobs/`.
|
|
193
|
+
*/
|
|
194
|
+
export const WORKFLOW_RUN_PARTS_DIR = "parts";
|
|
181
195
|
/**
|
|
182
196
|
* Filename of the per-address retention watermark blob, a direct child
|
|
183
197
|
* of `addresses/<urlEncoded>/` (a file, not a directory). Carries the
|
|
@@ -195,15 +209,28 @@ export const WORKFLOW_RUN_WATERMARK_FILE = "watermark.json";
|
|
|
195
209
|
* deduped by a retained consumed entry, short enough that `consumed/`
|
|
196
210
|
* reaches a bounded steady state of one day's message volume.
|
|
197
211
|
*
|
|
198
|
-
* INVARIANT (operator-owned): the horizon must be >= the longest
|
|
199
|
-
*
|
|
200
|
-
*
|
|
201
|
-
*
|
|
202
|
-
* the
|
|
203
|
-
*
|
|
204
|
-
*
|
|
205
|
-
* re-
|
|
206
|
-
*
|
|
212
|
+
* INVARIANT (operator-owned): the horizon must be >= the longest window in
|
|
213
|
+
* which the same `messageId` could legitimately be re-submitted and still must
|
|
214
|
+
* be caught as a duplicate. The hub now redelivers un-acked inbound mail
|
|
215
|
+
* (connected-window retry and reconnect-redelivery), so an at-least-once
|
|
216
|
+
* internal source DOES exist -- but the dedup guarantee against it does not
|
|
217
|
+
* rest on window arithmetic. It rests on a STRUCTURAL fact: `enqueueInbox` is
|
|
218
|
+
* only ever called with a freshly stamped `receivedAt` (a redelivery
|
|
219
|
+
* re-enters `onMailMessage` and re-stamps `Date.now()`, never carrying the
|
|
220
|
+
* original), and the watermark only ever advances to at most
|
|
221
|
+
* `consumedAt - retentionHorizonMs`, which is <= now, so a fresh `receivedAt`
|
|
222
|
+
* always sits a full horizon above the watermark and can never be stale-
|
|
223
|
+
* refused. A redelivery instead hits the `consumed/`/`processing/`/`inbox/`
|
|
224
|
+
* dedup index and is deduped there. The sole path that carries an original
|
|
225
|
+
* (old) `receivedAt` back into the queue is `replayProcessingToInbox`, which
|
|
226
|
+
* writes straight to `inbox/` and bypasses the stale gate entirely. So
|
|
227
|
+
* `claim_check_stale_enqueue` is unreachable via redelivery today. If any
|
|
228
|
+
* redelivery source is ever changed to carry the ORIGINAL `receivedAt` into
|
|
229
|
+
* `enqueueInbox`, stale becomes reachable, the horizon must then be >= that
|
|
230
|
+
* source's maximum redelivery window, and `StaleInboxEnqueueError`'s
|
|
231
|
+
* withhold-not-ack handling becomes load-bearing; a breach surfaces LOUDLY (an
|
|
232
|
+
* old-`receivedAt` re-submission is refused at enqueue) rather than as silent
|
|
233
|
+
* double-processing.
|
|
207
234
|
*/
|
|
208
235
|
export const DEFAULT_CONSUMED_RETENTION_MS = 24 * 60 * 60 * 1000;
|
|
209
236
|
/**
|
|
@@ -221,6 +248,38 @@ export const DEFAULT_CONSUMED_RETENTION_MS = 24 * 60 * 60 * 1000;
|
|
|
221
248
|
* `<agentKey>` directory layer below the prefix).
|
|
222
249
|
*/
|
|
223
250
|
export const WORKFLOW_RUN_AGENT_STATE_PREFIX = "agent-state";
|
|
251
|
+
/**
|
|
252
|
+
* Conversational-mailbox subtree for the warm single-step agent. The
|
|
253
|
+
* substrate mailbox backing commits the agent's durable inbox under
|
|
254
|
+
* `mailbox/INBOX/` so the full message history replicates to the hub
|
|
255
|
+
* alongside the run state. The layout is:
|
|
256
|
+
*
|
|
257
|
+
* - `mailbox/INBOX/index.json` — the mailbox index. MUTABLE: the
|
|
258
|
+
* backing rewrites it on every flush, so it is exempt from the
|
|
259
|
+
* retained-blob byte-equality walk the `<uid>.eml` blobs are subject
|
|
260
|
+
* to. It must nonetheless PERSIST once it existed: dropping it drops
|
|
261
|
+
* the whole mailbox and resets uidValidity/uidNext on the next open.
|
|
262
|
+
* - `mailbox/INBOX/<uid>.eml` — one message per file, carrying the
|
|
263
|
+
* raw signed message bytes. `<uid>` is a decimal integer >= 1. A
|
|
264
|
+
* RETAINED `<uid>.eml` (present in both prior and prospective) is
|
|
265
|
+
* opaque and IMMUTABLE: it must reappear byte-identically, exactly
|
|
266
|
+
* like `runs/<runId>/blobs/`. A prior `<uid>.eml` may be ABSENT from
|
|
267
|
+
* the prospective tree — that is the warm agent expunging a message.
|
|
268
|
+
* The raw bytes are not lost: they stay reachable through the parent
|
|
269
|
+
* commit, and a `workflow-run` repo's objects are never GC'd (its
|
|
270
|
+
* kind is off the GC allow-list), so an expunged message survives in
|
|
271
|
+
* history for the life of the run repo. The audit trail rests on
|
|
272
|
+
* "these objects are never pruned", not on the live tree being
|
|
273
|
+
* monotonic.
|
|
274
|
+
*
|
|
275
|
+
* The only entries permitted under `mailbox/` are the `INBOX/`
|
|
276
|
+
* directory; the only entries permitted under `mailbox/INBOX/` are
|
|
277
|
+
* `index.json` and `<uid>.eml` message files. Anything else fails the
|
|
278
|
+
* push.
|
|
279
|
+
*/
|
|
280
|
+
export const WORKFLOW_RUN_MAILBOX_PREFIX = "mailbox";
|
|
281
|
+
export const WORKFLOW_RUN_MAILBOX_INBOX_DIR = "INBOX";
|
|
282
|
+
export const WORKFLOW_RUN_MAILBOX_INDEX_FILE = "index.json";
|
|
224
283
|
/**
|
|
225
284
|
* Allowed top-level entries in the prospective tree. Anything else
|
|
226
285
|
* fails the push. `control/` has no v1 use and stays absent.
|
|
@@ -229,8 +288,16 @@ const ALLOWED_TOP_LEVEL = new Set([
|
|
|
229
288
|
WORKFLOW_RUN_RUNS_PREFIX,
|
|
230
289
|
WORKFLOW_RUN_ADDRESSES_PREFIX,
|
|
231
290
|
WORKFLOW_RUN_AGENT_STATE_PREFIX,
|
|
291
|
+
WORKFLOW_RUN_MAILBOX_PREFIX,
|
|
232
292
|
WORKFLOW_RUN_GITIGNORE_PATH,
|
|
233
293
|
]);
|
|
294
|
+
/**
|
|
295
|
+
* Per-message filename shape for the `mailbox/INBOX/` subtree:
|
|
296
|
+
* `<uid>.eml`, where `<uid>` is a decimal integer >= 1 (no leading zero,
|
|
297
|
+
* never `0`). Pins the shape so a malformed message name fails the push
|
|
298
|
+
* at the boundary rather than landing silently.
|
|
299
|
+
*/
|
|
300
|
+
const MAILBOX_EML_FILENAME_RE = /^[1-9][0-9]*\.eml$/;
|
|
234
301
|
const CLAIM_CHECK_SUBDIRS = new Set([
|
|
235
302
|
WORKFLOW_RUN_INBOX_DIR,
|
|
236
303
|
WORKFLOW_RUN_PROCESSING_DIR,
|
|
@@ -238,6 +305,42 @@ const CLAIM_CHECK_SUBDIRS = new Set([
|
|
|
238
305
|
]);
|
|
239
306
|
/** Per-event filename shape: a decimal integer followed by `.json`. */
|
|
240
307
|
const EVENT_FILENAME_RE = /^(0|[1-9][0-9]*)\.json$/;
|
|
308
|
+
/**
|
|
309
|
+
* Parse the seq from a per-event log filename `<seq>.json` under
|
|
310
|
+
* `runs/<runId>/events/`. Returns the non-negative integer seq, or
|
|
311
|
+
* `null` when the name is not a legal per-event filename. This is the
|
|
312
|
+
* one place the filename shape is defined; every reader of the event log
|
|
313
|
+
* narrows names through it rather than re-encoding the regex. Callers
|
|
314
|
+
* decide what an illegal name means -- a foreign entry to skip, or a
|
|
315
|
+
* substrate-invariant violation to surface -- since `validatePush` is
|
|
316
|
+
* the authority that keeps illegal names from landing in the first place.
|
|
317
|
+
*/
|
|
318
|
+
export function parseEventSeq(filename) {
|
|
319
|
+
const match = EVENT_FILENAME_RE.exec(filename);
|
|
320
|
+
if (match === null)
|
|
321
|
+
return null;
|
|
322
|
+
const seqStr = match[1];
|
|
323
|
+
if (seqStr === undefined)
|
|
324
|
+
return null;
|
|
325
|
+
return Number.parseInt(seqStr, 10);
|
|
326
|
+
}
|
|
327
|
+
/**
|
|
328
|
+
* Narrow a per-event filename to its seq, throwing when it is illegal.
|
|
329
|
+
* A reader that enumerates the committed event log to act on its entries
|
|
330
|
+
* uses this rather than `parseEventSeq`: `validatePush` is the authority
|
|
331
|
+
* that keeps an illegal name from ever landing under
|
|
332
|
+
* `runs/<runId>/events/`, so a name that reaches a reader is corruption,
|
|
333
|
+
* and silently skipping it would drop an event from processing. `context`
|
|
334
|
+
* is the repo-root-relative blob path, surfaced in the error so the
|
|
335
|
+
* offending entry is identifiable.
|
|
336
|
+
*/
|
|
337
|
+
export function requireEventSeq(filename, context) {
|
|
338
|
+
const seq = parseEventSeq(filename);
|
|
339
|
+
if (seq === null) {
|
|
340
|
+
throw new Error(`event_filename_invalid: ${context}`);
|
|
341
|
+
}
|
|
342
|
+
return seq;
|
|
343
|
+
}
|
|
241
344
|
/**
|
|
242
345
|
* Per-blob filename shape for the `runs/<runId>/blobs/` subtree: a
|
|
243
346
|
* lowercase 64-character sha256 hex string. Pins the regex to the key
|
|
@@ -248,11 +351,39 @@ const EVENT_FILENAME_RE = /^(0|[1-9][0-9]*)\.json$/;
|
|
|
248
351
|
*/
|
|
249
352
|
const BLOB_FILENAME_RE = /^[0-9a-f]{64}$/;
|
|
250
353
|
/**
|
|
251
|
-
*
|
|
252
|
-
* `
|
|
253
|
-
*
|
|
254
|
-
*
|
|
255
|
-
*
|
|
354
|
+
* Per-mail-part filename shape for the
|
|
355
|
+
* `runs/<runId>/parts/<urlEncoded(messageId)>/` subtree:
|
|
356
|
+
* `<index>-<name>`, where `index` is the decimal position of the
|
|
357
|
+
* mail part within its inbound message and `name` is a non-empty
|
|
358
|
+
* (possibly sanitized) filename. The workflow-host ingest owns the exact
|
|
359
|
+
* encoding and sanitizes untrusted names to satisfy this shape; this regex
|
|
360
|
+
* pins the shape so a malformed name fails the push rather than landing
|
|
361
|
+
* silently. The bytes themselves are opaque and immutable, exactly like
|
|
362
|
+
* `blobs/`.
|
|
363
|
+
*/
|
|
364
|
+
const PART_FILENAME_RE = /^(0|[1-9][0-9]*)-(.+)$/;
|
|
365
|
+
/**
|
|
366
|
+
* Maximum byte length of a single mail part path component (the
|
|
367
|
+
* URL-encoded message segment, and each `<index>-<name>` filename).
|
|
368
|
+
* messageIds and mail part names arrive from untrusted inbound mail;
|
|
369
|
+
* an over-long RFC 5322 message-id URL-encodes past the filesystem's
|
|
370
|
+
* 255-byte component limit and would otherwise fail at disk-write time,
|
|
371
|
+
* downstream of validation. Reject it at the boundary instead.
|
|
372
|
+
*/
|
|
373
|
+
export const MAX_MAIL_PART_PATH_COMPONENT_BYTES = 255;
|
|
374
|
+
function mailPartComponentByteLength(component) {
|
|
375
|
+
return new TextEncoder().encode(component).length;
|
|
376
|
+
}
|
|
377
|
+
/**
|
|
378
|
+
* Entries the kind handler accepts under `runs/<runId>/`. The `events/`
|
|
379
|
+
* subtree carries the append-only event log; the `blobs/` subtree carries
|
|
380
|
+
* opaque, content-addressed step outputs the `BlobSubstrate` adapter spills
|
|
381
|
+
* there when a value exceeds the inline-encoding threshold; `grants.json`
|
|
382
|
+
* carries the run's authorization grants, delivered by the hub's
|
|
383
|
+
* `run.grants` frame ahead of the trigger and read back by the sidecar's
|
|
384
|
+
* `onRunStart` barrier. The grants file is a run-dir sibling of `events/`,
|
|
385
|
+
* not part of the event log, so the event-shape and blob-immutability walks
|
|
386
|
+
* treat it as inert.
|
|
256
387
|
*/
|
|
257
388
|
const RUN_DIR_ALLOWED_CHILDREN = new Set([
|
|
258
389
|
WORKFLOW_RUN_EVENTS_DIR,
|
|
@@ -260,6 +391,11 @@ const RUN_DIR_ALLOWED_CHILDREN = new Set([
|
|
|
260
391
|
// A terminated run's event log, sealed from the per-event `events/`
|
|
261
392
|
// files into one combined file by a compaction commit.
|
|
262
393
|
WORKFLOW_RUN_EVENTS_FILE,
|
|
394
|
+
WORKFLOW_RUN_GRANTS_FILE,
|
|
395
|
+
// Inbound-mail mail part bytes committed as real files. See
|
|
396
|
+
// WORKFLOW_RUN_PARTS_DIR; validated by enumerateRunParts and
|
|
397
|
+
// held immutable by the same prior-tree byte-equality walk as blobs.
|
|
398
|
+
WORKFLOW_RUN_PARTS_DIR,
|
|
263
399
|
]);
|
|
264
400
|
/**
|
|
265
401
|
* Filename shape for inbox and processing entries:
|
|
@@ -318,6 +454,10 @@ const ConsumedEnvelope = type({
|
|
|
318
454
|
store: "string > 0",
|
|
319
455
|
path: "string > 0",
|
|
320
456
|
},
|
|
457
|
+
"rejection?": {
|
|
458
|
+
code: "string > 0",
|
|
459
|
+
message: "string > 0",
|
|
460
|
+
},
|
|
321
461
|
"+": "ignore",
|
|
322
462
|
});
|
|
323
463
|
/**
|
|
@@ -331,26 +471,38 @@ const WatermarkEnvelope = type({
|
|
|
331
471
|
watermark: "number >= 0",
|
|
332
472
|
"+": "ignore",
|
|
333
473
|
});
|
|
474
|
+
const TERMINAL_EVENT_STATUS = new Map([
|
|
475
|
+
["RunCompleted", "completed"],
|
|
476
|
+
["RunFailed", "failed"],
|
|
477
|
+
["RunCancelled", "cancelled"],
|
|
478
|
+
]);
|
|
334
479
|
/**
|
|
335
|
-
*
|
|
336
|
-
*
|
|
337
|
-
*
|
|
338
|
-
*
|
|
339
|
-
* This set is a hand-rolled copy of the runtime's terminal-run vocabulary
|
|
340
|
-
* (`isTerminalRunPhase` in `@intx/workflow` state-machine `state.ts`,
|
|
341
|
-
* re-exported from the state-machine index and consumed by
|
|
342
|
-
* `transition.ts`), duplicated here because `@intx/hub-sessions` must not
|
|
343
|
-
* depend on `@intx/workflow`. It MUST stay in sync with that canonical
|
|
344
|
-
* definition:
|
|
345
|
-
* if the runtime adds or removes a terminal run phase, update this set too.
|
|
346
|
-
* Drift silently reopens the restore-time double-driver collision that
|
|
347
|
-
* `readOwnedMessageIds` (below) exists to prevent.
|
|
480
|
+
* Membership set of terminal event types, derived from
|
|
481
|
+
* `TERMINAL_EVENT_STATUS` so it always covers exactly the mapped types and
|
|
482
|
+
* the two cannot drift apart.
|
|
348
483
|
*/
|
|
349
|
-
const TERMINAL_EVENT_TYPES = new Set(
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
484
|
+
const TERMINAL_EVENT_TYPES = new Set(TERMINAL_EVENT_STATUS.keys());
|
|
485
|
+
/**
|
|
486
|
+
* Classify a workflow-run event type against the terminal-status vocabulary.
|
|
487
|
+
* `TERMINAL_EVENT_STATUS` is the sole authority (see above), so a type absent
|
|
488
|
+
* from it is by definition not terminal: no separate membership set is
|
|
489
|
+
* consulted and no "unmapped terminal type" case can arise.
|
|
490
|
+
*/
|
|
491
|
+
export function classifyTerminalEvent(eventType) {
|
|
492
|
+
const status = TERMINAL_EVENT_STATUS.get(eventType);
|
|
493
|
+
return status === undefined
|
|
494
|
+
? { terminal: false }
|
|
495
|
+
: { terminal: true, status };
|
|
496
|
+
}
|
|
497
|
+
/**
|
|
498
|
+
* True when a blob is absent from the prior tree -- i.e. this commit is the
|
|
499
|
+
* one that authored it. Consumers act only on a newly-added blob; a blob
|
|
500
|
+
* already present in the prior tree was carried forward unchanged by a
|
|
501
|
+
* compaction commit and must not be re-acted upon.
|
|
502
|
+
*/
|
|
503
|
+
async function blobIsNewlyAdded(blobPath, priorReadBlob) {
|
|
504
|
+
return (await priorReadBlob(blobPath)) === null;
|
|
505
|
+
}
|
|
354
506
|
/**
|
|
355
507
|
* Recognised CancelRequested origins. Mirrors the workflow package's
|
|
356
508
|
* `CANCEL_ORIGINS` vocabulary; inlined here so the substrate does
|
|
@@ -406,12 +558,12 @@ const SidecarPrincipal = type({
|
|
|
406
558
|
});
|
|
407
559
|
const WorkflowProcessPrincipal = type({
|
|
408
560
|
kind: "'workflow-process'",
|
|
409
|
-
|
|
561
|
+
anchorRunId: "string",
|
|
410
562
|
"runId?": "string",
|
|
411
563
|
});
|
|
412
564
|
const SupervisorPrincipal = type({
|
|
413
565
|
kind: "'supervisor'",
|
|
414
|
-
|
|
566
|
+
anchorRunId: "string",
|
|
415
567
|
});
|
|
416
568
|
/**
|
|
417
569
|
* Resolve the substrate's `changedPathPrefixes` into the set of run ids
|
|
@@ -484,7 +636,7 @@ async function enumerateEventBlobs(listDir, scopeRunIds) {
|
|
|
484
636
|
if (offender !== undefined) {
|
|
485
637
|
return {
|
|
486
638
|
ok: false,
|
|
487
|
-
reason: `run directory ${runDirPath} contains unexpected entry ${JSON.stringify(offender)}; only "${WORKFLOW_RUN_EVENTS_DIR}", "${WORKFLOW_RUN_BLOBS_DIR}", and "${
|
|
639
|
+
reason: `run directory ${runDirPath} contains unexpected entry ${JSON.stringify(offender)}; only "${WORKFLOW_RUN_EVENTS_DIR}", "${WORKFLOW_RUN_BLOBS_DIR}", "${WORKFLOW_RUN_EVENTS_FILE}", "${WORKFLOW_RUN_GRANTS_FILE}", and "${WORKFLOW_RUN_PARTS_DIR}" are allowed`,
|
|
488
640
|
};
|
|
489
641
|
}
|
|
490
642
|
const hasCombined = runChildren.includes(WORKFLOW_RUN_EVENTS_FILE);
|
|
@@ -500,6 +652,18 @@ async function enumerateEventBlobs(listDir, scopeRunIds) {
|
|
|
500
652
|
if (hasCombined)
|
|
501
653
|
continue;
|
|
502
654
|
if (!hasPerEvent) {
|
|
655
|
+
// The pre-first-event window: `grants.json` (the hub's `run.grants`
|
|
656
|
+
// frame writes grants ahead of the trigger) and `parts/` (the
|
|
657
|
+
// supervisor commits inbound-mail mail part bytes before firing the
|
|
658
|
+
// trigger) may both land before the child emits its first event. Carry
|
|
659
|
+
// such a run forward untouched -- there is no event log to enumerate
|
|
660
|
+
// yet, and the mail parts subtree is validated by its own walk. Any
|
|
661
|
+
// other events-less shape (e.g. a bare `blobs/` with no events) remains
|
|
662
|
+
// rejected below.
|
|
663
|
+
const nonPreEvent = runChildren.filter((c) => c !== WORKFLOW_RUN_GRANTS_FILE && c !== WORKFLOW_RUN_PARTS_DIR);
|
|
664
|
+
if (nonPreEvent.length === 0) {
|
|
665
|
+
continue;
|
|
666
|
+
}
|
|
503
667
|
return {
|
|
504
668
|
ok: false,
|
|
505
669
|
reason: `run directory ${runDirPath} is missing required "${WORKFLOW_RUN_EVENTS_DIR}" subdirectory`,
|
|
@@ -738,6 +902,159 @@ async function enumerateRunBlobs(listDir, scopeRunIds) {
|
|
|
738
902
|
}
|
|
739
903
|
return { ok: true, blobs: out };
|
|
740
904
|
}
|
|
905
|
+
/**
|
|
906
|
+
* Walk every `runs/<runId>/parts/<messageSegment>/` directory and
|
|
907
|
+
* validate each entry: the `<messageSegment>` is a URL-encoded messageId
|
|
908
|
+
* that must round-trip cleanly (the same canonical-encoding discipline the
|
|
909
|
+
* claim-check `addresses/` subtree enforces) and stay within the path-
|
|
910
|
+
* component byte cap, must be a directory rather than a dangling blob, and
|
|
911
|
+
* each filename matches the `<index>-<name>` shape within the same cap. The
|
|
912
|
+
* `parts/` subdirectory is optional: a run that never received a
|
|
913
|
+
* non-text inbound message never produces one. Returns the flat entry list
|
|
914
|
+
* so the caller can apply immutability checks against the prior tree,
|
|
915
|
+
* exactly as it does for blobs.
|
|
916
|
+
*/
|
|
917
|
+
async function enumerateRunParts(listDir, scopeRunIds) {
|
|
918
|
+
const out = [];
|
|
919
|
+
// See enumerateRunBlobs: a defined `scopeRunIds` walks only the commit's
|
|
920
|
+
// touched runs; an untouched run's mail parts are carried forward
|
|
921
|
+
// byte-identical and were validated when written.
|
|
922
|
+
const runIds = scopeRunIds === undefined
|
|
923
|
+
? await listDir(WORKFLOW_RUN_RUNS_PREFIX)
|
|
924
|
+
: Array.from(scopeRunIds);
|
|
925
|
+
for (const runId of runIds) {
|
|
926
|
+
const runDirPath = `${WORKFLOW_RUN_RUNS_PREFIX}/${runId}`;
|
|
927
|
+
const runChildren = await listDir(runDirPath);
|
|
928
|
+
if (!runChildren.includes(WORKFLOW_RUN_PARTS_DIR))
|
|
929
|
+
continue;
|
|
930
|
+
const partsDirPath = `${runDirPath}/${WORKFLOW_RUN_PARTS_DIR}`;
|
|
931
|
+
const messageSegments = await listDir(partsDirPath);
|
|
932
|
+
for (const messageSegment of messageSegments) {
|
|
933
|
+
const roundTrip = checkUrlSegmentRoundTrip(messageSegment);
|
|
934
|
+
if (!roundTrip.ok) {
|
|
935
|
+
return {
|
|
936
|
+
ok: false,
|
|
937
|
+
reason: `mail part message ${roundTrip.reason} under ${partsDirPath}`,
|
|
938
|
+
};
|
|
939
|
+
}
|
|
940
|
+
if (mailPartComponentByteLength(messageSegment) >
|
|
941
|
+
MAX_MAIL_PART_PATH_COMPONENT_BYTES) {
|
|
942
|
+
return {
|
|
943
|
+
ok: false,
|
|
944
|
+
reason: `mail part message segment ${JSON.stringify(messageSegment)} under ${partsDirPath} exceeds the ${String(MAX_MAIL_PART_PATH_COMPONENT_BYTES)}-byte path-component limit`,
|
|
945
|
+
};
|
|
946
|
+
}
|
|
947
|
+
const messageDirPath = `${partsDirPath}/${messageSegment}`;
|
|
948
|
+
const filenames = await listDir(messageDirPath);
|
|
949
|
+
// A message segment must be a directory carrying at least one
|
|
950
|
+
// mail part file. An empty listing means either a dangling blob
|
|
951
|
+
// committed directly at `parts/<segment>` (the substrate lists a
|
|
952
|
+
// blob path as empty, exactly as the agent-state walk detects) or an
|
|
953
|
+
// empty directory; both are rejected so untrusted inbound content has
|
|
954
|
+
// no silent-accept path.
|
|
955
|
+
if (filenames.length === 0) {
|
|
956
|
+
return {
|
|
957
|
+
ok: false,
|
|
958
|
+
reason: `mail part message segment ${JSON.stringify(messageSegment)} under ${partsDirPath} is not a directory carrying mail part files`,
|
|
959
|
+
};
|
|
960
|
+
}
|
|
961
|
+
for (const filename of filenames) {
|
|
962
|
+
if (!PART_FILENAME_RE.test(filename)) {
|
|
963
|
+
return {
|
|
964
|
+
ok: false,
|
|
965
|
+
reason: `mail part filename ${messageDirPath}/${filename} does not match <index>-<name>`,
|
|
966
|
+
};
|
|
967
|
+
}
|
|
968
|
+
if (mailPartComponentByteLength(filename) >
|
|
969
|
+
MAX_MAIL_PART_PATH_COMPONENT_BYTES) {
|
|
970
|
+
return {
|
|
971
|
+
ok: false,
|
|
972
|
+
reason: `mail part filename ${messageDirPath}/${filename} exceeds the ${String(MAX_MAIL_PART_PATH_COMPONENT_BYTES)}-byte path-component limit`,
|
|
973
|
+
};
|
|
974
|
+
}
|
|
975
|
+
// Each mail part entry must be a leaf blob, not a nested directory.
|
|
976
|
+
// The `<index>-<name>` shape is permissive (`.+`), so a directory
|
|
977
|
+
// named e.g. `0-foo` would otherwise pass and admit an arbitrarily
|
|
978
|
+
// nested subtree, breaking the one-file-per-mail-part invariant and
|
|
979
|
+
// -- on a pack-receive validation with no `listDirOids` -- driving the
|
|
980
|
+
// immutability resolver to `readBlob` a tree path and throw. A blob
|
|
981
|
+
// lists as empty; a directory lists its children.
|
|
982
|
+
const filePath = `${messageDirPath}/${filename}`;
|
|
983
|
+
if ((await listDir(filePath)).length > 0) {
|
|
984
|
+
return {
|
|
985
|
+
ok: false,
|
|
986
|
+
reason: `mail part ${filePath} is a directory; each mail part must be a single file`,
|
|
987
|
+
};
|
|
988
|
+
}
|
|
989
|
+
out.push({
|
|
990
|
+
runId,
|
|
991
|
+
messageSegment,
|
|
992
|
+
filename,
|
|
993
|
+
blobPath: filePath,
|
|
994
|
+
});
|
|
995
|
+
}
|
|
996
|
+
}
|
|
997
|
+
}
|
|
998
|
+
return { ok: true, parts: out };
|
|
999
|
+
}
|
|
1000
|
+
/**
|
|
1001
|
+
* Validate the per-run `parts/` subtree's immutability against the
|
|
1002
|
+
* prior tree, both directions, mirroring the blobs walk. Mail part files
|
|
1003
|
+
* are write-once. A path present in the prior tree must carry the same git
|
|
1004
|
+
* blob OID in the prospective tree: unlike `blobs/`, a mail part filename
|
|
1005
|
+
* is `<index>-<name>` (not a content hash), so the same path can carry
|
|
1006
|
+
* different bytes -- the OID compare is the load-bearing immutability guard,
|
|
1007
|
+
* and comparing the OID git already computed avoids re-reading tens of MB of
|
|
1008
|
+
* mail part bytes on every commit that merely touches the run. A prior path
|
|
1009
|
+
* must reappear (no deletion): run reclaim drops the whole `runs/<runId>/`
|
|
1010
|
+
* subtree outside `validatePush`, so a partial mail part deletion is always
|
|
1011
|
+
* a violation. Structural shape is enforced by `enumerateRunParts`.
|
|
1012
|
+
*/
|
|
1013
|
+
async function validateRunPartsSubtree(args) {
|
|
1014
|
+
const prospective = await enumerateRunParts(args.listDir, args.scopeRunIds);
|
|
1015
|
+
if (!prospective.ok)
|
|
1016
|
+
return prospective;
|
|
1017
|
+
const prior = await enumerateRunParts(args.priorListDir, args.scopeRunIds);
|
|
1018
|
+
if (!prior.ok) {
|
|
1019
|
+
return {
|
|
1020
|
+
ok: false,
|
|
1021
|
+
reason: `prior tree's mail parts subtree is structurally invalid: ${prior.reason}`,
|
|
1022
|
+
};
|
|
1023
|
+
}
|
|
1024
|
+
const prospectiveOid = makeListingOidResolver("prospective", args.listDirOids, async (p) => (await git.hashBlob({ object: await args.readBlob(p) })).oid);
|
|
1025
|
+
const priorOid = makeListingOidResolver("prior", args.priorListDirOids, async (p) => {
|
|
1026
|
+
const bytes = await args.priorReadBlob(p);
|
|
1027
|
+
if (bytes === null) {
|
|
1028
|
+
throw new Error(`mail parts: prior entry ${p} was enumerated but its bytes could not be read`);
|
|
1029
|
+
}
|
|
1030
|
+
return (await git.hashBlob({ object: bytes })).oid;
|
|
1031
|
+
});
|
|
1032
|
+
const prospectivePaths = new Set(prospective.parts.map((e) => e.blobPath));
|
|
1033
|
+
const priorPaths = new Set(prior.parts.map((e) => e.blobPath));
|
|
1034
|
+
for (const entry of prospective.parts) {
|
|
1035
|
+
if (!priorPaths.has(entry.blobPath))
|
|
1036
|
+
continue; // newly added
|
|
1037
|
+
const [next, before] = await Promise.all([
|
|
1038
|
+
prospectiveOid(entry.blobPath),
|
|
1039
|
+
priorOid(entry.blobPath),
|
|
1040
|
+
]);
|
|
1041
|
+
if (next !== before) {
|
|
1042
|
+
return {
|
|
1043
|
+
ok: false,
|
|
1044
|
+
reason: `mail part ${entry.blobPath} bytes diverge from the prior tree (blob OID ${next} vs ${before}); mail part files are immutable once written`,
|
|
1045
|
+
};
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
1048
|
+
for (const entry of prior.parts) {
|
|
1049
|
+
if (prospectivePaths.has(entry.blobPath))
|
|
1050
|
+
continue;
|
|
1051
|
+
return {
|
|
1052
|
+
ok: false,
|
|
1053
|
+
reason: `mail part ${entry.blobPath} present in the prior tree is missing from the prospective tree; mail part files are immutable once written`,
|
|
1054
|
+
};
|
|
1055
|
+
}
|
|
1056
|
+
return { ok: true };
|
|
1057
|
+
}
|
|
741
1058
|
/**
|
|
742
1059
|
* Enforce blob immutability via prior-tree byte equality. The blob
|
|
743
1060
|
* value itself is opaque bytes (no JSON envelope, no arktype
|
|
@@ -849,13 +1166,16 @@ async function checkPriorByteEquality(blobPath, readBlob, priorReadBlob) {
|
|
|
849
1166
|
return { ok: true };
|
|
850
1167
|
}
|
|
851
1168
|
/**
|
|
852
|
-
* Round-trip
|
|
853
|
-
*
|
|
854
|
-
*
|
|
855
|
-
*
|
|
856
|
-
*
|
|
1169
|
+
* Round-trip a URL-encoded path segment through decode then encode. A
|
|
1170
|
+
* divergence means the segment is not the canonical encoding of any
|
|
1171
|
+
* value, which would leave consumers guessing which encoding to use
|
|
1172
|
+
* when reading the subtree. Surface as a concrete rejection at push
|
|
1173
|
+
* time. Shared by every subtree that keys a directory by an
|
|
1174
|
+
* `encodeURIComponent`-encoded identity (`addresses/`, `agent-state/`,
|
|
1175
|
+
* and the per-run `parts/` message segment); callers prepend
|
|
1176
|
+
* their own subtree context to the returned reason.
|
|
857
1177
|
*/
|
|
858
|
-
function
|
|
1178
|
+
function checkUrlSegmentRoundTrip(segment) {
|
|
859
1179
|
let decoded;
|
|
860
1180
|
try {
|
|
861
1181
|
decoded = decodeURIComponent(segment);
|
|
@@ -863,14 +1183,14 @@ function checkAddressSegmentRoundTrip(segment) {
|
|
|
863
1183
|
catch (cause) {
|
|
864
1184
|
return {
|
|
865
1185
|
ok: false,
|
|
866
|
-
reason: `
|
|
1186
|
+
reason: `segment ${JSON.stringify(segment)} is not a valid URL-encoded string: ${cause instanceof Error ? cause.message : String(cause)}`,
|
|
867
1187
|
};
|
|
868
1188
|
}
|
|
869
1189
|
const reencoded = encodeURIComponent(decoded);
|
|
870
1190
|
if (reencoded !== segment) {
|
|
871
1191
|
return {
|
|
872
1192
|
ok: false,
|
|
873
|
-
reason: `
|
|
1193
|
+
reason: `segment ${JSON.stringify(segment)} does not round-trip URL-encoding (re-encoded as ${JSON.stringify(reencoded)})`,
|
|
874
1194
|
};
|
|
875
1195
|
}
|
|
876
1196
|
return { ok: true, decoded };
|
|
@@ -903,9 +1223,10 @@ async function enumerateClaimCheckBlobs(listDir, resolveConsumedOid) {
|
|
|
903
1223
|
const perAddress = new Map();
|
|
904
1224
|
const segments = await listDir(WORKFLOW_RUN_ADDRESSES_PREFIX);
|
|
905
1225
|
for (const segment of segments) {
|
|
906
|
-
const roundTrip =
|
|
907
|
-
if (!roundTrip.ok)
|
|
908
|
-
return roundTrip;
|
|
1226
|
+
const roundTrip = checkUrlSegmentRoundTrip(segment);
|
|
1227
|
+
if (!roundTrip.ok) {
|
|
1228
|
+
return { ok: false, reason: `address ${roundTrip.reason}` };
|
|
1229
|
+
}
|
|
909
1230
|
const addrDir = `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${segment}`;
|
|
910
1231
|
const children = await listDir(addrDir);
|
|
911
1232
|
for (const child of children) {
|
|
@@ -1619,11 +1940,11 @@ async function validateAgentStateSubtree(topLevelTreePaths, listDir) {
|
|
|
1619
1940
|
}
|
|
1620
1941
|
const segments = await listDir(WORKFLOW_RUN_AGENT_STATE_PREFIX);
|
|
1621
1942
|
for (const segment of segments) {
|
|
1622
|
-
const roundTrip =
|
|
1943
|
+
const roundTrip = checkUrlSegmentRoundTrip(segment);
|
|
1623
1944
|
if (!roundTrip.ok) {
|
|
1624
1945
|
return {
|
|
1625
1946
|
ok: false,
|
|
1626
|
-
reason: `agent-state
|
|
1947
|
+
reason: `agent-state ${roundTrip.reason}`,
|
|
1627
1948
|
};
|
|
1628
1949
|
}
|
|
1629
1950
|
// Reject a blob dangling directly at `agent-state/<segment>`: every
|
|
@@ -1640,6 +1961,149 @@ async function validateAgentStateSubtree(topLevelTreePaths, listDir) {
|
|
|
1640
1961
|
}
|
|
1641
1962
|
return { ok: true };
|
|
1642
1963
|
}
|
|
1964
|
+
/**
|
|
1965
|
+
* Enforce mailbox `<uid>.eml` immutability via prior-tree byte equality.
|
|
1966
|
+
* A message blob RETAINED from the prior tree (present in both) must carry
|
|
1967
|
+
* byte-identical contents in the prospective tree. A prior blob absent
|
|
1968
|
+
* from the prospective tree is a legal expunge and never reaches here
|
|
1969
|
+
* (the caller only checks retained paths). Mirrors the blob-immutability
|
|
1970
|
+
* discipline (`checkBlobPriorByteEquality`) with mailbox-specific wording.
|
|
1971
|
+
*/
|
|
1972
|
+
async function checkMailboxEmlPriorByteEquality(emlPath, readBlob, priorReadBlob) {
|
|
1973
|
+
const prior = await priorReadBlob(emlPath);
|
|
1974
|
+
if (prior === null)
|
|
1975
|
+
return { ok: true };
|
|
1976
|
+
const prospective = await readBlob(emlPath);
|
|
1977
|
+
if (prior.byteLength !== prospective.byteLength) {
|
|
1978
|
+
return {
|
|
1979
|
+
ok: false,
|
|
1980
|
+
reason: `mailbox message ${emlPath} bytes diverge from the prior tree (lengths ${String(prior.byteLength)} vs ${String(prospective.byteLength)}); a retained mailbox message is immutable`,
|
|
1981
|
+
};
|
|
1982
|
+
}
|
|
1983
|
+
for (let i = 0; i < prior.byteLength; i++) {
|
|
1984
|
+
if (prior[i] !== prospective[i]) {
|
|
1985
|
+
return {
|
|
1986
|
+
ok: false,
|
|
1987
|
+
reason: `mailbox message ${emlPath} bytes diverge from the prior tree at offset ${String(i)}; a retained mailbox message is immutable`,
|
|
1988
|
+
};
|
|
1989
|
+
}
|
|
1990
|
+
}
|
|
1991
|
+
return { ok: true };
|
|
1992
|
+
}
|
|
1993
|
+
/**
|
|
1994
|
+
* Walk the `mailbox/INBOX/` subtree and validate its shape. The only
|
|
1995
|
+
* entry permitted directly under `mailbox/` is the `INBOX/` directory;
|
|
1996
|
+
* the only entries permitted under `mailbox/INBOX/` are the mutable
|
|
1997
|
+
* `index.json` file and `<uid>.eml` message files. Each entry must be a
|
|
1998
|
+
* leaf blob rather than a nested directory (a blob lists as empty, a
|
|
1999
|
+
* directory lists its children -- the same discrimination the agent-state
|
|
2000
|
+
* and mail-parts walks use). Returns the flat set of `<uid>.eml` blob
|
|
2001
|
+
* paths so the caller can hold retained messages immutable against the
|
|
2002
|
+
* prior tree, plus `indexPresent` -- whether `index.json` exists under
|
|
2003
|
+
* the INBOX -- so the caller can enforce index continuity. An absent
|
|
2004
|
+
* `mailbox/` subtree lists as empty and contributes no message paths.
|
|
2005
|
+
*/
|
|
2006
|
+
async function enumerateMailboxInbox(listDir) {
|
|
2007
|
+
const emlPaths = new Set();
|
|
2008
|
+
let indexPresent = false;
|
|
2009
|
+
const mailboxChildren = await listDir(WORKFLOW_RUN_MAILBOX_PREFIX);
|
|
2010
|
+
if (mailboxChildren.length === 0)
|
|
2011
|
+
return { ok: true, emlPaths, indexPresent };
|
|
2012
|
+
for (const child of mailboxChildren) {
|
|
2013
|
+
if (child !== WORKFLOW_RUN_MAILBOX_INBOX_DIR) {
|
|
2014
|
+
return {
|
|
2015
|
+
ok: false,
|
|
2016
|
+
reason: `mailbox subtree contains unexpected entry ${JSON.stringify(child)} under ${WORKFLOW_RUN_MAILBOX_PREFIX}/; only "${WORKFLOW_RUN_MAILBOX_INBOX_DIR}" is allowed`,
|
|
2017
|
+
};
|
|
2018
|
+
}
|
|
2019
|
+
}
|
|
2020
|
+
const inboxPath = `${WORKFLOW_RUN_MAILBOX_PREFIX}/${WORKFLOW_RUN_MAILBOX_INBOX_DIR}`;
|
|
2021
|
+
const inboxEntries = await listDir(inboxPath);
|
|
2022
|
+
// A `mailbox/` top-level whose `INBOX` child carries no entries is a
|
|
2023
|
+
// dangling blob committed directly at `mailbox/INBOX` (a blob lists as
|
|
2024
|
+
// empty), not the required directory. Git never records an empty
|
|
2025
|
+
// directory, so a present-but-empty listing is always the blob case.
|
|
2026
|
+
if (inboxEntries.length === 0) {
|
|
2027
|
+
return {
|
|
2028
|
+
ok: false,
|
|
2029
|
+
reason: `mailbox ${inboxPath} is a blob, not a directory carrying "${WORKFLOW_RUN_MAILBOX_INDEX_FILE}" and <uid>.eml message files`,
|
|
2030
|
+
};
|
|
2031
|
+
}
|
|
2032
|
+
for (const entry of inboxEntries) {
|
|
2033
|
+
const entryPath = `${inboxPath}/${entry}`;
|
|
2034
|
+
if (entry === WORKFLOW_RUN_MAILBOX_INDEX_FILE) {
|
|
2035
|
+
if ((await listDir(entryPath)).length > 0) {
|
|
2036
|
+
return {
|
|
2037
|
+
ok: false,
|
|
2038
|
+
reason: `mailbox ${entryPath} is a directory; the mailbox index must be a single file`,
|
|
2039
|
+
};
|
|
2040
|
+
}
|
|
2041
|
+
indexPresent = true;
|
|
2042
|
+
continue;
|
|
2043
|
+
}
|
|
2044
|
+
if (!MAILBOX_EML_FILENAME_RE.test(entry)) {
|
|
2045
|
+
return {
|
|
2046
|
+
ok: false,
|
|
2047
|
+
reason: `mailbox entry ${entryPath} does not match "${WORKFLOW_RUN_MAILBOX_INDEX_FILE}" or <uid>.eml (uid a decimal integer >= 1)`,
|
|
2048
|
+
};
|
|
2049
|
+
}
|
|
2050
|
+
if ((await listDir(entryPath)).length > 0) {
|
|
2051
|
+
return {
|
|
2052
|
+
ok: false,
|
|
2053
|
+
reason: `mailbox message ${entryPath} is a directory; each message must be a single .eml file`,
|
|
2054
|
+
};
|
|
2055
|
+
}
|
|
2056
|
+
emlPaths.add(entryPath);
|
|
2057
|
+
}
|
|
2058
|
+
return { ok: true, emlPaths, indexPresent };
|
|
2059
|
+
}
|
|
2060
|
+
/**
|
|
2061
|
+
* Validate the `mailbox/INBOX/` subtree (design conversational-mailbox).
|
|
2062
|
+
* Enforces the subtree shape via `enumerateMailboxInbox`, then holds a
|
|
2063
|
+
* RETAINED `<uid>.eml` message blob byte-identical against the prior tree.
|
|
2064
|
+
* A prior `<uid>.eml` absent from the prospective tree is a legal expunge:
|
|
2065
|
+
* the warm agent physically removes a message from the live INBOX. The raw
|
|
2066
|
+
* bytes are not lost -- they stay reachable through the parent commit, and
|
|
2067
|
+
* a `workflow-run` repo's objects are never GC'd (its kind is excluded
|
|
2068
|
+
* from the GC allow-list; see `DEFAULT_GC_KINDS` in `agent-repo`), so the
|
|
2069
|
+
* expunged message survives in history for the life of the run repo. The
|
|
2070
|
+
* audit trail therefore rests on "these objects are never pruned", not on
|
|
2071
|
+
* the live tree being monotonic.
|
|
2072
|
+
*
|
|
2073
|
+
* The `index.json` entry is mutable, but must PERSIST once it existed: if
|
|
2074
|
+
* the prior tree carried an index and the prospective tree drops it, the
|
|
2075
|
+
* whole mailbox has vanished, which would reset `uidValidity` / `uidNext`
|
|
2076
|
+
* on the next open and force uid reuse from 1. That is rejected. A
|
|
2077
|
+
* well-behaved backing always rewrites `index.json` on flush, so the guard
|
|
2078
|
+
* fails no legitimate push. Mirrors the blob-immutability discipline the
|
|
2079
|
+
* `runs/<runId>/blobs/` subtree uses, minus the deletion direction.
|
|
2080
|
+
*/
|
|
2081
|
+
async function validateMailboxSubtree(listDir, readBlob, priorListDir, priorReadBlob) {
|
|
2082
|
+
const prospective = await enumerateMailboxInbox(listDir);
|
|
2083
|
+
if (!prospective.ok)
|
|
2084
|
+
return prospective;
|
|
2085
|
+
const prior = await enumerateMailboxInbox(priorListDir);
|
|
2086
|
+
if (!prior.ok) {
|
|
2087
|
+
return {
|
|
2088
|
+
ok: false,
|
|
2089
|
+
reason: `prior tree's mailbox subtree is structurally invalid: ${prior.reason}`,
|
|
2090
|
+
};
|
|
2091
|
+
}
|
|
2092
|
+
for (const emlPath of prospective.emlPaths) {
|
|
2093
|
+
if (!prior.emlPaths.has(emlPath))
|
|
2094
|
+
continue; // newly added
|
|
2095
|
+
const immutable = await checkMailboxEmlPriorByteEquality(emlPath, readBlob, priorReadBlob);
|
|
2096
|
+
if (!immutable.ok)
|
|
2097
|
+
return immutable;
|
|
2098
|
+
}
|
|
2099
|
+
if (prior.indexPresent && !prospective.indexPresent) {
|
|
2100
|
+
return {
|
|
2101
|
+
ok: false,
|
|
2102
|
+
reason: `mailbox ${WORKFLOW_RUN_MAILBOX_PREFIX}/${WORKFLOW_RUN_MAILBOX_INBOX_DIR}/${WORKFLOW_RUN_MAILBOX_INDEX_FILE} present in the prior tree is missing from the prospective tree; the mailbox index must persist so uidValidity/uidNext continuity holds across reopens`,
|
|
2103
|
+
};
|
|
2104
|
+
}
|
|
2105
|
+
return { ok: true };
|
|
2106
|
+
}
|
|
1643
2107
|
export const workflowRunKindHandler = {
|
|
1644
2108
|
kind: "workflow-run",
|
|
1645
2109
|
directoryPrefix: "workflow-runs",
|
|
@@ -1669,7 +2133,7 @@ export const workflowRunKindHandler = {
|
|
|
1669
2133
|
if (!ALLOWED_TOP_LEVEL.has(entry)) {
|
|
1670
2134
|
return {
|
|
1671
2135
|
ok: false,
|
|
1672
|
-
reason: `unexpected top-level entry ${JSON.stringify(entry)}; allowed: "${WORKFLOW_RUN_RUNS_PREFIX}", "${WORKFLOW_RUN_ADDRESSES_PREFIX}", "${WORKFLOW_RUN_AGENT_STATE_PREFIX}", "${WORKFLOW_RUN_GITIGNORE_PATH}"`,
|
|
2136
|
+
reason: `unexpected top-level entry ${JSON.stringify(entry)}; allowed: "${WORKFLOW_RUN_RUNS_PREFIX}", "${WORKFLOW_RUN_ADDRESSES_PREFIX}", "${WORKFLOW_RUN_AGENT_STATE_PREFIX}", "${WORKFLOW_RUN_MAILBOX_PREFIX}", "${WORKFLOW_RUN_GITIGNORE_PATH}"`,
|
|
1673
2137
|
};
|
|
1674
2138
|
}
|
|
1675
2139
|
}
|
|
@@ -1698,6 +2162,21 @@ export const workflowRunKindHandler = {
|
|
|
1698
2162
|
return claimCheck;
|
|
1699
2163
|
}
|
|
1700
2164
|
}
|
|
2165
|
+
const mailboxPresent = topLevelTreePaths.includes(WORKFLOW_RUN_MAILBOX_PREFIX) ||
|
|
2166
|
+
priorTopLevels.includes(WORKFLOW_RUN_MAILBOX_PREFIX);
|
|
2167
|
+
if (mailboxPresent) {
|
|
2168
|
+
// Enter mailbox validation when the prospective OR prior tree
|
|
2169
|
+
// carries a `mailbox/` subtree. A prospective tree that drops the
|
|
2170
|
+
// subtree while the prior tree held one must still go through the
|
|
2171
|
+
// walk so the index-continuity guard fires on the vanished
|
|
2172
|
+
// `index.json` (a message-only expunge is legal; dropping the whole
|
|
2173
|
+
// mailbox is not).
|
|
2174
|
+
const mailboxCheck = await validateMailboxSubtree(listDir, readBlob, priorListDir, priorReadBlob);
|
|
2175
|
+
if (!mailboxCheck.ok) {
|
|
2176
|
+
logger.debug `workflow-run validatePush rejected ${repoId.kind}/${repoId.id} on ${ref}: ${mailboxCheck.reason}`;
|
|
2177
|
+
return mailboxCheck;
|
|
2178
|
+
}
|
|
2179
|
+
}
|
|
1701
2180
|
const runsPresent = topLevelTreePaths.includes(WORKFLOW_RUN_RUNS_PREFIX) ||
|
|
1702
2181
|
priorTopLevels.includes(WORKFLOW_RUN_RUNS_PREFIX);
|
|
1703
2182
|
if (!runsPresent) {
|
|
@@ -1768,10 +2247,26 @@ export const workflowRunKindHandler = {
|
|
|
1768
2247
|
reason: `event ${entry.blobPath} CancelRequested origin must be a string`,
|
|
1769
2248
|
};
|
|
1770
2249
|
}
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
2250
|
+
// Enforce the cancel-origin principal only for a NEWLY-ADDED blob.
|
|
2251
|
+
// A CancelRequested's origin-vs-signer rule is a write-time
|
|
2252
|
+
// authorization: it belongs to the commit that authors the event. A
|
|
2253
|
+
// later commit that merely carries the event forward -- e.g. the
|
|
2254
|
+
// run's own workflow-process cascade write of CancelPropagated /
|
|
2255
|
+
// RunCancelled, which re-lists the whole events prefix -- must not be
|
|
2256
|
+
// rejected because the carried-forward cancel was authored under a
|
|
2257
|
+
// different (supervisor) signer. Re-checking it protects nothing: the
|
|
2258
|
+
// byte-equality check above already proves a carried-forward blob is
|
|
2259
|
+
// unchanged, and the deletion-direction check proves it cannot be
|
|
2260
|
+
// dropped. A tampered (byte-diverged) blob never reaches here --
|
|
2261
|
+
// checkPriorByteEquality rejects it first. Mirrors the newly-terminal
|
|
2262
|
+
// gate below, which likewise acts only on a blob absent from the
|
|
2263
|
+
// prior tree.
|
|
2264
|
+
if (await blobIsNewlyAdded(entry.blobPath, priorReadBlob)) {
|
|
2265
|
+
const principalCheck = checkCancelOriginPrincipal(entry.blobPath, origin, principal);
|
|
2266
|
+
if (!principalCheck.ok) {
|
|
2267
|
+
logger.debug `workflow-run validatePush rejected ${repoId.kind}/${repoId.id} on ${ref}: ${principalCheck.reason}`;
|
|
2268
|
+
return principalCheck;
|
|
2269
|
+
}
|
|
1775
2270
|
}
|
|
1776
2271
|
}
|
|
1777
2272
|
if (terminalSeq !== null) {
|
|
@@ -1780,7 +2275,8 @@ export const workflowRunKindHandler = {
|
|
|
1780
2275
|
reason: `run ${runId} has event at seq ${String(entry.filenameSeq)} after terminal ${terminalType} at seq ${String(terminalSeq)}`,
|
|
1781
2276
|
};
|
|
1782
2277
|
}
|
|
1783
|
-
|
|
2278
|
+
const classified = classifyTerminalEvent(parsed.parsed.body.type);
|
|
2279
|
+
if (classified.terminal) {
|
|
1784
2280
|
terminalSeq = entry.filenameSeq;
|
|
1785
2281
|
terminalType = parsed.parsed.body.type;
|
|
1786
2282
|
// Surface the run as newly terminal only when this commit is
|
|
@@ -1791,10 +2287,11 @@ export const workflowRunKindHandler = {
|
|
|
1791
2287
|
// terminal blob already present in the prior tree and emits no
|
|
1792
2288
|
// signal, so a downstream consumer keyed on the signal does
|
|
1793
2289
|
// not double-fire.
|
|
1794
|
-
if (
|
|
2290
|
+
if (await blobIsNewlyAdded(entry.blobPath, priorReadBlob)) {
|
|
1795
2291
|
const terminalBytes = await readBlob(entry.blobPath);
|
|
1796
2292
|
newlyTerminalRuns.push({
|
|
1797
2293
|
runId,
|
|
2294
|
+
status: classified.status,
|
|
1798
2295
|
terminalEventJson: new TextDecoder().decode(terminalBytes),
|
|
1799
2296
|
});
|
|
1800
2297
|
}
|
|
@@ -1868,6 +2365,19 @@ export const workflowRunKindHandler = {
|
|
|
1868
2365
|
reason: `blob ${b.blobPath} present in the prior tree is missing from the prospective tree; blob entries are immutable once written`,
|
|
1869
2366
|
};
|
|
1870
2367
|
}
|
|
2368
|
+
const partsCheck = await validateRunPartsSubtree({
|
|
2369
|
+
listDir,
|
|
2370
|
+
priorListDir,
|
|
2371
|
+
readBlob,
|
|
2372
|
+
priorReadBlob,
|
|
2373
|
+
listDirOids,
|
|
2374
|
+
priorListDirOids,
|
|
2375
|
+
scopeRunIds,
|
|
2376
|
+
});
|
|
2377
|
+
if (!partsCheck.ok) {
|
|
2378
|
+
logger.debug `workflow-run validatePush rejected ${repoId.kind}/${repoId.id} on ${ref}: ${partsCheck.reason}`;
|
|
2379
|
+
return partsCheck;
|
|
2380
|
+
}
|
|
1871
2381
|
return { ok: true, newlyTerminalRuns };
|
|
1872
2382
|
},
|
|
1873
2383
|
onRefUpdated() {
|
|
@@ -1893,10 +2403,10 @@ export const workflowRunAuthorize = (principal, repoId, ref, action) => {
|
|
|
1893
2403
|
reason: `workflow-process principal is malformed: ${parsed.summary}`,
|
|
1894
2404
|
};
|
|
1895
2405
|
}
|
|
1896
|
-
if (parsed.
|
|
2406
|
+
if (parsed.anchorRunId !== repoId.id) {
|
|
1897
2407
|
return {
|
|
1898
2408
|
allowed: false,
|
|
1899
|
-
reason: `workflow-process deployment ${parsed.
|
|
2409
|
+
reason: `workflow-process deployment ${parsed.anchorRunId} cannot access workflow-run ${repoId.id}`,
|
|
1900
2410
|
};
|
|
1901
2411
|
}
|
|
1902
2412
|
switch (action) {
|
|
@@ -1923,10 +2433,10 @@ export const workflowRunAuthorize = (principal, repoId, ref, action) => {
|
|
|
1923
2433
|
reason: `supervisor principal is malformed: ${parsed.summary}`,
|
|
1924
2434
|
};
|
|
1925
2435
|
}
|
|
1926
|
-
if (parsed.
|
|
2436
|
+
if (parsed.anchorRunId !== repoId.id) {
|
|
1927
2437
|
return {
|
|
1928
2438
|
allowed: false,
|
|
1929
|
-
reason: `supervisor deployment ${parsed.
|
|
2439
|
+
reason: `supervisor deployment ${parsed.anchorRunId} cannot access workflow-run ${repoId.id}`,
|
|
1930
2440
|
};
|
|
1931
2441
|
}
|
|
1932
2442
|
switch (action) {
|
|
@@ -1974,64 +2484,18 @@ export const workflowRunAuthorize = (principal, repoId, ref, action) => {
|
|
|
1974
2484
|
}
|
|
1975
2485
|
}
|
|
1976
2486
|
if (principal.kind === "user") {
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
const parsed = UserPrincipal(principal);
|
|
1985
|
-
if (parsed instanceof type.errors) {
|
|
1986
|
-
return {
|
|
1987
|
-
allowed: false,
|
|
1988
|
-
reason: `user principal is malformed: ${parsed.summary}`,
|
|
1989
|
-
};
|
|
1990
|
-
}
|
|
1991
|
-
if (!parsed.tokenClaims.actions.includes(action)) {
|
|
1992
|
-
return {
|
|
1993
|
-
allowed: false,
|
|
1994
|
-
reason: `token does not grant action ${action}`,
|
|
1995
|
-
};
|
|
1996
|
-
}
|
|
1997
|
-
// `ref === "*"` is the substrate's sentinel for the bulk read
|
|
1998
|
-
// performed by `listRefs`. Per-ref filtering is the advertise-refs
|
|
1999
|
-
// layer's responsibility, so the bulk read is gated on action and
|
|
2000
|
-
// expiry alone.
|
|
2001
|
-
if (ref !== "*" && !glob.match(parsed.tokenClaims.refPattern, ref)) {
|
|
2002
|
-
return {
|
|
2003
|
-
allowed: false,
|
|
2004
|
-
reason: `token refPattern ${parsed.tokenClaims.refPattern} does not match ${ref}`,
|
|
2005
|
-
};
|
|
2006
|
-
}
|
|
2007
|
-
if (Date.now() >= parsed.tokenClaims.expiresAt) {
|
|
2008
|
-
return {
|
|
2009
|
-
allowed: false,
|
|
2010
|
-
reason: `token expired at ${parsed.tokenClaims.expiresAt}`,
|
|
2011
|
-
};
|
|
2012
|
-
}
|
|
2013
|
-
const expectedResource = `workflow-run:${repoId.id}`;
|
|
2014
|
-
if (parsed.authz.resource !== expectedResource) {
|
|
2015
|
-
return {
|
|
2016
|
-
allowed: false,
|
|
2017
|
-
reason: `authz verdict resource ${parsed.authz.resource} does not match ${expectedResource}`,
|
|
2018
|
-
};
|
|
2019
|
-
}
|
|
2020
|
-
const expectedGrantVerb = repoActionToGrantVerb(action);
|
|
2021
|
-
if (parsed.authz.grantVerb !== expectedGrantVerb) {
|
|
2022
|
-
return {
|
|
2023
|
-
allowed: false,
|
|
2024
|
-
reason: `authz verdict grantVerb ${parsed.authz.grantVerb} does not match ${expectedGrantVerb}`,
|
|
2025
|
-
};
|
|
2026
|
-
}
|
|
2027
|
-
if (parsed.authz.effect === "allow") {
|
|
2028
|
-
return { allowed: true };
|
|
2029
|
-
}
|
|
2030
|
-
return {
|
|
2031
|
-
allowed: false,
|
|
2032
|
-
reason: `authz verdict denied for ${expectedResource} ${expectedGrantVerb}`,
|
|
2033
|
-
};
|
|
2487
|
+
return authorizeUserPrincipal({
|
|
2488
|
+
principal,
|
|
2489
|
+
repoId,
|
|
2490
|
+
ref,
|
|
2491
|
+
action,
|
|
2492
|
+
resourcePrefix: "workflow-run",
|
|
2493
|
+
});
|
|
2034
2494
|
}
|
|
2495
|
+
// Fail closed on any kind not handled above. The tenant-level
|
|
2496
|
+
// `workflow` principal kind (`@intx/types` principalKinds) is a
|
|
2497
|
+
// grant owner, not a workflow-run repo bearer, and never carries a
|
|
2498
|
+
// workflow-run push here -- so it is intentionally left denied.
|
|
2035
2499
|
return {
|
|
2036
2500
|
allowed: false,
|
|
2037
2501
|
reason: `unknown principal kind: ${principal.kind}`,
|
|
@@ -2191,6 +2655,49 @@ function parseWatermark(bytes, watermarkFull) {
|
|
|
2191
2655
|
}
|
|
2192
2656
|
return validated.watermark;
|
|
2193
2657
|
}
|
|
2658
|
+
/**
|
|
2659
|
+
* Internal signal thrown from the `enqueueInbox` merge callback when the
|
|
2660
|
+
* messageId is already present in a queue state. Caught at the `enqueueInbox`
|
|
2661
|
+
* boundary and turned into an `already-present` outcome; never escapes. It
|
|
2662
|
+
* carries the specific `reason` so the boundary maps it without re-deriving.
|
|
2663
|
+
*/
|
|
2664
|
+
class InboxEntryAlreadyPresent extends Error {
|
|
2665
|
+
reason;
|
|
2666
|
+
constructor(reason, message) {
|
|
2667
|
+
super(message);
|
|
2668
|
+
this.reason = reason;
|
|
2669
|
+
this.name = "InboxEntryAlreadyPresent";
|
|
2670
|
+
}
|
|
2671
|
+
}
|
|
2672
|
+
/**
|
|
2673
|
+
* Thrown by `enqueueInbox` when the inbound's `receivedAt` is strictly below
|
|
2674
|
+
* the address's retention watermark. This is refusal under UNCERTAINTY, not
|
|
2675
|
+
* proof of prior receipt: the consumed dedup entry that would rule out a
|
|
2676
|
+
* duplicate may have been pruned, so the substrate can no longer tell a
|
|
2677
|
+
* duplicate from a never-processed message and refuses rather than risk
|
|
2678
|
+
* reprocessing. A caller gating a durable-receipt ack MUST NOT acknowledge on
|
|
2679
|
+
* this -- acking an "I cannot tell" would terminally drop a message that was
|
|
2680
|
+
* never written. It is its own type (not a generic Error) so that a caller,
|
|
2681
|
+
* and monitoring, can surface it as a distinct loud signal rather than
|
|
2682
|
+
* blending it into ordinary I/O-failure noise.
|
|
2683
|
+
*
|
|
2684
|
+
* Structurally unreachable on the mail-inbound path today: `enqueueInbox` is
|
|
2685
|
+
* only ever called with a freshly stamped `receivedAt` (a redelivery
|
|
2686
|
+
* re-stamps `Date.now()` rather than carrying the original), and the watermark
|
|
2687
|
+
* only ever advances to at most `consumedAt - retentionHorizonMs <= now`, so a
|
|
2688
|
+
* fresh `receivedAt` sits a full horizon above it. The sole path that carries
|
|
2689
|
+
* an original (old) `receivedAt` back into the queue is
|
|
2690
|
+
* `replayProcessingToInbox`, which writes straight to `inbox/` and bypasses
|
|
2691
|
+
* this gate entirely. If any redelivery source is ever changed to carry the
|
|
2692
|
+
* original `receivedAt` into `enqueueInbox`, this becomes reachable and its
|
|
2693
|
+
* withhold-not-ack handling becomes load-bearing.
|
|
2694
|
+
*/
|
|
2695
|
+
export class StaleInboxEnqueueError extends Error {
|
|
2696
|
+
constructor(message) {
|
|
2697
|
+
super(message);
|
|
2698
|
+
this.name = "StaleInboxEnqueueError";
|
|
2699
|
+
}
|
|
2700
|
+
}
|
|
2194
2701
|
/**
|
|
2195
2702
|
* Append a new inbox entry for `address`. The merge callback reads
|
|
2196
2703
|
* the address subtree under the per-repo lock, augments the inbox
|
|
@@ -2219,46 +2726,63 @@ export async function enqueueInbox(store, principal, repoId, args) {
|
|
|
2219
2726
|
const inboxFname = `${inboxKey}.json`;
|
|
2220
2727
|
const consumedFname = `${args.messageId}.json`;
|
|
2221
2728
|
const messageIdSuffix = `-${args.messageId}.json`;
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
|
|
2244
|
-
|
|
2245
|
-
|
|
2246
|
-
|
|
2247
|
-
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
|
|
2729
|
+
// The already-present cases throw `InboxEntryAlreadyPresent` from the merge
|
|
2730
|
+
// callback and are caught here into an `already-present` outcome; the
|
|
2731
|
+
// stale-refusal throws `StaleInboxEnqueueError`, and a substrate/I/O failure
|
|
2732
|
+
// throws a generic error -- both of those propagate. The return/throw split
|
|
2733
|
+
// is deliberate and load-bearing: it is the ack/withhold boundary for a
|
|
2734
|
+
// caller gating a durable-receipt ack (return = safe to ack, throw =
|
|
2735
|
+
// withhold), so `stale` sits with I/O on the throw side, NOT with the
|
|
2736
|
+
// already-present cases (see `StaleInboxEnqueueError`).
|
|
2737
|
+
let commitSha;
|
|
2738
|
+
try {
|
|
2739
|
+
({ commitSha } = await store.writeTreeDelta(principal, repoId, ref, {
|
|
2740
|
+
changedPathPrefixes: new Set([addressPrefix(addressSegment)]),
|
|
2741
|
+
message: `enqueue inbox ${args.address} ${args.messageId}`,
|
|
2742
|
+
computeDelta: async (_parentCommitSha, prior) => {
|
|
2743
|
+
const listing = await readAddressListing(prior, addressSegment);
|
|
2744
|
+
// Refuse a definitively-stale enqueue: a message whose receivedAt
|
|
2745
|
+
// is strictly below the retention watermark could have had its
|
|
2746
|
+
// consumed/ dedup entry pruned, so a duplicate can no longer be
|
|
2747
|
+
// ruled out. Reject it LOUDLY rather than risk reprocessing. This
|
|
2748
|
+
// is the second half of the exactly-once guarantee: above the
|
|
2749
|
+
// watermark the consumed/ index is authoritative; below it, refuse.
|
|
2750
|
+
if (args.receivedAt < listing.watermark) {
|
|
2751
|
+
throw new StaleInboxEnqueueError(`claim_check_stale_enqueue: address ${args.address} message ${args.messageId} receivedAt ${String(args.receivedAt)} is below the retention watermark ${String(listing.watermark)}; its dedup entry may have been pruned, so it is refused as definitively-stale`);
|
|
2752
|
+
}
|
|
2753
|
+
if (listing.inbox.some((e) => e.name === inboxFname)) {
|
|
2754
|
+
throw new InboxEntryAlreadyPresent("duplicate", `claim_check_duplicate_inbox: ${newInboxPath} already exists`);
|
|
2755
|
+
}
|
|
2756
|
+
// consumed/ is keyed by messageId alone, so this is an exact
|
|
2757
|
+
// filename lookup against the dedup index.
|
|
2758
|
+
if (listing.consumed.some((e) => e.name === consumedFname)) {
|
|
2759
|
+
throw new InboxEntryAlreadyPresent("consumed", `claim_check_already_consumed: address ${args.address} message ${args.messageId} is already in the consumed dedup index`);
|
|
2760
|
+
}
|
|
2761
|
+
if (listing.processing.some((e) => e.name.endsWith(messageIdSuffix))) {
|
|
2762
|
+
throw new InboxEntryAlreadyPresent("processing", `claim_check_already_processing: address ${args.address} message ${args.messageId} is currently in processing`);
|
|
2763
|
+
}
|
|
2764
|
+
// Reject a second inbox entry for the same messageId at a
|
|
2765
|
+
// different receivedAt. The validatePush atomicity check also
|
|
2766
|
+
// catches this on the commit path, but surfacing it here gives the
|
|
2767
|
+
// caller a precise error and keeps the bad tree off the substrate.
|
|
2768
|
+
const inboxDup = listing.inbox.find((e) => e.name.endsWith(messageIdSuffix));
|
|
2769
|
+
if (inboxDup !== undefined) {
|
|
2770
|
+
throw new InboxEntryAlreadyPresent("already_inbox", `claim_check_already_inbox: address ${args.address} message ${args.messageId} is already in the inbox at ${inboxPath(addressSegment, inboxDup.name.slice(0, -".json".length))}`);
|
|
2771
|
+
}
|
|
2772
|
+
return {
|
|
2773
|
+
puts: { [newInboxPath]: utf8(JSON.stringify(envelope)) },
|
|
2774
|
+
deletes: [],
|
|
2775
|
+
};
|
|
2776
|
+
},
|
|
2777
|
+
}));
|
|
2778
|
+
}
|
|
2779
|
+
catch (err) {
|
|
2780
|
+
if (err instanceof InboxEntryAlreadyPresent) {
|
|
2781
|
+
return { outcome: "already-present", reason: err.reason };
|
|
2782
|
+
}
|
|
2783
|
+
throw err;
|
|
2784
|
+
}
|
|
2785
|
+
return { outcome: "enqueued", commitSha, inboxKey, envelope };
|
|
2262
2786
|
}
|
|
2263
2787
|
/**
|
|
2264
2788
|
* Move the FIFO-first inbox entry for `address` to processing.
|
|
@@ -2346,7 +2870,7 @@ export async function dequeueToProcessing(store, principal, repoId, address) {
|
|
|
2346
2870
|
* committed -- which is exactly when the supervisor forwards
|
|
2347
2871
|
* `trigger.fired` -- observes the processing entry. Reading the working tree (rather than walking the
|
|
2348
2872
|
* committed git tree) matches the workflow-process child's sibling
|
|
2349
|
-
*
|
|
2873
|
+
* read of `runs/<runId>/events/`. Because the
|
|
2350
2874
|
* read issues no commit it cannot race the supervisor's `markConsumed`
|
|
2351
2875
|
* write; it returns a point-in-time snapshot of the directory.
|
|
2352
2876
|
*/
|
|
@@ -2431,6 +2955,7 @@ export async function markConsumed(store, principal, repoId, args) {
|
|
|
2431
2955
|
runId: args.runId,
|
|
2432
2956
|
consumedAt: args.consumedAt,
|
|
2433
2957
|
mailAuditRef: processingEnvelope.mailAuditRef,
|
|
2958
|
+
...(args.rejection !== undefined ? { rejection: args.rejection } : {}),
|
|
2434
2959
|
};
|
|
2435
2960
|
consumedEnvelope = envelope;
|
|
2436
2961
|
// The watermark may only advance, and never past the entry this
|
|
@@ -2479,26 +3004,31 @@ export async function markConsumed(store, principal, repoId, args) {
|
|
|
2479
3004
|
};
|
|
2480
3005
|
}
|
|
2481
3006
|
/**
|
|
2482
|
-
*
|
|
2483
|
-
*
|
|
2484
|
-
* live run still owns. The caller (the supervisor's spawn-time replay)
|
|
2485
|
-
* feeds this into `replayProcessingToInbox`'s `ownedMessageIds` so a
|
|
2486
|
-
* parked run's message is not re-admitted to inbox and dispatched a
|
|
2487
|
-
* second time while the run is recovered by re-driving its durable log.
|
|
2488
|
-
* Without this, the re-drive AND the re-triggered fresh run both re-park
|
|
2489
|
-
* the same awaitSignal gate on the same runId, and the two concurrent
|
|
2490
|
-
* runtime bodies race to a corrupt terminal.
|
|
3007
|
+
* Walk `runs/` once and return the two boot-recovery inputs the supervisor's
|
|
3008
|
+
* spawn needs, from a single traversal of the working tree via `getRepoDir`:
|
|
2491
3009
|
*
|
|
2492
|
-
*
|
|
2493
|
-
*
|
|
2494
|
-
*
|
|
2495
|
-
*
|
|
2496
|
-
*
|
|
2497
|
-
*
|
|
2498
|
-
*
|
|
2499
|
-
*
|
|
3010
|
+
* - `ownedMessageIds`: the `consumedMessageId`s of NON-terminal runs -- the
|
|
3011
|
+
* messages a live run still owns. Spawn feeds this into
|
|
3012
|
+
* `replayProcessingToInbox`'s `ownedMessageIds` so a parked run's message is
|
|
3013
|
+
* not re-admitted to inbox and dispatched a second time while the run is
|
|
3014
|
+
* recovered by re-driving its durable log. Without this, the re-drive AND the
|
|
3015
|
+
* re-triggered fresh run both re-park the same awaitSignal gate on the same
|
|
3016
|
+
* runId, and the two concurrent runtime bodies race to a corrupt terminal.
|
|
3017
|
+
* - `pendingSealRunIds`: runs that are terminal but still in per-event form --
|
|
3018
|
+
* an interrupted fold left them unsealed. Spawn hands these to the recovery
|
|
3019
|
+
* sweep, which re-runs the idempotent fold. A terminal event is a *proposal*:
|
|
3020
|
+
* the authoritative decision is `compactRunEvents`, which independently
|
|
3021
|
+
* re-checks the run's max-seq event and no-ops a run that is not actually
|
|
3022
|
+
* terminal, so this scan may be loose.
|
|
3023
|
+
*
|
|
3024
|
+
* The working tree tracks the run-event ref (`refs/heads/main`); the
|
|
3025
|
+
* claim-check ref (`refs/heads/events`) cannot see it, which is why this lives
|
|
3026
|
+
* at the caller rather than inside `replayProcessingToInbox`'s single-ref
|
|
3027
|
+
* delta. A run whose log is sealed (combined `events.jsonl`, only permitted for
|
|
3028
|
+
* a terminated run) contributes to neither set; an absent `runs/` directory
|
|
3029
|
+
* yields empty results.
|
|
2500
3030
|
*/
|
|
2501
|
-
export async function
|
|
3031
|
+
export async function scanRunsForBoot(store, repoId) {
|
|
2502
3032
|
const fs = await import("node:fs/promises");
|
|
2503
3033
|
const path = await import("node:path");
|
|
2504
3034
|
const repoDir = store.getRepoDir(repoId);
|
|
@@ -2509,22 +3039,32 @@ export async function readOwnedMessageIds(store, repoId) {
|
|
|
2509
3039
|
}
|
|
2510
3040
|
catch (cause) {
|
|
2511
3041
|
if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") {
|
|
2512
|
-
return new Set();
|
|
3042
|
+
return { ownedMessageIds: new Set(), pendingSealRunIds: [] };
|
|
2513
3043
|
}
|
|
2514
3044
|
throw cause;
|
|
2515
3045
|
}
|
|
2516
3046
|
const owned = new Set();
|
|
3047
|
+
const pendingSealRunIds = [];
|
|
2517
3048
|
for (const runId of runIds) {
|
|
2518
3049
|
const runDir = path.join(runsDir, runId);
|
|
2519
3050
|
// A sealed run (combined events file) is terminal by the handler's
|
|
2520
3051
|
// own invariant -- only a terminated run is sealed -- so it owns
|
|
2521
|
-
// nothing. Its presence also means the per-event
|
|
3052
|
+
// nothing and is already folded. Its presence also means the per-event
|
|
3053
|
+
// directory is absent.
|
|
2522
3054
|
let sealed = false;
|
|
2523
3055
|
try {
|
|
2524
3056
|
await fs.access(path.join(runDir, WORKFLOW_RUN_EVENTS_FILE));
|
|
2525
3057
|
sealed = true;
|
|
2526
3058
|
}
|
|
2527
|
-
catch {
|
|
3059
|
+
catch (cause) {
|
|
3060
|
+
// ENOENT is the normal "not sealed" case. Any other stat error leaves
|
|
3061
|
+
// the run to fall through to the events-dir read below, which resolves
|
|
3062
|
+
// it, so this catch is benign; warn so the anomaly is still visible.
|
|
3063
|
+
if (!(cause instanceof Error) ||
|
|
3064
|
+
!("code" in cause) ||
|
|
3065
|
+
cause.code !== "ENOENT") {
|
|
3066
|
+
logger.warn `scanRunsForBoot: stat of the sealed-log file for run ${runId} failed: ${cause instanceof Error ? cause.message : String(cause)}`;
|
|
3067
|
+
}
|
|
2528
3068
|
sealed = false;
|
|
2529
3069
|
}
|
|
2530
3070
|
if (sealed)
|
|
@@ -2534,7 +3074,19 @@ export async function readOwnedMessageIds(store, repoId) {
|
|
|
2534
3074
|
try {
|
|
2535
3075
|
files = await fs.readdir(eventsDir);
|
|
2536
3076
|
}
|
|
2537
|
-
catch {
|
|
3077
|
+
catch (cause) {
|
|
3078
|
+
// ENOENT means the run has neither a sealed log nor a per-event
|
|
3079
|
+
// directory (grants may be staged before the first event); skip it.
|
|
3080
|
+
// A non-ENOENT error drops the run from BOTH result sets, and a live
|
|
3081
|
+
// run dropped from ownedMessageIds gets its message re-admitted and
|
|
3082
|
+
// dispatched a second time on the same runId -- the double-driver
|
|
3083
|
+
// corruption this scan exists to prevent. Surface it, but still skip:
|
|
3084
|
+
// aborting the whole boot scan over one run is worse.
|
|
3085
|
+
if (!(cause instanceof Error) ||
|
|
3086
|
+
!("code" in cause) ||
|
|
3087
|
+
cause.code !== "ENOENT") {
|
|
3088
|
+
logger.error `scanRunsForBoot: reading events for run ${runId} failed; skipping it may re-admit its message and start a second run on the same runId: ${cause instanceof Error ? cause.message : String(cause)}`;
|
|
3089
|
+
}
|
|
2538
3090
|
continue;
|
|
2539
3091
|
}
|
|
2540
3092
|
let terminal = false;
|
|
@@ -2546,7 +3098,17 @@ export async function readOwnedMessageIds(store, repoId) {
|
|
|
2546
3098
|
try {
|
|
2547
3099
|
parsed = JSON.parse(await fs.readFile(path.join(eventsDir, file), "utf8"));
|
|
2548
3100
|
}
|
|
2549
|
-
catch {
|
|
3101
|
+
catch (cause) {
|
|
3102
|
+
// A corrupt or unreadable event file drops this run's
|
|
3103
|
+
// classification: a missed RunStarted re-admits its message (a
|
|
3104
|
+
// second run on the same runId), a missed terminal event skips a
|
|
3105
|
+
// needed seal. Surface it, but skip the file rather than abort the
|
|
3106
|
+
// scan. ENOENT here is a benign race (the file vanished mid-scan).
|
|
3107
|
+
if (!(cause instanceof Error) ||
|
|
3108
|
+
!("code" in cause) ||
|
|
3109
|
+
cause.code !== "ENOENT") {
|
|
3110
|
+
logger.error `scanRunsForBoot: reading event ${file} for run ${runId} failed; skipping it may re-admit its message and start a second run on the same runId: ${cause instanceof Error ? cause.message : String(cause)}`;
|
|
3111
|
+
}
|
|
2550
3112
|
continue;
|
|
2551
3113
|
}
|
|
2552
3114
|
if (typeof parsed !== "object" ||
|
|
@@ -2567,12 +3129,136 @@ export async function readOwnedMessageIds(store, repoId) {
|
|
|
2567
3129
|
consumedMessageId = mid;
|
|
2568
3130
|
}
|
|
2569
3131
|
}
|
|
2570
|
-
if (terminal)
|
|
3132
|
+
if (terminal) {
|
|
3133
|
+
pendingSealRunIds.push(runId);
|
|
2571
3134
|
continue;
|
|
3135
|
+
}
|
|
2572
3136
|
if (consumedMessageId !== undefined)
|
|
2573
3137
|
owned.add(consumedMessageId);
|
|
2574
3138
|
}
|
|
2575
|
-
return owned;
|
|
3139
|
+
return { ownedMessageIds: owned, pendingSealRunIds };
|
|
3140
|
+
}
|
|
3141
|
+
/**
|
|
3142
|
+
* Classify a run's lifecycle from a read surface. The committed (git-object)
|
|
3143
|
+
* and working-tree (node:fs) readers share this core: a sealed combined log is
|
|
3144
|
+
* terminal; otherwise the latest per-event file decides terminal-vs-live, and a
|
|
3145
|
+
* run with no events is absent. The surface owns every read detail -- the
|
|
3146
|
+
* absent/ENOENT discrimination, the per-surface entry filter, and wrapping an
|
|
3147
|
+
* unreadable latest event as `workflow_run_event_unreadable` -- so this core
|
|
3148
|
+
* never sees a raw read error.
|
|
3149
|
+
*/
|
|
3150
|
+
async function classifyRunLifecycle(surface) {
|
|
3151
|
+
if (await surface.sealedLogPresent())
|
|
3152
|
+
return "terminal";
|
|
3153
|
+
const entries = await surface.listEventEntries();
|
|
3154
|
+
const latest = entries.reduce((candidate, entry) => candidate === undefined || entry.seq > candidate.seq ? entry : candidate, undefined);
|
|
3155
|
+
if (latest !== undefined) {
|
|
3156
|
+
const parsed = await surface.readEvent(latest);
|
|
3157
|
+
if (typeof parsed === "object" &&
|
|
3158
|
+
parsed !== null &&
|
|
3159
|
+
"type" in parsed &&
|
|
3160
|
+
typeof parsed.type === "string" &&
|
|
3161
|
+
TERMINAL_EVENT_TYPES.has(parsed.type)) {
|
|
3162
|
+
return "terminal";
|
|
3163
|
+
}
|
|
3164
|
+
}
|
|
3165
|
+
return entries.length === 0 ? "absent" : "live";
|
|
3166
|
+
}
|
|
3167
|
+
/** Read one run's lifecycle from a committed workflow-run tree. */
|
|
3168
|
+
export async function readCommittedWorkflowRunLifecycle(reads, runId) {
|
|
3169
|
+
if (reads === null)
|
|
3170
|
+
return "absent";
|
|
3171
|
+
const runPath = `${WORKFLOW_RUN_RUNS_PREFIX}/${runId}`;
|
|
3172
|
+
const eventsPath = `${runPath}/${WORKFLOW_RUN_EVENTS_DIR}`;
|
|
3173
|
+
return classifyRunLifecycle({
|
|
3174
|
+
async sealedLogPresent() {
|
|
3175
|
+
const runChildren = await reads.listDir(runPath);
|
|
3176
|
+
return runChildren.some((entry) => entry.type === "blob" && entry.name === WORKFLOW_RUN_EVENTS_FILE);
|
|
3177
|
+
},
|
|
3178
|
+
async listEventEntries() {
|
|
3179
|
+
const eventEntries = await reads.listDir(eventsPath);
|
|
3180
|
+
return eventEntries.flatMap((entry) => {
|
|
3181
|
+
if (entry.type !== "blob")
|
|
3182
|
+
return [];
|
|
3183
|
+
const seq = parseEventSeq(entry.name);
|
|
3184
|
+
return seq === null ? [] : [{ seq, name: entry.name, oid: entry.oid }];
|
|
3185
|
+
});
|
|
3186
|
+
},
|
|
3187
|
+
async readEvent(entry) {
|
|
3188
|
+
const eventPath = `${eventsPath}/${entry.name}`;
|
|
3189
|
+
try {
|
|
3190
|
+
return JSON.parse(new TextDecoder().decode(await reads.readBlobByOid(entry.oid)));
|
|
3191
|
+
}
|
|
3192
|
+
catch (cause) {
|
|
3193
|
+
throw new Error(`workflow_run_event_unreadable: ${eventPath}`, {
|
|
3194
|
+
cause,
|
|
3195
|
+
});
|
|
3196
|
+
}
|
|
3197
|
+
},
|
|
3198
|
+
});
|
|
3199
|
+
}
|
|
3200
|
+
/**
|
|
3201
|
+
* Read the durable lifecycle of one run from the workflow-run working tree.
|
|
3202
|
+
* `grants.json` alone is still an absent run: grants are staged before the
|
|
3203
|
+
* first trigger, while the first event is the durable proof that the run was
|
|
3204
|
+
* fired. A sealed event log is terminal by the kind handler's compaction
|
|
3205
|
+
* invariant.
|
|
3206
|
+
*
|
|
3207
|
+
* The supervisor uses this when in-memory cohort membership is empty. That
|
|
3208
|
+
* happens both for a genuinely new deployment and briefly during recovery,
|
|
3209
|
+
* so treating both states as "fire" would start a second driver for a live
|
|
3210
|
+
* log or reuse a terminal run.
|
|
3211
|
+
*/
|
|
3212
|
+
export async function readWorkflowRunLifecycle(store, repoId, runId) {
|
|
3213
|
+
const fs = await import("node:fs/promises");
|
|
3214
|
+
const path = await import("node:path");
|
|
3215
|
+
const runDir = path.join(store.getRepoDir(repoId), WORKFLOW_RUN_RUNS_PREFIX, runId);
|
|
3216
|
+
const eventsDir = path.join(runDir, WORKFLOW_RUN_EVENTS_DIR);
|
|
3217
|
+
return classifyRunLifecycle({
|
|
3218
|
+
async sealedLogPresent() {
|
|
3219
|
+
try {
|
|
3220
|
+
await fs.access(path.join(runDir, WORKFLOW_RUN_EVENTS_FILE));
|
|
3221
|
+
return true;
|
|
3222
|
+
}
|
|
3223
|
+
catch (cause) {
|
|
3224
|
+
if (!(cause instanceof Error) ||
|
|
3225
|
+
!("code" in cause) ||
|
|
3226
|
+
cause.code !== "ENOENT") {
|
|
3227
|
+
throw cause;
|
|
3228
|
+
}
|
|
3229
|
+
return false;
|
|
3230
|
+
}
|
|
3231
|
+
},
|
|
3232
|
+
async listEventEntries() {
|
|
3233
|
+
let files;
|
|
3234
|
+
try {
|
|
3235
|
+
files = await fs.readdir(eventsDir);
|
|
3236
|
+
}
|
|
3237
|
+
catch (cause) {
|
|
3238
|
+
if (cause instanceof Error &&
|
|
3239
|
+
"code" in cause &&
|
|
3240
|
+
cause.code === "ENOENT") {
|
|
3241
|
+
return [];
|
|
3242
|
+
}
|
|
3243
|
+
throw cause;
|
|
3244
|
+
}
|
|
3245
|
+
return files.flatMap((file) => {
|
|
3246
|
+
const seq = parseEventSeq(file);
|
|
3247
|
+
return seq === null ? [] : [{ seq, name: file }];
|
|
3248
|
+
});
|
|
3249
|
+
},
|
|
3250
|
+
async readEvent(entry) {
|
|
3251
|
+
const eventPath = path.join(eventsDir, entry.name);
|
|
3252
|
+
try {
|
|
3253
|
+
return JSON.parse(await fs.readFile(eventPath, "utf8"));
|
|
3254
|
+
}
|
|
3255
|
+
catch (cause) {
|
|
3256
|
+
throw new Error(`workflow_run_event_unreadable: ${eventPath}`, {
|
|
3257
|
+
cause,
|
|
3258
|
+
});
|
|
3259
|
+
}
|
|
3260
|
+
},
|
|
3261
|
+
});
|
|
2576
3262
|
}
|
|
2577
3263
|
/**
|
|
2578
3264
|
* Recovery path: move every processing entry at `address` back to
|