@intx/hub-sessions 0.1.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +176 -0
- package/README.md +82 -1
- package/dist/agent-repo.d.ts +93 -0
- package/dist/agent-repo.js +109 -0
- package/dist/agent-state-kind.d.ts +12 -0
- package/dist/agent-state-kind.js +189 -0
- package/dist/asset-service.d.ts +104 -0
- package/dist/asset-service.js +267 -0
- package/dist/committed-source-tree.d.ts +10 -0
- package/dist/committed-source-tree.js +35 -0
- package/dist/credential-push.d.ts +33 -0
- package/dist/credential-push.js +109 -0
- package/dist/event-collector-registry.d.ts +20 -0
- package/dist/event-collector-registry.js +115 -0
- package/dist/event-collector.d.ts +39 -0
- package/dist/event-collector.js +365 -0
- package/dist/hub-session-lookups.d.ts +135 -0
- package/dist/hub-session-lookups.js +663 -0
- package/dist/hub-session-orchestrator.d.ts +25 -0
- package/dist/hub-session-orchestrator.js +87 -0
- package/dist/index.d.ts +27 -0
- package/dist/index.js +24 -0
- package/dist/package-registry-kind.d.ts +70 -0
- package/dist/package-registry-kind.js +260 -0
- package/dist/repo-store/index.d.ts +4 -0
- package/dist/repo-store/index.js +3 -0
- package/dist/repo-store/store.d.ts +41 -0
- package/dist/repo-store/store.js +1829 -0
- package/dist/repo-store/subscribe-kind.d.ts +56 -0
- package/dist/repo-store/subscribe-kind.js +144 -0
- package/dist/repo-store/types.d.ts +571 -0
- package/dist/repo-store/types.js +42 -0
- package/dist/session-service.d.ts +416 -0
- package/dist/session-service.js +1191 -0
- package/dist/sidecar-allocation/contracts.d.ts +78 -0
- package/dist/sidecar-allocation/contracts.js +21 -0
- package/dist/sidecar-allocation/index.d.ts +4 -0
- package/dist/sidecar-allocation/index.js +3 -0
- package/dist/sidecar-allocation/placement-policy.d.ts +11 -0
- package/dist/sidecar-allocation/placement-policy.js +21 -0
- package/dist/sidecar-allocation/plugin-registry.d.ts +11 -0
- package/dist/sidecar-allocation/plugin-registry.js +37 -0
- package/dist/sidecar-allocation/reconciler.d.ts +42 -0
- package/dist/sidecar-allocation/reconciler.js +431 -0
- package/dist/skill-kind.d.ts +41 -0
- package/dist/skill-kind.js +292 -0
- package/dist/substrate.d.ts +8 -0
- package/dist/substrate.js +21 -0
- package/dist/workflow-allocation-service.d.ts +58 -0
- package/dist/workflow-allocation-service.js +239 -0
- package/dist/workflow-closure-resolution.d.ts +106 -0
- package/dist/workflow-closure-resolution.js +123 -0
- package/dist/workflow-definition-ensure.d.ts +24 -0
- package/dist/workflow-definition-ensure.js +75 -0
- package/dist/workflow-dispatch-service.d.ts +40 -0
- package/dist/workflow-dispatch-service.js +146 -0
- package/dist/workflow-dispatch-settlement.d.ts +29 -0
- package/dist/workflow-dispatch-settlement.js +140 -0
- package/dist/workflow-kind.d.ts +37 -0
- package/dist/workflow-kind.js +310 -0
- package/dist/workflow-probe-gate.d.ts +214 -0
- package/dist/workflow-probe-gate.js +207 -0
- package/dist/workflow-run-event-log.d.ts +21 -0
- package/dist/workflow-run-event-log.js +51 -0
- package/dist/workflow-run-kind.d.ts +440 -0
- package/dist/workflow-run-kind.js +2916 -0
- package/dist/workflow-run-reader.d.ts +47 -0
- package/dist/workflow-run-reader.js +153 -0
- package/dist/workflow-run-restore.d.ts +15 -0
- package/dist/workflow-run-restore.js +26 -0
- package/dist/workflow-source-closure.d.ts +35 -0
- package/dist/workflow-source-closure.js +342 -0
- package/dist/ws/index.d.ts +3 -0
- package/dist/ws/index.js +3 -0
- package/dist/ws/sidecar-events.d.ts +222 -0
- package/dist/ws/sidecar-events.js +72 -0
- package/dist/ws/sidecar-handler.d.ts +305 -0
- package/dist/ws/sidecar-handler.js +2537 -0
- package/dist/ws/sidecar-token-authenticator.d.ts +17 -0
- package/dist/ws/sidecar-token-authenticator.js +81 -0
- package/package.json +35 -12
- package/src/agent-repo.test.ts +0 -310
- package/src/agent-repo.ts +0 -165
- package/src/agent-state-kind.test.ts +0 -247
- package/src/agent-state-kind.ts +0 -204
- package/src/asset-service.test.ts +0 -540
- package/src/asset-service.ts +0 -378
- package/src/available-skills-stanza.test.ts +0 -87
- package/src/available-skills-stanza.ts +0 -47
- package/src/credential-push.ts +0 -65
- package/src/event-collector-registry.test.ts +0 -73
- package/src/event-collector-registry.ts +0 -171
- package/src/event-collector.test.ts +0 -1387
- package/src/event-collector.ts +0 -424
- package/src/hub-session-lookups.ts +0 -206
- package/src/hub-session-orchestrator.test.ts +0 -510
- package/src/hub-session-orchestrator.ts +0 -213
- package/src/index.ts +0 -78
- package/src/repo-store/index.ts +0 -15
- package/src/repo-store/store.test.ts +0 -1169
- package/src/repo-store/store.ts +0 -428
- package/src/repo-store/types.ts +0 -253
- package/src/session-service.test.ts +0 -895
- package/src/session-service.ts +0 -464
- package/src/skill-kind.test.ts +0 -599
- package/src/skill-kind.ts +0 -350
- package/src/ws/index.ts +0 -18
- package/src/ws/sidecar-events.test.ts +0 -96
- package/src/ws/sidecar-events.ts +0 -231
- package/src/ws/sidecar-handler.test.ts +0 -2217
- package/src/ws/sidecar-handler.ts +0 -1574
- package/tsconfig.json +0 -4
- package/tsconfig.tsbuildinfo +0 -1
|
@@ -0,0 +1,2916 @@
|
|
|
1
|
+
// KindHandler for the `workflow-run` repo kind.
|
|
2
|
+
//
|
|
3
|
+
// A workflow-run repo holds per-deployment runtime state for one or
|
|
4
|
+
// more in-flight workflow runs. `RepoId.id` is the owning deployment
|
|
5
|
+
// id. The repo's top-level layout is:
|
|
6
|
+
//
|
|
7
|
+
// - `runs/<runId>/events/<seq>.json` — per-run event log entries.
|
|
8
|
+
// Each entry is a JSON object whose body carries a `type`
|
|
9
|
+
// discriminator (the on-disk event vocabulary used by the
|
|
10
|
+
// workflow-run repo) and a `seq` field that matches the integer
|
|
11
|
+
// in the filename. Filenames are decimal integers ranging from
|
|
12
|
+
// `0` upward; the on-disk seq numbering owns the ordering and
|
|
13
|
+
// the per-blob `seq` field is the redundant cross-check.
|
|
14
|
+
// - `runs/<runId>/blobs/<sha256-hex>` — content-addressed step
|
|
15
|
+
// outputs the production `BlobSubstrate` adapter spills here when
|
|
16
|
+
// a value's JSON-stringified form exceeds the inline-encoding
|
|
17
|
+
// threshold. The filename is a lowercase 64-character sha256 hex
|
|
18
|
+
// string; the blob value is opaque bytes. Blobs are append-only
|
|
19
|
+
// and immutable: any blob present in the prior tree must carry
|
|
20
|
+
// byte-identical contents in the prospective tree.
|
|
21
|
+
// - `addresses/<urlEncoded(address)>/inbox/<receivedAt>-<messageId>.json`
|
|
22
|
+
// — pending inbound mail for the address, FIFO-ordered by the
|
|
23
|
+
// filename's parsed numeric `receivedAt` prefix (with a
|
|
24
|
+
// lexicographic messageId tiebreak). The filename keeps the
|
|
25
|
+
// decimal `<receivedAt>` form unpadded; the substrate sorts by
|
|
26
|
+
// parsed integer rather than string so the FIFO invariant holds
|
|
27
|
+
// for non-uniform digit widths (e.g. `99-…` precedes `100-…`).
|
|
28
|
+
// - `addresses/<urlEncoded(address)>/processing/<receivedAt>-<messageId>.json`
|
|
29
|
+
// — messages currently being handled. Same filename shape and
|
|
30
|
+
// JSON envelope as the inbox entry; a `dequeueToProcessing`
|
|
31
|
+
// commit atomically removes the inbox entry and adds the
|
|
32
|
+
// processing entry preserving the filename key.
|
|
33
|
+
// - `addresses/<urlEncoded(address)>/consumed/<messageId>.json` —
|
|
34
|
+
// dedup index keyed by messageId. A `markConsumed` commit
|
|
35
|
+
// atomically removes the matching processing entry and writes
|
|
36
|
+
// this dedup entry. The dedup index is bounded by a per-address
|
|
37
|
+
// retention watermark (see `watermark.json`): a `markConsumed`
|
|
38
|
+
// commit prunes consumed entries whose `receivedAt` falls below
|
|
39
|
+
// the watermark so the index reaches a bounded steady state
|
|
40
|
+
// instead of growing one entry per message forever.
|
|
41
|
+
// - `addresses/<urlEncoded(address)>/watermark.json` — the
|
|
42
|
+
// per-address retention watermark. Carries a single
|
|
43
|
+
// `receivedAt`-horizon value: the oldest `receivedAt` a consumed
|
|
44
|
+
// entry may still retain. The watermark only ever advances
|
|
45
|
+
// (monotonic non-decreasing). `enqueueInbox` rejects any inbound
|
|
46
|
+
// whose `receivedAt` is strictly below the watermark as
|
|
47
|
+
// definitively-stale (its dedup entry may have been pruned, so a
|
|
48
|
+
// duplicate cannot be ruled out -- refuse loudly rather than risk
|
|
49
|
+
// reprocessing). Above the watermark the `consumed/` index is
|
|
50
|
+
// authoritative; below it, refuse. The watermark advances only as
|
|
51
|
+
// the prune advances, both under the single writer, so the two
|
|
52
|
+
// never diverge. The stale-reject applies ONLY to fresh inbound at
|
|
53
|
+
// `enqueueInbox`; `replayProcessingToInbox` is intentionally exempt
|
|
54
|
+
// (a recovered in-flight `processing/` entry is already past dedup
|
|
55
|
+
// -- see that function's note).
|
|
56
|
+
// - `.gitignore` — supplied by the asset routes' genesis init body.
|
|
57
|
+
//
|
|
58
|
+
// The control-plane subtree (`control/...`) is not part of this
|
|
59
|
+
// commit's surface and has no v1 use case.
|
|
60
|
+
//
|
|
61
|
+
// Event-log invariants enforced at push:
|
|
62
|
+
// - Each event body's `seq` matches the integer in its filename.
|
|
63
|
+
// - Per-run event filenames are unique decimal integers (guaranteed
|
|
64
|
+
// by the tree shape) and validatePush verifies the body's `seq`
|
|
65
|
+
// field carries the same number, so the on-disk seq sequence and
|
|
66
|
+
// the per-blob seq cannot diverge.
|
|
67
|
+
// - Terminal-phase lock: once a run's events include a `RunCompleted`,
|
|
68
|
+
// `RunFailed`, or `RunCancelled` entry, no event with a strictly
|
|
69
|
+
// greater seq may appear for the same run.
|
|
70
|
+
// - Append-only via prior-tree byte comparison: every event blob
|
|
71
|
+
// that exists at the same path in the parent commit's tree must
|
|
72
|
+
// match the prospective blob byte-for-byte. Newly-added event
|
|
73
|
+
// paths (those absent from the prior tree) are accepted. The
|
|
74
|
+
// substrate exposes the prior tree via `priorReadBlob` /
|
|
75
|
+
// `priorListDir` on the validatePush args so the constraint is
|
|
76
|
+
// owned by this handler rather than relying on caller-layer
|
|
77
|
+
// discipline.
|
|
78
|
+
// - A `CancelRequested` event must carry an `origin` in the known
|
|
79
|
+
// set (`self`, `supervisor-drain`, `supervisor-operator`,
|
|
80
|
+
// `hub-admin`) and a non-empty `reason`.
|
|
81
|
+
// - Principal-vs-origin enforcement for `CancelRequested`: a
|
|
82
|
+
// `hub-admin` origin requires the signing principal to be `hub`;
|
|
83
|
+
// the other three
|
|
84
|
+
// origins (`self`, `supervisor-drain`, `supervisor-operator`)
|
|
85
|
+
// require the signing principal to be `supervisor` — the
|
|
86
|
+
// supervisor signs on the child's behalf for `self`, and signs
|
|
87
|
+
// for itself on the drain / operator cases. A principal that does
|
|
88
|
+
// not match the declared origin produces a rejection naming both
|
|
89
|
+
// sides so a misconfigured writer surfaces at the boundary
|
|
90
|
+
// rather than as a downstream mystery.
|
|
91
|
+
//
|
|
92
|
+
// Claim-check subtree invariants enforced at push:
|
|
93
|
+
// - The `<urlEncoded>` segment under `addresses/` must round-trip
|
|
94
|
+
// cleanly through `decodeURIComponent` followed by
|
|
95
|
+
// `encodeURIComponent`. A segment that does not round-trip is
|
|
96
|
+
// rejected so consumers can rely on a single canonical encoding.
|
|
97
|
+
// - The only entries permitted under an `addresses/<urlEncoded>/`
|
|
98
|
+
// subtree are the directories `inbox`, `processing`, and
|
|
99
|
+
// `consumed`, plus the single `watermark.json` file. Other names
|
|
100
|
+
// under an address fail the push.
|
|
101
|
+
// - Inbox and processing filenames must match
|
|
102
|
+
// `<receivedAt>-<messageId>.json` where `receivedAt` is a decimal
|
|
103
|
+
// epoch-ms integer. The body's `receivedAt` matches the filename
|
|
104
|
+
// `receivedAt` and the body's `messageId` matches the filename
|
|
105
|
+
// `messageId`. The body's `address` field must decode to the
|
|
106
|
+
// URL-encoded segment.
|
|
107
|
+
// - Consumed filenames must match `<messageId>.json`. The body's
|
|
108
|
+
// `messageId` matches the filename `messageId`. The body carries
|
|
109
|
+
// a `consumedBy` run id and the `receivedAt` of the original
|
|
110
|
+
// consume for audit.
|
|
111
|
+
// - Atomicity: a given `<messageId>` appears in at most one
|
|
112
|
+
// filename across `inbox`, `processing`, and `consumed` combined,
|
|
113
|
+
// per address per prospective commit. Two inbox entries with the
|
|
114
|
+
// same `<messageId>` but different `<receivedAt>` are rejected as
|
|
115
|
+
// a same-state collision; the cross-state check fires when the
|
|
116
|
+
// same messageId appears in inbox+processing, inbox+consumed, or
|
|
117
|
+
// processing+consumed.
|
|
118
|
+
// - `consumed/<messageId>.json` bytes are immutable: a prospective
|
|
119
|
+
// commit that mutates the bytes of a consumed entry RETAINED from
|
|
120
|
+
// the prior tree is rejected by the same prior-tree byte-equality
|
|
121
|
+
// guard used for run events. A retained consumed entry may be
|
|
122
|
+
// DELETED only as a watermark-consistent retention prune (see the
|
|
123
|
+
// watermark invariants below); any other deletion is rejected.
|
|
124
|
+
// - Retention prune (the bounded-`consumed/` contract): the consumed
|
|
125
|
+
// dedup index may shrink only by a watermark-passed prune. A
|
|
126
|
+
// consumed entry present in the prior tree may be absent from the
|
|
127
|
+
// prospective tree only when (a) its `receivedAt` is strictly
|
|
128
|
+
// below the prospective `watermark.json` value (you may prune only
|
|
129
|
+
// what the watermark passed) and (b) the watermark did not regress
|
|
130
|
+
// (`prospective watermark >= prior watermark`). A RETAINED entry is
|
|
131
|
+
// NOT required to sit at or above the watermark: a message consumed
|
|
132
|
+
// long after receipt (or one replayed back in-flight after a crash)
|
|
133
|
+
// may legitimately carry a below-watermark `receivedAt` and survive
|
|
134
|
+
// until a later commit prunes it. Retaining it gives only EXTRA
|
|
135
|
+
// dedup -- a re-submission at or above the watermark still hits the
|
|
136
|
+
// entry, one below it is stale-rejected at enqueue -- so it never
|
|
137
|
+
// weakens exactly-once.
|
|
138
|
+
// - Inbox→processing transition: a processing entry that is newly
|
|
139
|
+
// added (not present in the prior tree) must be backed by a
|
|
140
|
+
// matching inbox entry in the prior tree at the same
|
|
141
|
+
// `<receivedAt>-<messageId>.json` key. If the prior tree does
|
|
142
|
+
// not show that inbox entry the transition is rejected so a
|
|
143
|
+
// direct write into `processing/` cannot bypass the inbox.
|
|
144
|
+
// - Processing→consumed transition: a consumed entry that is
|
|
145
|
+
// newly added (not present in the prior tree) must be backed by
|
|
146
|
+
// a processing entry in the prior tree at the same address with
|
|
147
|
+
// the same messageId. The receivedAt and messageId carried in
|
|
148
|
+
// the prior processing envelope must equal the values carried in
|
|
149
|
+
// the new consumed envelope so the audit trail is unambiguous.
|
|
150
|
+
//
|
|
151
|
+
// Authz:
|
|
152
|
+
// - `hub` principal: full access.
|
|
153
|
+
// - `workflow-process` principal: read/write its own deployment's
|
|
154
|
+
// event log. The principal carries `{ anchorRunId, runId? }`;
|
|
155
|
+
// this handler verifies `repoId.id === anchorRunId`.
|
|
156
|
+
// - `supervisor` principal: read/write its own deployment's event
|
|
157
|
+
// log. The principal carries `{ anchorRunId }`; this handler
|
|
158
|
+
// verifies `repoId.id === anchorRunId`.
|
|
159
|
+
// - `sidecar` principal: read-only (createPack, resolveRef) for
|
|
160
|
+
// resume.
|
|
161
|
+
// - `user` principal: gated by bearer-token claims and the route
|
|
162
|
+
// layer's pre-resolved authz verdict, mirroring the convention
|
|
163
|
+
// used by the other kinds.
|
|
164
|
+
import fs from "node:fs";
|
|
165
|
+
import git from "isomorphic-git";
|
|
166
|
+
import { type } from "arktype";
|
|
167
|
+
import { getLogger } from "@intx/log";
|
|
168
|
+
import { glob, repoActionToGrantVerb } from "@intx/hub-common";
|
|
169
|
+
import { UserPrincipal, } from "./repo-store/index.js";
|
|
170
|
+
import { WORKFLOW_RUN_EVENTS_FILE, splitCombinedEventLog, encodeCombinedEventLog, } from "./workflow-run-event-log.js";
|
|
171
|
+
const logger = getLogger(["hub-sessions", "workflow-run-kind"]);
|
|
172
|
+
export const WORKFLOW_RUN_GITIGNORE_PATH = ".gitignore";
|
|
173
|
+
export const WORKFLOW_RUN_RUNS_PREFIX = "runs";
|
|
174
|
+
export const WORKFLOW_RUN_EVENTS_DIR = "events";
|
|
175
|
+
export const WORKFLOW_RUN_BLOBS_DIR = "blobs";
|
|
176
|
+
export const WORKFLOW_RUN_GRANTS_FILE = "grants.json";
|
|
177
|
+
export const WORKFLOW_RUN_ADDRESSES_PREFIX = "addresses";
|
|
178
|
+
export const WORKFLOW_RUN_CONTROL_PREFIX = "control";
|
|
179
|
+
export const WORKFLOW_RUN_INBOX_DIR = "inbox";
|
|
180
|
+
export const WORKFLOW_RUN_PROCESSING_DIR = "processing";
|
|
181
|
+
export const WORKFLOW_RUN_CONSUMED_DIR = "consumed";
|
|
182
|
+
/**
|
|
183
|
+
* Filename of the per-address retention watermark blob, a direct child
|
|
184
|
+
* of `addresses/<urlEncoded>/` (a file, not a directory). Carries the
|
|
185
|
+
* monotonic `receivedAt`-horizon below which consumed entries may be
|
|
186
|
+
* pruned and at-or-below which inbound enqueues are refused as stale.
|
|
187
|
+
*/
|
|
188
|
+
export const WORKFLOW_RUN_WATERMARK_FILE = "watermark.json";
|
|
189
|
+
/**
|
|
190
|
+
* Default retention horizon for the consumed dedup index, in
|
|
191
|
+
* milliseconds. The boot edge resolves the operator's
|
|
192
|
+
* `CONSUMED_RETENTION_MS` config to a concrete value and threads it
|
|
193
|
+
* into `markConsumed`; this default applies only when no operator
|
|
194
|
+
* value is supplied. 24 hours is the conservative default: long enough
|
|
195
|
+
* that a duplicate from a retrying upstream within a day is still
|
|
196
|
+
* deduped by a retained consumed entry, short enough that `consumed/`
|
|
197
|
+
* reaches a bounded steady state of one day's message volume.
|
|
198
|
+
*
|
|
199
|
+
* INVARIANT (operator-owned): the horizon must be >= the longest window in
|
|
200
|
+
* which the same `messageId` could legitimately be re-submitted and still must
|
|
201
|
+
* be caught as a duplicate. The hub now redelivers un-acked inbound mail
|
|
202
|
+
* (connected-window retry and reconnect-redelivery), so an at-least-once
|
|
203
|
+
* internal source DOES exist -- but the dedup guarantee against it does not
|
|
204
|
+
* rest on window arithmetic. It rests on a STRUCTURAL fact: `enqueueInbox` is
|
|
205
|
+
* only ever called with a freshly stamped `receivedAt` (a redelivery
|
|
206
|
+
* re-enters `onMailMessage` and re-stamps `Date.now()`, never carrying the
|
|
207
|
+
* original), and the watermark only ever advances to at most
|
|
208
|
+
* `consumedAt - retentionHorizonMs`, which is <= now, so a fresh `receivedAt`
|
|
209
|
+
* always sits a full horizon above the watermark and can never be stale-
|
|
210
|
+
* refused. A redelivery instead hits the `consumed/`/`processing/`/`inbox/`
|
|
211
|
+
* dedup index and is deduped there. The sole path that carries an original
|
|
212
|
+
* (old) `receivedAt` back into the queue is `replayProcessingToInbox`, which
|
|
213
|
+
* writes straight to `inbox/` and bypasses the stale gate entirely. So
|
|
214
|
+
* `claim_check_stale_enqueue` is unreachable via redelivery today. If any
|
|
215
|
+
* redelivery source is ever changed to carry the ORIGINAL `receivedAt` into
|
|
216
|
+
* `enqueueInbox`, stale becomes reachable, the horizon must then be >= that
|
|
217
|
+
* source's maximum redelivery window, and `StaleInboxEnqueueError`'s
|
|
218
|
+
* withhold-not-ack handling becomes load-bearing; a breach surfaces LOUDLY (an
|
|
219
|
+
* old-`receivedAt` re-submission is refused at enqueue) rather than as silent
|
|
220
|
+
* double-processing.
|
|
221
|
+
*/
|
|
222
|
+
export const DEFAULT_CONSUMED_RETENTION_MS = 24 * 60 * 60 * 1000;
|
|
223
|
+
/**
|
|
224
|
+
* Per-agent durable conversation-state subtree (design §3c). A
|
|
225
|
+
* long-lived single-step agent's multi-turn conversation context is
|
|
226
|
+
* committed under `agent-state/<agentKey>/...` so it survives child
|
|
227
|
+
* respawn: on respawn the rebuilt warm agent reads its prior
|
|
228
|
+
* conversation back from here before the resumed run replays.
|
|
229
|
+
*
|
|
230
|
+
* Unlike `runs/` (append-only events, immutable blobs) this subtree is
|
|
231
|
+
* MUTABLE: each run boundary overwrites the agent's conversation
|
|
232
|
+
* snapshot with the latest turns. It is therefore exempt from the
|
|
233
|
+
* append-only / deletion-direction walks `runs/` is subject to; the
|
|
234
|
+
* only push-time constraint is segment shape (a single round-trip-safe
|
|
235
|
+
* `<agentKey>` directory layer below the prefix).
|
|
236
|
+
*/
|
|
237
|
+
export const WORKFLOW_RUN_AGENT_STATE_PREFIX = "agent-state";
|
|
238
|
+
/**
|
|
239
|
+
* Allowed top-level entries in the prospective tree. Anything else
|
|
240
|
+
* fails the push. `control/` has no v1 use and stays absent.
|
|
241
|
+
*/
|
|
242
|
+
const ALLOWED_TOP_LEVEL = new Set([
|
|
243
|
+
WORKFLOW_RUN_RUNS_PREFIX,
|
|
244
|
+
WORKFLOW_RUN_ADDRESSES_PREFIX,
|
|
245
|
+
WORKFLOW_RUN_AGENT_STATE_PREFIX,
|
|
246
|
+
WORKFLOW_RUN_GITIGNORE_PATH,
|
|
247
|
+
]);
|
|
248
|
+
const CLAIM_CHECK_SUBDIRS = new Set([
|
|
249
|
+
WORKFLOW_RUN_INBOX_DIR,
|
|
250
|
+
WORKFLOW_RUN_PROCESSING_DIR,
|
|
251
|
+
WORKFLOW_RUN_CONSUMED_DIR,
|
|
252
|
+
]);
|
|
253
|
+
/** Per-event filename shape: a decimal integer followed by `.json`. */
|
|
254
|
+
const EVENT_FILENAME_RE = /^(0|[1-9][0-9]*)\.json$/;
|
|
255
|
+
/**
|
|
256
|
+
* Parse the seq from a per-event log filename `<seq>.json` under
|
|
257
|
+
* `runs/<runId>/events/`. Returns the non-negative integer seq, or
|
|
258
|
+
* `null` when the name is not a legal per-event filename. This is the
|
|
259
|
+
* one place the filename shape is defined; every reader of the event log
|
|
260
|
+
* narrows names through it rather than re-encoding the regex. Callers
|
|
261
|
+
* decide what an illegal name means -- a foreign entry to skip, or a
|
|
262
|
+
* substrate-invariant violation to surface -- since `validatePush` is
|
|
263
|
+
* the authority that keeps illegal names from landing in the first place.
|
|
264
|
+
*/
|
|
265
|
+
export function parseEventSeq(filename) {
|
|
266
|
+
const match = EVENT_FILENAME_RE.exec(filename);
|
|
267
|
+
if (match === null)
|
|
268
|
+
return null;
|
|
269
|
+
const seqStr = match[1];
|
|
270
|
+
if (seqStr === undefined)
|
|
271
|
+
return null;
|
|
272
|
+
return Number.parseInt(seqStr, 10);
|
|
273
|
+
}
|
|
274
|
+
/**
|
|
275
|
+
* Narrow a per-event filename to its seq, throwing when it is illegal.
|
|
276
|
+
* A reader that enumerates the committed event log to act on its entries
|
|
277
|
+
* uses this rather than `parseEventSeq`: `validatePush` is the authority
|
|
278
|
+
* that keeps an illegal name from ever landing under
|
|
279
|
+
* `runs/<runId>/events/`, so a name that reaches a reader is corruption,
|
|
280
|
+
* and silently skipping it would drop an event from processing. `context`
|
|
281
|
+
* is the repo-root-relative blob path, surfaced in the error so the
|
|
282
|
+
* offending entry is identifiable.
|
|
283
|
+
*/
|
|
284
|
+
export function requireEventSeq(filename, context) {
|
|
285
|
+
const seq = parseEventSeq(filename);
|
|
286
|
+
if (seq === null) {
|
|
287
|
+
throw new Error(`event_filename_invalid: ${context}`);
|
|
288
|
+
}
|
|
289
|
+
return seq;
|
|
290
|
+
}
|
|
291
|
+
/**
|
|
292
|
+
* Per-blob filename shape for the `runs/<runId>/blobs/` subtree: a
|
|
293
|
+
* lowercase 64-character sha256 hex string. Pins the regex to the key
|
|
294
|
+
* the production `BlobSubstrate` adapter computes via `sha256Hex` so a
|
|
295
|
+
* non-canonical key (uppercase hex, truncated digest, alternate
|
|
296
|
+
* encoding) fails the push at the boundary rather than landing
|
|
297
|
+
* silently.
|
|
298
|
+
*/
|
|
299
|
+
const BLOB_FILENAME_RE = /^[0-9a-f]{64}$/;
|
|
300
|
+
/**
|
|
301
|
+
* Entries the kind handler accepts under `runs/<runId>/`. The `events/`
|
|
302
|
+
* subtree carries the append-only event log; the `blobs/` subtree carries
|
|
303
|
+
* opaque, content-addressed step outputs the `BlobSubstrate` adapter spills
|
|
304
|
+
* there when a value exceeds the inline-encoding threshold; `grants.json`
|
|
305
|
+
* carries the run's authorization grants, delivered by the hub's
|
|
306
|
+
* `run.grants` frame ahead of the trigger and read back by the sidecar's
|
|
307
|
+
* `onRunStart` barrier. The grants file is a run-dir sibling of `events/`,
|
|
308
|
+
* not part of the event log, so the event-shape and blob-immutability walks
|
|
309
|
+
* treat it as inert.
|
|
310
|
+
*/
|
|
311
|
+
const RUN_DIR_ALLOWED_CHILDREN = new Set([
|
|
312
|
+
WORKFLOW_RUN_EVENTS_DIR,
|
|
313
|
+
WORKFLOW_RUN_BLOBS_DIR,
|
|
314
|
+
// A terminated run's event log, sealed from the per-event `events/`
|
|
315
|
+
// files into one combined file by a compaction commit.
|
|
316
|
+
WORKFLOW_RUN_EVENTS_FILE,
|
|
317
|
+
WORKFLOW_RUN_GRANTS_FILE,
|
|
318
|
+
]);
|
|
319
|
+
/**
|
|
320
|
+
* Filename shape for inbox and processing entries:
|
|
321
|
+
* `<receivedAt>-<messageId>.json`. `receivedAt` is a decimal integer
|
|
322
|
+
* (epoch ms); `messageId` is captured as the rest of the basename and
|
|
323
|
+
* is validated separately against the body's `messageId`.
|
|
324
|
+
*/
|
|
325
|
+
const QUEUE_FILENAME_RE = /^(0|[1-9][0-9]*)-(.+)\.json$/;
|
|
326
|
+
/** Filename shape for consumed entries: `<messageId>.json`. */
|
|
327
|
+
const CONSUMED_FILENAME_RE = /^(.+)\.json$/;
|
|
328
|
+
/**
|
|
329
|
+
* JSON envelope carried by inbox and processing entries. Keys:
|
|
330
|
+
* - `messageId`: dedup key for the inbound message.
|
|
331
|
+
* - `receivedAt`: epoch-ms timestamp the reactor accepted the
|
|
332
|
+
* message; sortable FIFO key prefix.
|
|
333
|
+
* - `address`: decoded canonical address (not URL-encoded).
|
|
334
|
+
* - `mailAuditRef`: pointer to the raw mail bytes in the mail-audit
|
|
335
|
+
* store. For the in-process single-agent path a separate
|
|
336
|
+
* `MailAuditStore` holds the authoritative bytes and this ref joins
|
|
337
|
+
* onto it.
|
|
338
|
+
* - `rawMessage`: base64 of the inbound mail's raw MIME bytes,
|
|
339
|
+
* inlined so the workflow-process child can read its step input by
|
|
340
|
+
* messageId at `trigger.fired` time. The supervisor is the sole
|
|
341
|
+
* mail owner under the unified-execution host (§3a); it has no
|
|
342
|
+
* separate durable byte store the child can read, so the bytes ride
|
|
343
|
+
* the claim-check envelope itself. Present whenever the supervisor
|
|
344
|
+
* enqueued the entry; omitted by callers that only stamp the audit
|
|
345
|
+
* ref. The bytes survive the inbox→processing transition verbatim
|
|
346
|
+
* (the dequeue copies the entry bytes), so a `trigger.fired` for a
|
|
347
|
+
* processing entry can always recover the input.
|
|
348
|
+
*/
|
|
349
|
+
const ClaimCheckEnvelope = type({
|
|
350
|
+
messageId: "string > 0",
|
|
351
|
+
receivedAt: "number >= 0",
|
|
352
|
+
address: "string > 0",
|
|
353
|
+
mailAuditRef: {
|
|
354
|
+
store: "string > 0",
|
|
355
|
+
path: "string > 0",
|
|
356
|
+
},
|
|
357
|
+
"rawMessage?": "string > 0",
|
|
358
|
+
"+": "ignore",
|
|
359
|
+
});
|
|
360
|
+
/**
|
|
361
|
+
* JSON envelope carried by consumed entries. The consumed entry is the
|
|
362
|
+
* canonical dedup index keyed by messageId; the envelope preserves
|
|
363
|
+
* the originating receivedAt for audit and carries the runId that
|
|
364
|
+
* consumed the message.
|
|
365
|
+
*/
|
|
366
|
+
const ConsumedEnvelope = type({
|
|
367
|
+
messageId: "string > 0",
|
|
368
|
+
receivedAt: "number >= 0",
|
|
369
|
+
address: "string > 0",
|
|
370
|
+
runId: "string > 0",
|
|
371
|
+
consumedAt: "number >= 0",
|
|
372
|
+
mailAuditRef: {
|
|
373
|
+
store: "string > 0",
|
|
374
|
+
path: "string > 0",
|
|
375
|
+
},
|
|
376
|
+
"rejection?": {
|
|
377
|
+
code: "string > 0",
|
|
378
|
+
message: "string > 0",
|
|
379
|
+
},
|
|
380
|
+
"+": "ignore",
|
|
381
|
+
});
|
|
382
|
+
/**
|
|
383
|
+
* JSON envelope carried by the per-address `watermark.json` blob. The
|
|
384
|
+
* `watermark` is a `receivedAt` horizon (epoch ms): the oldest
|
|
385
|
+
* `receivedAt` a consumed entry may still retain. It only ever
|
|
386
|
+
* advances. A retention prune drops consumed entries strictly below
|
|
387
|
+
* it; `enqueueInbox` refuses any inbound strictly below it.
|
|
388
|
+
*/
|
|
389
|
+
const WatermarkEnvelope = type({
|
|
390
|
+
watermark: "number >= 0",
|
|
391
|
+
"+": "ignore",
|
|
392
|
+
});
|
|
393
|
+
/**
|
|
394
|
+
* Terminal event discriminators mapped to the `workflow_run.status` value
|
|
395
|
+
* each settles the run into. A run whose log contains an entry with one of
|
|
396
|
+
* these `type` values must not receive any event with a strictly greater
|
|
397
|
+
* seq.
|
|
398
|
+
*
|
|
399
|
+
* This map is a hand-rolled copy of the runtime's terminal-run vocabulary
|
|
400
|
+
* (`isTerminalRunPhase` in `@intx/workflow` state-machine `state.ts`,
|
|
401
|
+
* re-exported from the state-machine index and consumed by `transition.ts`),
|
|
402
|
+
* duplicated here because `@intx/hub-sessions` must not depend on
|
|
403
|
+
* `@intx/workflow`. It is the sole authority for that vocabulary and MUST
|
|
404
|
+
* stay in sync with the canonical runtime definition:
|
|
405
|
+
* if the runtime adds or removes a terminal run phase, update this map too.
|
|
406
|
+
* Drift silently reopens the restore-time double-driver collision that
|
|
407
|
+
* `readOwnedMessageIds` (below) exists to prevent.
|
|
408
|
+
*/
|
|
409
|
+
const TERMINAL_EVENT_STATUS = new Map([
|
|
410
|
+
["RunCompleted", "completed"],
|
|
411
|
+
["RunFailed", "failed"],
|
|
412
|
+
["RunCancelled", "cancelled"],
|
|
413
|
+
]);
|
|
414
|
+
/**
|
|
415
|
+
* Membership set of terminal event types, derived from
|
|
416
|
+
* `TERMINAL_EVENT_STATUS` so it always covers exactly the mapped types and
|
|
417
|
+
* the two cannot drift apart.
|
|
418
|
+
*/
|
|
419
|
+
const TERMINAL_EVENT_TYPES = new Set(TERMINAL_EVENT_STATUS.keys());
|
|
420
|
+
/**
|
|
421
|
+
* Recognised CancelRequested origins. Mirrors the workflow package's
|
|
422
|
+
* `CANCEL_ORIGINS` vocabulary; inlined here so the substrate does
|
|
423
|
+
* not depend on `@intx/workflow`.
|
|
424
|
+
*/
|
|
425
|
+
const CANCEL_REQUESTED_ORIGINS = new Set([
|
|
426
|
+
"self",
|
|
427
|
+
"supervisor-drain",
|
|
428
|
+
"supervisor-operator",
|
|
429
|
+
"hub-admin",
|
|
430
|
+
]);
|
|
431
|
+
/**
|
|
432
|
+
* Per-origin signing-principal kind. `hub-admin` is the only origin
|
|
433
|
+
* a `hub` principal may mint; the other three originate inside the
|
|
434
|
+
* supervisor's trust boundary (the supervisor signs `self` on behalf
|
|
435
|
+
* of the workflow-process since the child has no asymmetric keypair,
|
|
436
|
+
* and signs the `supervisor-drain` / `supervisor-operator` audit-
|
|
437
|
+
* distinction
|
|
438
|
+
* cases for itself). Lookup misses fail the push.
|
|
439
|
+
*/
|
|
440
|
+
const CANCEL_ORIGIN_TO_PRINCIPAL_KIND = new Map([
|
|
441
|
+
["self", "supervisor"],
|
|
442
|
+
["supervisor-drain", "supervisor"],
|
|
443
|
+
["supervisor-operator", "supervisor"],
|
|
444
|
+
["hub-admin", "hub"],
|
|
445
|
+
]);
|
|
446
|
+
/**
|
|
447
|
+
* Cross-event shape carried by every blob committed under
|
|
448
|
+
* `runs/<runId>/events/`. The discriminator field on disk is `type`,
|
|
449
|
+
* matching the convention used by the substrate's `subscribeKind`
|
|
450
|
+
* helper and the workflow-host scheduler.
|
|
451
|
+
*/
|
|
452
|
+
const EventEnvelope = type({
|
|
453
|
+
type: "string",
|
|
454
|
+
seq: "number >= 0",
|
|
455
|
+
"+": "ignore",
|
|
456
|
+
});
|
|
457
|
+
/**
|
|
458
|
+
* Structural validator for the `CancelRequested` payload's
|
|
459
|
+
* cancellation-specific fields. The kind handler verifies the origin
|
|
460
|
+
* is a known CancelOrigin and the reason is a non-empty string; the
|
|
461
|
+
* principal-vs-origin map collapses because every origin is
|
|
462
|
+
* supervisor-signed in this design.
|
|
463
|
+
*/
|
|
464
|
+
const CancelRequestedFields = type({
|
|
465
|
+
origin: "string",
|
|
466
|
+
reason: "string > 0",
|
|
467
|
+
"+": "ignore",
|
|
468
|
+
});
|
|
469
|
+
const SidecarPrincipal = type({
|
|
470
|
+
kind: "'sidecar'",
|
|
471
|
+
agentId: "string",
|
|
472
|
+
});
|
|
473
|
+
const WorkflowProcessPrincipal = type({
|
|
474
|
+
kind: "'workflow-process'",
|
|
475
|
+
anchorRunId: "string",
|
|
476
|
+
"runId?": "string",
|
|
477
|
+
});
|
|
478
|
+
const SupervisorPrincipal = type({
|
|
479
|
+
kind: "'supervisor'",
|
|
480
|
+
anchorRunId: "string",
|
|
481
|
+
});
|
|
482
|
+
/**
|
|
483
|
+
* Resolve the substrate's `changedPathPrefixes` into the set of run ids
|
|
484
|
+
* the commit could have touched, or `undefined` to validate every run.
|
|
485
|
+
*
|
|
486
|
+
* Returns `undefined` (validate-all) when the substrate could not bound
|
|
487
|
+
* the change set, or when a change prefix reaches into `runs/` without
|
|
488
|
+
* naming a specific run (`runs` or `runs/` alone). A change prefix that
|
|
489
|
+
* never touches `runs/` -- e.g. a claim-check write under `addresses/`
|
|
490
|
+
* -- contributes no run ids; an empty result set means the commit
|
|
491
|
+
* touched no run, so the per-run walks legitimately validate nothing.
|
|
492
|
+
*/
|
|
493
|
+
function runScopeFromChangedPrefixes(changedPathPrefixes) {
|
|
494
|
+
if (changedPathPrefixes === undefined)
|
|
495
|
+
return undefined;
|
|
496
|
+
const runsPrefix = `${WORKFLOW_RUN_RUNS_PREFIX}/`;
|
|
497
|
+
const runIds = new Set();
|
|
498
|
+
for (const prefix of changedPathPrefixes) {
|
|
499
|
+
if (prefix === WORKFLOW_RUN_RUNS_PREFIX || prefix === runsPrefix) {
|
|
500
|
+
// The `runs/` subtree changed but the substrate could not name
|
|
501
|
+
// which run; fall back to validating every run.
|
|
502
|
+
return undefined;
|
|
503
|
+
}
|
|
504
|
+
if (!prefix.startsWith(runsPrefix))
|
|
505
|
+
continue;
|
|
506
|
+
const rest = prefix.slice(runsPrefix.length);
|
|
507
|
+
const slash = rest.indexOf("/");
|
|
508
|
+
if (slash <= 0)
|
|
509
|
+
return undefined;
|
|
510
|
+
runIds.add(rest.slice(0, slash));
|
|
511
|
+
}
|
|
512
|
+
return runIds;
|
|
513
|
+
}
|
|
514
|
+
/**
|
|
515
|
+
* Build the (runId → events[]) map by walking the prospective tree.
|
|
516
|
+
* The substrate's listDir yields names directly under the given
|
|
517
|
+
* directory, so the walk is `runs/` → run-id subdirs → `events/` →
|
|
518
|
+
* event filenames. Filenames outside the `<seq>.json` shape fail the
|
|
519
|
+
* push. When `scopeRunIds` is supplied, only those runs are walked --
|
|
520
|
+
* see the substrate's `changedPathPrefixes` contract.
|
|
521
|
+
*/
|
|
522
|
+
async function enumerateEventBlobs(listDir, scopeRunIds) {
|
|
523
|
+
const runs = new Map();
|
|
524
|
+
// When the substrate bounds the commit's change set to a specific set
|
|
525
|
+
// of runs, walk only those `runs/<runId>/` directories instead of
|
|
526
|
+
// listing every run. An untouched run is carried forward
|
|
527
|
+
// byte-identical by the substrate's prefix-preserving commit, so its
|
|
528
|
+
// per-run invariants -- already validated when it was written --
|
|
529
|
+
// cannot change. `scopeRunIds` may name a run absent from the tree
|
|
530
|
+
// (e.g. a prior-tree walk for a run the prospective tree dropped);
|
|
531
|
+
// `listDir` on a missing directory returns `[]`, which the
|
|
532
|
+
// empty-children guards below handle.
|
|
533
|
+
const runIds = scopeRunIds === undefined
|
|
534
|
+
? await listDir(WORKFLOW_RUN_RUNS_PREFIX)
|
|
535
|
+
: Array.from(scopeRunIds);
|
|
536
|
+
for (const runId of runIds) {
|
|
537
|
+
const runDirPath = `${WORKFLOW_RUN_RUNS_PREFIX}/${runId}`;
|
|
538
|
+
const runChildren = await listDir(runDirPath);
|
|
539
|
+
// A scoped run id can name a run that is absent from the tree being
|
|
540
|
+
// walked -- the substrate's change set is the union of prospective
|
|
541
|
+
// and prior touched runs, so the prospective walk may receive a run
|
|
542
|
+
// that exists only in the prior tree (and vice versa). An absent run
|
|
543
|
+
// directory lists as empty; skip it here so only runs actually
|
|
544
|
+
// present in this tree are validated. The unscoped walk never
|
|
545
|
+
// reaches this branch because its run ids come from listing the
|
|
546
|
+
// present `runs/` directory.
|
|
547
|
+
if (scopeRunIds !== undefined && runChildren.length === 0)
|
|
548
|
+
continue;
|
|
549
|
+
const offender = runChildren.find((c) => !RUN_DIR_ALLOWED_CHILDREN.has(c));
|
|
550
|
+
if (offender !== undefined) {
|
|
551
|
+
return {
|
|
552
|
+
ok: false,
|
|
553
|
+
reason: `run directory ${runDirPath} contains unexpected entry ${JSON.stringify(offender)}; only "${WORKFLOW_RUN_EVENTS_DIR}", "${WORKFLOW_RUN_BLOBS_DIR}", "${WORKFLOW_RUN_EVENTS_FILE}", and "${WORKFLOW_RUN_GRANTS_FILE}" are allowed`,
|
|
554
|
+
};
|
|
555
|
+
}
|
|
556
|
+
const hasCombined = runChildren.includes(WORKFLOW_RUN_EVENTS_FILE);
|
|
557
|
+
const hasPerEvent = runChildren.includes(WORKFLOW_RUN_EVENTS_DIR);
|
|
558
|
+
if (hasCombined && hasPerEvent) {
|
|
559
|
+
return {
|
|
560
|
+
ok: false,
|
|
561
|
+
reason: `run directory ${runDirPath} carries both a combined "${WORKFLOW_RUN_EVENTS_FILE}" and a per-event "${WORKFLOW_RUN_EVENTS_DIR}" subtree`,
|
|
562
|
+
};
|
|
563
|
+
}
|
|
564
|
+
// A sealed (combined) run carries no per-event entries; it is validated
|
|
565
|
+
// by the combined-form path, not this per-event enumeration.
|
|
566
|
+
if (hasCombined)
|
|
567
|
+
continue;
|
|
568
|
+
if (!hasPerEvent) {
|
|
569
|
+
// A run dir whose only child is `grants.json` is the legitimate
|
|
570
|
+
// pre-first-event window: the hub's `run.grants` frame writes the
|
|
571
|
+
// grants ahead of the trigger, so the grants file lands before the
|
|
572
|
+
// child emits its first event. Carry it forward untouched -- there is
|
|
573
|
+
// no event log to enumerate yet. Any other events-less shape (e.g. a
|
|
574
|
+
// bare `blobs/` with no events) remains rejected below.
|
|
575
|
+
if (runChildren.length === 1 &&
|
|
576
|
+
runChildren[0] === WORKFLOW_RUN_GRANTS_FILE) {
|
|
577
|
+
continue;
|
|
578
|
+
}
|
|
579
|
+
return {
|
|
580
|
+
ok: false,
|
|
581
|
+
reason: `run directory ${runDirPath} is missing required "${WORKFLOW_RUN_EVENTS_DIR}" subdirectory`,
|
|
582
|
+
};
|
|
583
|
+
}
|
|
584
|
+
const eventsDirPath = `${runDirPath}/${WORKFLOW_RUN_EVENTS_DIR}`;
|
|
585
|
+
const filenames = await listDir(eventsDirPath);
|
|
586
|
+
const entries = [];
|
|
587
|
+
for (const filename of filenames) {
|
|
588
|
+
const match = EVENT_FILENAME_RE.exec(filename);
|
|
589
|
+
if (match === null) {
|
|
590
|
+
return {
|
|
591
|
+
ok: false,
|
|
592
|
+
reason: `event filename ${eventsDirPath}/${filename} does not match <seq>.json`,
|
|
593
|
+
};
|
|
594
|
+
}
|
|
595
|
+
const seqStr = match[1];
|
|
596
|
+
if (seqStr === undefined) {
|
|
597
|
+
return {
|
|
598
|
+
ok: false,
|
|
599
|
+
reason: `event filename ${eventsDirPath}/${filename} produced no seq capture`,
|
|
600
|
+
};
|
|
601
|
+
}
|
|
602
|
+
entries.push({
|
|
603
|
+
runId,
|
|
604
|
+
filename,
|
|
605
|
+
filenameSeq: Number.parseInt(seqStr, 10),
|
|
606
|
+
blobPath: `${eventsDirPath}/${filename}`,
|
|
607
|
+
});
|
|
608
|
+
}
|
|
609
|
+
entries.sort((a, b) => a.filenameSeq - b.filenameSeq);
|
|
610
|
+
runs.set(runId, entries);
|
|
611
|
+
}
|
|
612
|
+
return { ok: true, runs };
|
|
613
|
+
}
|
|
614
|
+
/**
|
|
615
|
+
* Validate the prospective tree's combined-form (sealed) runs and return
|
|
616
|
+
* the set of run ids that legitimately carry a combined `events.jsonl`.
|
|
617
|
+
* The deletion-direction guard uses that set to allow a run's per-event
|
|
618
|
+
* files to disappear when (and only when) they were folded into the
|
|
619
|
+
* combined file under this same validation.
|
|
620
|
+
*
|
|
621
|
+
* Three prior states are accepted:
|
|
622
|
+
* - prior already combined -> the sealed file is immutable; prospective
|
|
623
|
+
* bytes must equal prior bytes.
|
|
624
|
+
* - prior per-event -> the compaction transition; the combined
|
|
625
|
+
* file must be the byte-for-byte fold of the prior per-event blobs in
|
|
626
|
+
* seq order. This is the audit-integrity boundary: a loose check here
|
|
627
|
+
* would let compaction silently rewrite history.
|
|
628
|
+
* - prior absent -> a freshly-delivered sealed run (e.g. a
|
|
629
|
+
* pack receive); its own structure is validated.
|
|
630
|
+
*/
|
|
631
|
+
async function validateCombinedEventRuns(listDir, readBlob, priorListDir, priorReadBlob, scopeRunIds) {
|
|
632
|
+
const combinedRunIds = new Set();
|
|
633
|
+
const runIds = scopeRunIds === undefined
|
|
634
|
+
? await listDir(WORKFLOW_RUN_RUNS_PREFIX)
|
|
635
|
+
: Array.from(scopeRunIds);
|
|
636
|
+
for (const runId of runIds) {
|
|
637
|
+
const runDirPath = `${WORKFLOW_RUN_RUNS_PREFIX}/${runId}`;
|
|
638
|
+
const children = await listDir(runDirPath);
|
|
639
|
+
if (!children.includes(WORKFLOW_RUN_EVENTS_FILE))
|
|
640
|
+
continue;
|
|
641
|
+
const combinedPath = `${runDirPath}/${WORKFLOW_RUN_EVENTS_FILE}`;
|
|
642
|
+
const combinedBytes = await readBlob(combinedPath);
|
|
643
|
+
const content = new TextDecoder().decode(combinedBytes);
|
|
644
|
+
const priorChildren = await priorListDir(runDirPath);
|
|
645
|
+
if (priorChildren.includes(WORKFLOW_RUN_EVENTS_FILE)) {
|
|
646
|
+
// Sealed once, immutable thereafter.
|
|
647
|
+
const immutable = await checkPriorByteEquality(combinedPath, readBlob, priorReadBlob);
|
|
648
|
+
if (!immutable.ok)
|
|
649
|
+
return immutable;
|
|
650
|
+
}
|
|
651
|
+
else if (priorChildren.includes(WORKFLOW_RUN_EVENTS_DIR)) {
|
|
652
|
+
const structure = checkCombinedStructure(runId, combinedPath, content);
|
|
653
|
+
if (!structure.ok)
|
|
654
|
+
return structure;
|
|
655
|
+
const fold = await checkCompactionFold(runId, runDirPath, combinedBytes, priorListDir, priorReadBlob);
|
|
656
|
+
if (!fold.ok)
|
|
657
|
+
return fold;
|
|
658
|
+
}
|
|
659
|
+
else {
|
|
660
|
+
const structure = checkCombinedStructure(runId, combinedPath, content);
|
|
661
|
+
if (!structure.ok)
|
|
662
|
+
return structure;
|
|
663
|
+
}
|
|
664
|
+
combinedRunIds.add(runId);
|
|
665
|
+
}
|
|
666
|
+
return { ok: true, combinedRunIds };
|
|
667
|
+
}
|
|
668
|
+
/**
|
|
669
|
+
* The audit-integrity bridge. A compaction commit replaces a run's prior
|
|
670
|
+
* `events/<seq>.json` files with one combined file; this asserts the
|
|
671
|
+
* combined file reproduces those prior blobs' bytes verbatim, in seq
|
|
672
|
+
* order, with nothing added, dropped, reordered, or mutated. It rebuilds
|
|
673
|
+
* the expected combined bytes from the prior tree through the same encoder
|
|
674
|
+
* the writer uses, so the two cannot drift, and compares for exact
|
|
675
|
+
* equality.
|
|
676
|
+
*/
|
|
677
|
+
async function checkCompactionFold(runId, runDirPath, combinedBytes, priorListDir, priorReadBlob) {
|
|
678
|
+
const priorEventsDir = `${runDirPath}/${WORKFLOW_RUN_EVENTS_DIR}`;
|
|
679
|
+
const priorEntries = [];
|
|
680
|
+
for (const filename of await priorListDir(priorEventsDir)) {
|
|
681
|
+
const match = EVENT_FILENAME_RE.exec(filename);
|
|
682
|
+
if (match === null || match[1] === undefined) {
|
|
683
|
+
return {
|
|
684
|
+
ok: false,
|
|
685
|
+
reason: `prior event filename ${priorEventsDir}/${filename} does not match <seq>.json; cannot validate compaction of run ${runId}`,
|
|
686
|
+
};
|
|
687
|
+
}
|
|
688
|
+
priorEntries.push({
|
|
689
|
+
seq: Number.parseInt(match[1], 10),
|
|
690
|
+
path: `${priorEventsDir}/${filename}`,
|
|
691
|
+
});
|
|
692
|
+
}
|
|
693
|
+
priorEntries.sort((a, b) => a.seq - b.seq);
|
|
694
|
+
const priorBlobs = [];
|
|
695
|
+
for (const entry of priorEntries) {
|
|
696
|
+
const bytes = await priorReadBlob(entry.path);
|
|
697
|
+
if (bytes === null) {
|
|
698
|
+
return {
|
|
699
|
+
ok: false,
|
|
700
|
+
reason: `prior event ${entry.path} is unreadable; cannot validate compaction of run ${runId}`,
|
|
701
|
+
};
|
|
702
|
+
}
|
|
703
|
+
priorBlobs.push(bytes);
|
|
704
|
+
}
|
|
705
|
+
// Byte equality, not decoded-string equality: each event is signed over
|
|
706
|
+
// its own bytes, so the sealed file must be the verbatim concatenation
|
|
707
|
+
// of the prior blobs, not merely decode-equivalent to it.
|
|
708
|
+
const expected = encodeCombinedEventLog(priorBlobs);
|
|
709
|
+
const sameBytes = combinedBytes.byteLength === expected.byteLength &&
|
|
710
|
+
combinedBytes.every((b, i) => b === expected[i]);
|
|
711
|
+
if (!sameBytes) {
|
|
712
|
+
return {
|
|
713
|
+
ok: false,
|
|
714
|
+
reason: `run ${runId} compaction does not fold its prior events verbatim: ${runDirPath}/${WORKFLOW_RUN_EVENTS_FILE} must equal the run's prior events/<seq>.json blobs joined in seq order`,
|
|
715
|
+
};
|
|
716
|
+
}
|
|
717
|
+
return { ok: true };
|
|
718
|
+
}
|
|
719
|
+
/**
|
|
720
|
+
* Validate a combined event log's own structure: every line a valid event
|
|
721
|
+
* envelope, contiguous seqs, exactly one terminal event and it is last
|
|
722
|
+
* (so a sealed run is genuinely terminal). Used for a sealed run with no
|
|
723
|
+
* prior per-event form to bridge against.
|
|
724
|
+
*/
|
|
725
|
+
function checkCombinedStructure(runId, combinedPath, content) {
|
|
726
|
+
const lines = splitCombinedEventLog(content);
|
|
727
|
+
if (lines.length === 0) {
|
|
728
|
+
return { ok: false, reason: `combined event log ${combinedPath} is empty` };
|
|
729
|
+
}
|
|
730
|
+
let baseSeq = null;
|
|
731
|
+
let terminalSeq = null;
|
|
732
|
+
for (const [i, line] of lines.entries()) {
|
|
733
|
+
let body;
|
|
734
|
+
try {
|
|
735
|
+
body = JSON.parse(line);
|
|
736
|
+
}
|
|
737
|
+
catch {
|
|
738
|
+
return {
|
|
739
|
+
ok: false,
|
|
740
|
+
reason: `combined event log ${combinedPath} line ${String(i)} is not valid JSON`,
|
|
741
|
+
};
|
|
742
|
+
}
|
|
743
|
+
const validated = EventEnvelope(body);
|
|
744
|
+
if (validated instanceof type.errors) {
|
|
745
|
+
return {
|
|
746
|
+
ok: false,
|
|
747
|
+
reason: `combined event log ${combinedPath} line ${String(i)} envelope invalid: ${validated.summary}`,
|
|
748
|
+
};
|
|
749
|
+
}
|
|
750
|
+
if (baseSeq === null) {
|
|
751
|
+
baseSeq = validated.seq;
|
|
752
|
+
}
|
|
753
|
+
else if (validated.seq !== baseSeq + i) {
|
|
754
|
+
return {
|
|
755
|
+
ok: false,
|
|
756
|
+
reason: `combined event log ${combinedPath} has a sequence gap at line ${String(i)} (expected seq ${String(baseSeq + i)}, got ${String(validated.seq)})`,
|
|
757
|
+
};
|
|
758
|
+
}
|
|
759
|
+
if (terminalSeq !== null) {
|
|
760
|
+
return {
|
|
761
|
+
ok: false,
|
|
762
|
+
reason: `combined event log ${combinedPath} has an event at seq ${String(validated.seq)} after terminal at seq ${String(terminalSeq)}`,
|
|
763
|
+
};
|
|
764
|
+
}
|
|
765
|
+
if (TERMINAL_EVENT_TYPES.has(validated.type)) {
|
|
766
|
+
terminalSeq = validated.seq;
|
|
767
|
+
}
|
|
768
|
+
}
|
|
769
|
+
if (terminalSeq === null) {
|
|
770
|
+
return {
|
|
771
|
+
ok: false,
|
|
772
|
+
reason: `combined event log ${combinedPath} for run ${runId} has no terminal event; only a terminated run is sealed`,
|
|
773
|
+
};
|
|
774
|
+
}
|
|
775
|
+
return { ok: true };
|
|
776
|
+
}
|
|
777
|
+
/**
|
|
778
|
+
* Walk every `runs/<runId>/blobs/` directory and validate each blob
|
|
779
|
+
* filename matches the sha256-hex shape the production `BlobSubstrate`
|
|
780
|
+
* adapter writes. The `blobs/` subdirectory itself is optional: a run
|
|
781
|
+
* that has not yet spilled an output to a blob never produces a
|
|
782
|
+
* `blobs/` directory, and a run with only inline-encoded outputs never
|
|
783
|
+
* will. Returns the flat list of blob entries so the caller can apply
|
|
784
|
+
* immutability checks against the prior tree.
|
|
785
|
+
*/
|
|
786
|
+
async function enumerateRunBlobs(listDir, scopeRunIds) {
|
|
787
|
+
const out = [];
|
|
788
|
+
// See enumerateEventBlobs: a defined `scopeRunIds` walks only the
|
|
789
|
+
// commit's touched runs; an untouched run's blobs are carried forward
|
|
790
|
+
// byte-identical and were validated when written.
|
|
791
|
+
const runIds = scopeRunIds === undefined
|
|
792
|
+
? await listDir(WORKFLOW_RUN_RUNS_PREFIX)
|
|
793
|
+
: Array.from(scopeRunIds);
|
|
794
|
+
for (const runId of runIds) {
|
|
795
|
+
const runDirPath = `${WORKFLOW_RUN_RUNS_PREFIX}/${runId}`;
|
|
796
|
+
const runChildren = await listDir(runDirPath);
|
|
797
|
+
if (!runChildren.includes(WORKFLOW_RUN_BLOBS_DIR))
|
|
798
|
+
continue;
|
|
799
|
+
const blobsDirPath = `${runDirPath}/${WORKFLOW_RUN_BLOBS_DIR}`;
|
|
800
|
+
const filenames = await listDir(blobsDirPath);
|
|
801
|
+
for (const filename of filenames) {
|
|
802
|
+
if (!BLOB_FILENAME_RE.test(filename)) {
|
|
803
|
+
return {
|
|
804
|
+
ok: false,
|
|
805
|
+
reason: `blob filename ${blobsDirPath}/${filename} does not match a lowercase 64-character sha256 hex string`,
|
|
806
|
+
};
|
|
807
|
+
}
|
|
808
|
+
out.push({
|
|
809
|
+
runId,
|
|
810
|
+
filename,
|
|
811
|
+
blobPath: `${blobsDirPath}/${filename}`,
|
|
812
|
+
});
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
return { ok: true, blobs: out };
|
|
816
|
+
}
|
|
817
|
+
/**
|
|
818
|
+
* Enforce blob immutability via prior-tree byte equality. The blob
|
|
819
|
+
* value itself is opaque bytes (no JSON envelope, no arktype
|
|
820
|
+
* validation); the only structural rule beyond filename shape is that
|
|
821
|
+
* a blob entry present in the prior tree must carry byte-identical
|
|
822
|
+
* contents in the prospective tree. Mirrors the consumed-entry
|
|
823
|
+
* discipline in the claim-check subtree.
|
|
824
|
+
*/
|
|
825
|
+
async function checkBlobPriorByteEquality(blobPath, readBlob, priorReadBlob) {
|
|
826
|
+
const prior = await priorReadBlob(blobPath);
|
|
827
|
+
if (prior === null)
|
|
828
|
+
return { ok: true };
|
|
829
|
+
const prospective = await readBlob(blobPath);
|
|
830
|
+
if (prior.byteLength !== prospective.byteLength) {
|
|
831
|
+
return {
|
|
832
|
+
ok: false,
|
|
833
|
+
reason: `blob ${blobPath} bytes diverge from the prior tree (lengths ${String(prior.byteLength)} vs ${String(prospective.byteLength)}); blob entries are immutable once written`,
|
|
834
|
+
};
|
|
835
|
+
}
|
|
836
|
+
for (let i = 0; i < prior.byteLength; i++) {
|
|
837
|
+
if (prior[i] !== prospective[i]) {
|
|
838
|
+
return {
|
|
839
|
+
ok: false,
|
|
840
|
+
reason: `blob ${blobPath} bytes diverge from the prior tree at offset ${String(i)}; blob entries are immutable once written`,
|
|
841
|
+
};
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
return { ok: true };
|
|
845
|
+
}
|
|
846
|
+
async function parseEventBlob(entry, readBlob) {
|
|
847
|
+
let raw;
|
|
848
|
+
try {
|
|
849
|
+
raw = await readBlob(entry.blobPath);
|
|
850
|
+
}
|
|
851
|
+
catch (cause) {
|
|
852
|
+
return {
|
|
853
|
+
ok: false,
|
|
854
|
+
reason: `event ${entry.blobPath} could not be read from the tree: ${cause instanceof Error ? cause.message : String(cause)}`,
|
|
855
|
+
};
|
|
856
|
+
}
|
|
857
|
+
let body;
|
|
858
|
+
try {
|
|
859
|
+
body = JSON.parse(new TextDecoder().decode(raw));
|
|
860
|
+
}
|
|
861
|
+
catch (cause) {
|
|
862
|
+
return {
|
|
863
|
+
ok: false,
|
|
864
|
+
reason: `event ${entry.blobPath} is not valid JSON: ${cause instanceof Error ? cause.message : String(cause)}`,
|
|
865
|
+
};
|
|
866
|
+
}
|
|
867
|
+
const validated = EventEnvelope(body);
|
|
868
|
+
if (validated instanceof type.errors) {
|
|
869
|
+
return {
|
|
870
|
+
ok: false,
|
|
871
|
+
reason: `event ${entry.blobPath} envelope invalid: ${validated.summary}`,
|
|
872
|
+
};
|
|
873
|
+
}
|
|
874
|
+
if (validated.seq !== entry.filenameSeq) {
|
|
875
|
+
return {
|
|
876
|
+
ok: false,
|
|
877
|
+
reason: `event ${entry.blobPath} body.seq ${String(validated.seq)} does not match filename seq ${String(entry.filenameSeq)}`,
|
|
878
|
+
};
|
|
879
|
+
}
|
|
880
|
+
if (validated.type === "CancelRequested") {
|
|
881
|
+
const cancelFields = CancelRequestedFields(body);
|
|
882
|
+
if (cancelFields instanceof type.errors) {
|
|
883
|
+
return {
|
|
884
|
+
ok: false,
|
|
885
|
+
reason: `event ${entry.blobPath} CancelRequested payload invalid: ${cancelFields.summary}`,
|
|
886
|
+
};
|
|
887
|
+
}
|
|
888
|
+
if (!CANCEL_REQUESTED_ORIGINS.has(cancelFields.origin)) {
|
|
889
|
+
return {
|
|
890
|
+
ok: false,
|
|
891
|
+
reason: `event ${entry.blobPath} CancelRequested origin ${JSON.stringify(cancelFields.origin)} is not a recognised CancelOrigin`,
|
|
892
|
+
};
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
return { ok: true, parsed: { entry, body: validated } };
|
|
896
|
+
}
|
|
897
|
+
/**
|
|
898
|
+
* Compare the prospective bytes of `blobPath` against the bytes at
|
|
899
|
+
* the same path in the prior tree. Returns `{ ok: true }` when the
|
|
900
|
+
* blob is newly added (no prior entry) or when the prior and
|
|
901
|
+
* prospective bytes are byte-identical; returns a rejection otherwise.
|
|
902
|
+
* Surfaces append-only at the handler scope: the event log invariant
|
|
903
|
+
* lives here rather than relying on caller-layer discipline at
|
|
904
|
+
* `writeTreePreservingPrefix`.
|
|
905
|
+
*/
|
|
906
|
+
async function checkPriorByteEquality(blobPath, readBlob, priorReadBlob) {
|
|
907
|
+
const prior = await priorReadBlob(blobPath);
|
|
908
|
+
if (prior === null)
|
|
909
|
+
return { ok: true };
|
|
910
|
+
const prospective = await readBlob(blobPath);
|
|
911
|
+
if (prior.byteLength !== prospective.byteLength) {
|
|
912
|
+
return {
|
|
913
|
+
ok: false,
|
|
914
|
+
reason: `event ${blobPath} bytes diverge from the prior tree (lengths ${String(prior.byteLength)} vs ${String(prospective.byteLength)}); event blobs are append-only`,
|
|
915
|
+
};
|
|
916
|
+
}
|
|
917
|
+
for (let i = 0; i < prior.byteLength; i++) {
|
|
918
|
+
if (prior[i] !== prospective[i]) {
|
|
919
|
+
return {
|
|
920
|
+
ok: false,
|
|
921
|
+
reason: `event ${blobPath} bytes diverge from the prior tree at offset ${String(i)}; event blobs are append-only`,
|
|
922
|
+
};
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
return { ok: true };
|
|
926
|
+
}
|
|
927
|
+
/**
|
|
928
|
+
* Round-trip an `<urlEncoded(address)>` segment through decode then
|
|
929
|
+
* encode. A divergence means the segment is not the canonical
|
|
930
|
+
* encoding of any address, which would leave consumers guessing
|
|
931
|
+
* which encoding to use when reading the subtree. Surface as a
|
|
932
|
+
* concrete rejection at push time.
|
|
933
|
+
*/
|
|
934
|
+
function checkAddressSegmentRoundTrip(segment) {
|
|
935
|
+
let decoded;
|
|
936
|
+
try {
|
|
937
|
+
decoded = decodeURIComponent(segment);
|
|
938
|
+
}
|
|
939
|
+
catch (cause) {
|
|
940
|
+
return {
|
|
941
|
+
ok: false,
|
|
942
|
+
reason: `address segment ${JSON.stringify(segment)} is not a valid URL-encoded string: ${cause instanceof Error ? cause.message : String(cause)}`,
|
|
943
|
+
};
|
|
944
|
+
}
|
|
945
|
+
const reencoded = encodeURIComponent(decoded);
|
|
946
|
+
if (reencoded !== segment) {
|
|
947
|
+
return {
|
|
948
|
+
ok: false,
|
|
949
|
+
reason: `address segment ${JSON.stringify(segment)} does not round-trip URL-encoding (re-encoded as ${JSON.stringify(reencoded)})`,
|
|
950
|
+
};
|
|
951
|
+
}
|
|
952
|
+
return { ok: true, decoded };
|
|
953
|
+
}
|
|
954
|
+
/**
|
|
955
|
+
* FIFO comparator for inbox/processing entries. Sorts by the parsed
|
|
956
|
+
* numeric `receivedAt` (filename prefix); ties break on the
|
|
957
|
+
* messageId tail. The numeric compare is the load-bearing piece —
|
|
958
|
+
* lexicographic compare on `<receivedAt>-…` filenames with
|
|
959
|
+
* non-uniform digit widths disagrees with chronological order
|
|
960
|
+
* (e.g. `"100-…"` < `"99-…"` because `'1' < '9'`).
|
|
961
|
+
*/
|
|
962
|
+
function compareQueueEntries(a, b) {
|
|
963
|
+
const aReceivedAt = a.receivedAtFromFilename;
|
|
964
|
+
const bReceivedAt = b.receivedAtFromFilename;
|
|
965
|
+
if (aReceivedAt === null || bReceivedAt === null) {
|
|
966
|
+
throw new Error("compareQueueEntries: queue entries must carry a parsed receivedAt");
|
|
967
|
+
}
|
|
968
|
+
if (aReceivedAt !== bReceivedAt)
|
|
969
|
+
return aReceivedAt - bReceivedAt;
|
|
970
|
+
const aId = a.messageIdFromFilename;
|
|
971
|
+
const bId = b.messageIdFromFilename;
|
|
972
|
+
if (aId < bId)
|
|
973
|
+
return -1;
|
|
974
|
+
if (aId > bId)
|
|
975
|
+
return 1;
|
|
976
|
+
return 0;
|
|
977
|
+
}
|
|
978
|
+
async function enumerateClaimCheckBlobs(listDir, resolveConsumedOid) {
|
|
979
|
+
const perAddress = new Map();
|
|
980
|
+
const segments = await listDir(WORKFLOW_RUN_ADDRESSES_PREFIX);
|
|
981
|
+
for (const segment of segments) {
|
|
982
|
+
const roundTrip = checkAddressSegmentRoundTrip(segment);
|
|
983
|
+
if (!roundTrip.ok)
|
|
984
|
+
return roundTrip;
|
|
985
|
+
const addrDir = `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${segment}`;
|
|
986
|
+
const children = await listDir(addrDir);
|
|
987
|
+
for (const child of children) {
|
|
988
|
+
if (CLAIM_CHECK_SUBDIRS.has(child))
|
|
989
|
+
continue;
|
|
990
|
+
if (child === WORKFLOW_RUN_WATERMARK_FILE)
|
|
991
|
+
continue;
|
|
992
|
+
return {
|
|
993
|
+
ok: false,
|
|
994
|
+
reason: `address directory ${addrDir} contains unexpected entry ${JSON.stringify(child)}; allowed: "${WORKFLOW_RUN_INBOX_DIR}", "${WORKFLOW_RUN_PROCESSING_DIR}", "${WORKFLOW_RUN_CONSUMED_DIR}", "${WORKFLOW_RUN_WATERMARK_FILE}"`,
|
|
995
|
+
};
|
|
996
|
+
}
|
|
997
|
+
const bucket = perAddress.get(segment) ?? {
|
|
998
|
+
decodedAddress: roundTrip.decoded,
|
|
999
|
+
inbox: [],
|
|
1000
|
+
processing: [],
|
|
1001
|
+
consumed: [],
|
|
1002
|
+
watermarkPath: null,
|
|
1003
|
+
};
|
|
1004
|
+
if (children.includes(WORKFLOW_RUN_WATERMARK_FILE)) {
|
|
1005
|
+
bucket.watermarkPath = `${addrDir}/${WORKFLOW_RUN_WATERMARK_FILE}`;
|
|
1006
|
+
}
|
|
1007
|
+
for (const subdir of CLAIM_CHECK_SUBDIRS) {
|
|
1008
|
+
if (!children.includes(subdir))
|
|
1009
|
+
continue;
|
|
1010
|
+
const dirPath = `${addrDir}/${subdir}`;
|
|
1011
|
+
const filenames = await listDir(dirPath);
|
|
1012
|
+
for (const filename of filenames) {
|
|
1013
|
+
if (subdir === WORKFLOW_RUN_INBOX_DIR ||
|
|
1014
|
+
subdir === WORKFLOW_RUN_PROCESSING_DIR) {
|
|
1015
|
+
const match = QUEUE_FILENAME_RE.exec(filename);
|
|
1016
|
+
if (match === null) {
|
|
1017
|
+
return {
|
|
1018
|
+
ok: false,
|
|
1019
|
+
reason: `${subdir} filename ${dirPath}/${filename} does not match <receivedAt>-<messageId>.json`,
|
|
1020
|
+
};
|
|
1021
|
+
}
|
|
1022
|
+
const receivedAtStr = match[1];
|
|
1023
|
+
const messageId = match[2];
|
|
1024
|
+
if (receivedAtStr === undefined || messageId === undefined) {
|
|
1025
|
+
return {
|
|
1026
|
+
ok: false,
|
|
1027
|
+
reason: `${subdir} filename ${dirPath}/${filename} produced no captures`,
|
|
1028
|
+
};
|
|
1029
|
+
}
|
|
1030
|
+
const entry = {
|
|
1031
|
+
kind: subdir === WORKFLOW_RUN_INBOX_DIR ? "inbox" : "processing",
|
|
1032
|
+
addressSegment: segment,
|
|
1033
|
+
decodedAddress: roundTrip.decoded,
|
|
1034
|
+
filename,
|
|
1035
|
+
receivedAtFromFilename: Number.parseInt(receivedAtStr, 10),
|
|
1036
|
+
messageIdFromFilename: messageId,
|
|
1037
|
+
blobPath: `${dirPath}/${filename}`,
|
|
1038
|
+
};
|
|
1039
|
+
if (subdir === WORKFLOW_RUN_INBOX_DIR)
|
|
1040
|
+
bucket.inbox.push(entry);
|
|
1041
|
+
else
|
|
1042
|
+
bucket.processing.push(entry);
|
|
1043
|
+
}
|
|
1044
|
+
else {
|
|
1045
|
+
const match = CONSUMED_FILENAME_RE.exec(filename);
|
|
1046
|
+
if (match === null) {
|
|
1047
|
+
return {
|
|
1048
|
+
ok: false,
|
|
1049
|
+
reason: `${WORKFLOW_RUN_CONSUMED_DIR} filename ${dirPath}/${filename} does not match <messageId>.json`,
|
|
1050
|
+
};
|
|
1051
|
+
}
|
|
1052
|
+
const messageId = match[1];
|
|
1053
|
+
if (messageId === undefined) {
|
|
1054
|
+
return {
|
|
1055
|
+
ok: false,
|
|
1056
|
+
reason: `${WORKFLOW_RUN_CONSUMED_DIR} filename ${dirPath}/${filename} produced no message-id capture`,
|
|
1057
|
+
};
|
|
1058
|
+
}
|
|
1059
|
+
const consumedBlobPath = `${dirPath}/${filename}`;
|
|
1060
|
+
const consumedEntry = {
|
|
1061
|
+
kind: "consumed",
|
|
1062
|
+
addressSegment: segment,
|
|
1063
|
+
decodedAddress: roundTrip.decoded,
|
|
1064
|
+
filename,
|
|
1065
|
+
receivedAtFromFilename: null,
|
|
1066
|
+
messageIdFromFilename: messageId,
|
|
1067
|
+
blobPath: consumedBlobPath,
|
|
1068
|
+
};
|
|
1069
|
+
if (resolveConsumedOid !== undefined) {
|
|
1070
|
+
consumedEntry.oid = await resolveConsumedOid(consumedBlobPath);
|
|
1071
|
+
}
|
|
1072
|
+
bucket.consumed.push(consumedEntry);
|
|
1073
|
+
}
|
|
1074
|
+
}
|
|
1075
|
+
}
|
|
1076
|
+
// FIFO ordering: sort by the parsed numeric receivedAt prefix
|
|
1077
|
+
// with a lexicographic messageId tiebreak. String-sorting the
|
|
1078
|
+
// raw filename would put "99-…" after "100-…" because '9' > '1',
|
|
1079
|
+
// breaking the FIFO invariant for non-uniform digit widths.
|
|
1080
|
+
bucket.inbox.sort(compareQueueEntries);
|
|
1081
|
+
bucket.processing.sort(compareQueueEntries);
|
|
1082
|
+
bucket.consumed.sort((a, b) => a.filename < b.filename ? -1 : a.filename > b.filename ? 1 : 0);
|
|
1083
|
+
perAddress.set(segment, bucket);
|
|
1084
|
+
}
|
|
1085
|
+
return { ok: true, perAddress };
|
|
1086
|
+
}
|
|
1087
|
+
/**
|
|
1088
|
+
* Read + validate the per-address `watermark.json` value from a blob
|
|
1089
|
+
* reader. `null` means the tree has no watermark blob (treated as
|
|
1090
|
+
* watermark 0 -- no entry pruned, nothing refused). The reader may be
|
|
1091
|
+
* the prospective `readBlob` or the `priorReadBlob` (the latter
|
|
1092
|
+
* returns `null` for an absent path, which is the legitimate
|
|
1093
|
+
* never-pruned genesis state).
|
|
1094
|
+
*/
|
|
1095
|
+
async function readWatermark(watermarkPath, readBlob) {
|
|
1096
|
+
let raw;
|
|
1097
|
+
try {
|
|
1098
|
+
raw = await readBlob(watermarkPath);
|
|
1099
|
+
}
|
|
1100
|
+
catch (cause) {
|
|
1101
|
+
return {
|
|
1102
|
+
ok: false,
|
|
1103
|
+
reason: `watermark ${watermarkPath} could not be read from the tree: ${cause instanceof Error ? cause.message : String(cause)}`,
|
|
1104
|
+
};
|
|
1105
|
+
}
|
|
1106
|
+
if (raw === null)
|
|
1107
|
+
return { ok: true, watermark: 0 };
|
|
1108
|
+
let bodyJson;
|
|
1109
|
+
try {
|
|
1110
|
+
bodyJson = JSON.parse(new TextDecoder().decode(raw));
|
|
1111
|
+
}
|
|
1112
|
+
catch (cause) {
|
|
1113
|
+
return {
|
|
1114
|
+
ok: false,
|
|
1115
|
+
reason: `watermark ${watermarkPath} is not valid JSON: ${cause instanceof Error ? cause.message : String(cause)}`,
|
|
1116
|
+
};
|
|
1117
|
+
}
|
|
1118
|
+
const validated = WatermarkEnvelope(bodyJson);
|
|
1119
|
+
if (validated instanceof type.errors) {
|
|
1120
|
+
return {
|
|
1121
|
+
ok: false,
|
|
1122
|
+
reason: `watermark ${watermarkPath} envelope invalid: ${validated.summary}`,
|
|
1123
|
+
};
|
|
1124
|
+
}
|
|
1125
|
+
return { ok: true, watermark: validated.watermark };
|
|
1126
|
+
}
|
|
1127
|
+
async function parseConsumedBlob(entry, readBlob) {
|
|
1128
|
+
return parseConsumedBlobFrom(entry, readBlob);
|
|
1129
|
+
}
|
|
1130
|
+
/**
|
|
1131
|
+
* Read + validate a consumed entry's envelope from a blob reader that
|
|
1132
|
+
* may return `null` for an absent path (the `priorReadBlob` shape).
|
|
1133
|
+
* `null` is treated as a read failure: the caller only passes a path
|
|
1134
|
+
* the prior tree is known to carry, so a `null` is structural damage.
|
|
1135
|
+
*/
|
|
1136
|
+
async function parseConsumedBlobFrom(entry, readBlob) {
|
|
1137
|
+
let raw;
|
|
1138
|
+
try {
|
|
1139
|
+
raw = await readBlob(entry.blobPath);
|
|
1140
|
+
}
|
|
1141
|
+
catch (cause) {
|
|
1142
|
+
return {
|
|
1143
|
+
ok: false,
|
|
1144
|
+
reason: `consumed ${entry.blobPath} could not be read from the tree: ${cause instanceof Error ? cause.message : String(cause)}`,
|
|
1145
|
+
};
|
|
1146
|
+
}
|
|
1147
|
+
if (raw === null) {
|
|
1148
|
+
return {
|
|
1149
|
+
ok: false,
|
|
1150
|
+
reason: `consumed ${entry.blobPath} was enumerated in the tree but its bytes could not be read`,
|
|
1151
|
+
};
|
|
1152
|
+
}
|
|
1153
|
+
let bodyJson;
|
|
1154
|
+
try {
|
|
1155
|
+
bodyJson = JSON.parse(new TextDecoder().decode(raw));
|
|
1156
|
+
}
|
|
1157
|
+
catch (cause) {
|
|
1158
|
+
return {
|
|
1159
|
+
ok: false,
|
|
1160
|
+
reason: `consumed ${entry.blobPath} is not valid JSON: ${cause instanceof Error ? cause.message : String(cause)}`,
|
|
1161
|
+
};
|
|
1162
|
+
}
|
|
1163
|
+
const validated = ConsumedEnvelope(bodyJson);
|
|
1164
|
+
if (validated instanceof type.errors) {
|
|
1165
|
+
return {
|
|
1166
|
+
ok: false,
|
|
1167
|
+
reason: `consumed ${entry.blobPath} envelope invalid: ${validated.summary}`,
|
|
1168
|
+
};
|
|
1169
|
+
}
|
|
1170
|
+
if (validated.messageId !== entry.messageIdFromFilename) {
|
|
1171
|
+
return {
|
|
1172
|
+
ok: false,
|
|
1173
|
+
reason: `consumed ${entry.blobPath} body.messageId ${JSON.stringify(validated.messageId)} does not match filename messageId ${JSON.stringify(entry.messageIdFromFilename)}`,
|
|
1174
|
+
};
|
|
1175
|
+
}
|
|
1176
|
+
if (validated.address !== entry.decodedAddress) {
|
|
1177
|
+
return {
|
|
1178
|
+
ok: false,
|
|
1179
|
+
reason: `consumed ${entry.blobPath} body.address ${JSON.stringify(validated.address)} does not match decoded address segment ${JSON.stringify(entry.decodedAddress)}`,
|
|
1180
|
+
};
|
|
1181
|
+
}
|
|
1182
|
+
return { ok: true, body: validated };
|
|
1183
|
+
}
|
|
1184
|
+
async function parseQueueBlob(entry, readBlob) {
|
|
1185
|
+
let raw;
|
|
1186
|
+
try {
|
|
1187
|
+
raw = await readBlob(entry.blobPath);
|
|
1188
|
+
}
|
|
1189
|
+
catch (cause) {
|
|
1190
|
+
return {
|
|
1191
|
+
ok: false,
|
|
1192
|
+
reason: `${entry.kind} ${entry.blobPath} could not be read from the tree: ${cause instanceof Error ? cause.message : String(cause)}`,
|
|
1193
|
+
};
|
|
1194
|
+
}
|
|
1195
|
+
let bodyJson;
|
|
1196
|
+
try {
|
|
1197
|
+
bodyJson = JSON.parse(new TextDecoder().decode(raw));
|
|
1198
|
+
}
|
|
1199
|
+
catch (cause) {
|
|
1200
|
+
return {
|
|
1201
|
+
ok: false,
|
|
1202
|
+
reason: `${entry.kind} ${entry.blobPath} is not valid JSON: ${cause instanceof Error ? cause.message : String(cause)}`,
|
|
1203
|
+
};
|
|
1204
|
+
}
|
|
1205
|
+
const validated = ClaimCheckEnvelope(bodyJson);
|
|
1206
|
+
if (validated instanceof type.errors) {
|
|
1207
|
+
return {
|
|
1208
|
+
ok: false,
|
|
1209
|
+
reason: `${entry.kind} ${entry.blobPath} envelope invalid: ${validated.summary}`,
|
|
1210
|
+
};
|
|
1211
|
+
}
|
|
1212
|
+
if (validated.messageId !== entry.messageIdFromFilename) {
|
|
1213
|
+
return {
|
|
1214
|
+
ok: false,
|
|
1215
|
+
reason: `${entry.kind} ${entry.blobPath} body.messageId ${JSON.stringify(validated.messageId)} does not match filename messageId ${JSON.stringify(entry.messageIdFromFilename)}`,
|
|
1216
|
+
};
|
|
1217
|
+
}
|
|
1218
|
+
if (validated.receivedAt !== entry.receivedAtFromFilename) {
|
|
1219
|
+
return {
|
|
1220
|
+
ok: false,
|
|
1221
|
+
reason: `${entry.kind} ${entry.blobPath} body.receivedAt ${String(validated.receivedAt)} does not match filename receivedAt ${String(entry.receivedAtFromFilename)}`,
|
|
1222
|
+
};
|
|
1223
|
+
}
|
|
1224
|
+
if (validated.address !== entry.decodedAddress) {
|
|
1225
|
+
return {
|
|
1226
|
+
ok: false,
|
|
1227
|
+
reason: `${entry.kind} ${entry.blobPath} body.address ${JSON.stringify(validated.address)} does not match decoded address segment ${JSON.stringify(entry.decodedAddress)}`,
|
|
1228
|
+
};
|
|
1229
|
+
}
|
|
1230
|
+
return { ok: true, body: validated };
|
|
1231
|
+
}
|
|
1232
|
+
/**
|
|
1233
|
+
* Compute the git blob OID of a consumed entry from a byte reader,
|
|
1234
|
+
* used only when the delta-scoped path lacks a substrate-provided prior
|
|
1235
|
+
* OID listing (e.g. a hand-built test validatePush). `git.hashBlob`
|
|
1236
|
+
* reproduces the same content-addressed OID a `git.readTree` listing
|
|
1237
|
+
* carries, so the delta path's intersection compare is identical
|
|
1238
|
+
* whether the OID came from the tree listing or from hashing the bytes.
|
|
1239
|
+
*/
|
|
1240
|
+
async function hashConsumedBlobOid(bytes) {
|
|
1241
|
+
const { oid } = await git.hashBlob({ object: bytes });
|
|
1242
|
+
return oid;
|
|
1243
|
+
}
|
|
1244
|
+
/**
|
|
1245
|
+
* Resolve each consumed entry's git blob OID for the delta-scoped path.
|
|
1246
|
+
* When the substrate supplies a directory OID listing (`listDirOids`) the
|
|
1247
|
+
* OID comes straight from the tree — one `readTree` per consumed
|
|
1248
|
+
* directory, cached — so that side is not re-read blob-by-blob. When the
|
|
1249
|
+
* listing is absent (a hand-built validatePush in a unit test) each OID
|
|
1250
|
+
* falls back to hashing the entry's bytes, which preserves identical
|
|
1251
|
+
* semantics at O(retained) cost. Both the prior and prospective sides use
|
|
1252
|
+
* this; `sideLabel` distinguishes them in the missing-OID error.
|
|
1253
|
+
*/
|
|
1254
|
+
function makeListingOidResolver(sideLabel, listDirOids, hashFallback) {
|
|
1255
|
+
const dirOidCache = new Map();
|
|
1256
|
+
return async (blobPath) => {
|
|
1257
|
+
if (listDirOids !== undefined) {
|
|
1258
|
+
const slash = blobPath.lastIndexOf("/");
|
|
1259
|
+
const dir = blobPath.slice(0, slash);
|
|
1260
|
+
const name = blobPath.slice(slash + 1);
|
|
1261
|
+
let byName = dirOidCache.get(dir);
|
|
1262
|
+
if (byName === undefined) {
|
|
1263
|
+
byName = new Map();
|
|
1264
|
+
for (const entry of await listDirOids(dir)) {
|
|
1265
|
+
byName.set(entry.name, entry.oid);
|
|
1266
|
+
}
|
|
1267
|
+
dirOidCache.set(dir, byName);
|
|
1268
|
+
}
|
|
1269
|
+
const oid = byName.get(name);
|
|
1270
|
+
if (oid === undefined) {
|
|
1271
|
+
throw new Error(`delta claim-check: ${sideLabel} tree listing has no OID for enumerated consumed entry ${blobPath}`);
|
|
1272
|
+
}
|
|
1273
|
+
return oid;
|
|
1274
|
+
}
|
|
1275
|
+
return hashFallback(blobPath);
|
|
1276
|
+
};
|
|
1277
|
+
}
|
|
1278
|
+
function makePriorConsumedOidResolver(priorReadBlob, priorListDirOids) {
|
|
1279
|
+
return makeListingOidResolver("prior", priorListDirOids, async (blobPath) => {
|
|
1280
|
+
const bytes = await priorReadBlob(blobPath);
|
|
1281
|
+
if (bytes === null) {
|
|
1282
|
+
throw new Error(`delta claim-check: consumed entry ${blobPath} was enumerated in the prior tree but its bytes could not be read`);
|
|
1283
|
+
}
|
|
1284
|
+
return hashConsumedBlobOid(bytes);
|
|
1285
|
+
});
|
|
1286
|
+
}
|
|
1287
|
+
/**
|
|
1288
|
+
* Validate the `addresses/<urlEncoded>/{inbox,processing,consumed}`
|
|
1289
|
+
* subtree as a whole. The walk enforces filename shape, JSON envelope
|
|
1290
|
+
* structure, address round-trip, per-messageId atomicity across the
|
|
1291
|
+
* three queue states, consumed-blob immutability, and the
|
|
1292
|
+
* inbox→processing / processing→consumed transition invariants against
|
|
1293
|
+
* the prior tree.
|
|
1294
|
+
*
|
|
1295
|
+
* The consumed dedup index is validated by its per-commit DELTA against
|
|
1296
|
+
* the prior tree rather than by re-walking the whole retained set:
|
|
1297
|
+
* retained entries (same filename, same blob OID) are skipped as
|
|
1298
|
+
* already-validated-and-immutable, added entries are parsed and
|
|
1299
|
+
* validated, and removed entries are checked against the retention
|
|
1300
|
+
* watermark. `priorListDirOids` and `listDirOids`, when supplied by the
|
|
1301
|
+
* substrate, surface the prior and prospective consumed OIDs straight
|
|
1302
|
+
* from their tree listings so neither side is re-read blob-by-blob.
|
|
1303
|
+
*/
|
|
1304
|
+
async function validateClaimCheckSubtree(listDir, readBlob, priorReadBlob, priorListDir, priorListDirOids, listDirOids) {
|
|
1305
|
+
// Surface each consumed entry's git blob OID during enumeration
|
|
1306
|
+
// straight from the tree listing on both sides when the substrate
|
|
1307
|
+
// provides it, falling back to hashing the bytes otherwise.
|
|
1308
|
+
const prospectiveConsumedOid = makeListingOidResolver("prospective", listDirOids, async (blobPath) => hashConsumedBlobOid(await readBlob(blobPath)));
|
|
1309
|
+
const priorConsumedOid = makePriorConsumedOidResolver(priorReadBlob, priorListDirOids);
|
|
1310
|
+
const enumerated = await enumerateClaimCheckBlobs(listDir, prospectiveConsumedOid);
|
|
1311
|
+
if (!enumerated.ok)
|
|
1312
|
+
return enumerated;
|
|
1313
|
+
const priorEnumerated = await enumerateClaimCheckBlobs(priorListDir, priorConsumedOid);
|
|
1314
|
+
if (!priorEnumerated.ok) {
|
|
1315
|
+
// The prior tree is the committed state — if its claim-check
|
|
1316
|
+
// shape is already broken, surface it with a distinct rejection
|
|
1317
|
+
// prefix so an operator can tell prior-state damage from a
|
|
1318
|
+
// misconfigured push.
|
|
1319
|
+
return {
|
|
1320
|
+
ok: false,
|
|
1321
|
+
reason: `prior tree's claim-check subtree is structurally invalid: ${priorEnumerated.reason}`,
|
|
1322
|
+
};
|
|
1323
|
+
}
|
|
1324
|
+
const emptyBucket = (decodedAddress) => ({
|
|
1325
|
+
decodedAddress,
|
|
1326
|
+
inbox: [],
|
|
1327
|
+
processing: [],
|
|
1328
|
+
consumed: [],
|
|
1329
|
+
watermarkPath: null,
|
|
1330
|
+
});
|
|
1331
|
+
// Iterate the UNION of prospective and prior address segments so a
|
|
1332
|
+
// prospective tree that wipes an address subtree entirely still
|
|
1333
|
+
// runs the prior-retention checks against that segment's
|
|
1334
|
+
// prior-tree consumed/processing entries.
|
|
1335
|
+
const allSegments = new Set([
|
|
1336
|
+
...enumerated.perAddress.keys(),
|
|
1337
|
+
...priorEnumerated.perAddress.keys(),
|
|
1338
|
+
]);
|
|
1339
|
+
for (const segment of allSegments) {
|
|
1340
|
+
const priorBucket = priorEnumerated.perAddress.get(segment);
|
|
1341
|
+
const prospectiveBucketForSegment = enumerated.perAddress.get(segment);
|
|
1342
|
+
const decodedAddress = prospectiveBucketForSegment?.decodedAddress ??
|
|
1343
|
+
priorBucket?.decodedAddress;
|
|
1344
|
+
if (decodedAddress === undefined) {
|
|
1345
|
+
throw new Error(`validateClaimCheckSubtree: segment ${JSON.stringify(segment)} appeared in the union of prospective and prior segments but neither bucket carries a decoded address`);
|
|
1346
|
+
}
|
|
1347
|
+
const bucket = prospectiveBucketForSegment ?? emptyBucket(decodedAddress);
|
|
1348
|
+
// Per-messageId atomicity: each messageId may appear at most
|
|
1349
|
+
// once across inbox/processing/consumed combined. The check
|
|
1350
|
+
// keys on (messageId, kind, filename) so two inbox entries with
|
|
1351
|
+
// the same messageId at different `receivedAt` values surface
|
|
1352
|
+
// as a same-state collision (the Set-of-kinds shape would
|
|
1353
|
+
// collapse both into a single "inbox" member and miss the
|
|
1354
|
+
// case).
|
|
1355
|
+
const messageIdToLocations = new Map();
|
|
1356
|
+
for (const entry of [...bucket.inbox, ...bucket.processing]) {
|
|
1357
|
+
const parsed = await parseQueueBlob(entry, readBlob);
|
|
1358
|
+
if (!parsed.ok)
|
|
1359
|
+
return parsed;
|
|
1360
|
+
const list = messageIdToLocations.get(entry.messageIdFromFilename) ?? [];
|
|
1361
|
+
list.push({ kind: entry.kind, filename: entry.filename });
|
|
1362
|
+
messageIdToLocations.set(entry.messageIdFromFilename, list);
|
|
1363
|
+
}
|
|
1364
|
+
for (const entry of bucket.consumed) {
|
|
1365
|
+
// Cross-state atomicity needs each consumed messageId in the map;
|
|
1366
|
+
// the messageId is the filename stem, so this needs no blob read.
|
|
1367
|
+
// Retained consumed entries are not re-parsed (their envelope was
|
|
1368
|
+
// validated when first written and their bytes are proven
|
|
1369
|
+
// immutable by the OID compare below); added consumed entries are
|
|
1370
|
+
// parsed and validated by the transition check further down.
|
|
1371
|
+
const list = messageIdToLocations.get(entry.messageIdFromFilename) ?? [];
|
|
1372
|
+
list.push({ kind: entry.kind, filename: entry.filename });
|
|
1373
|
+
messageIdToLocations.set(entry.messageIdFromFilename, list);
|
|
1374
|
+
}
|
|
1375
|
+
for (const [messageId, locations] of messageIdToLocations) {
|
|
1376
|
+
if (locations.length > 1) {
|
|
1377
|
+
const sorted = [...locations].sort((a, b) => {
|
|
1378
|
+
if (a.kind !== b.kind)
|
|
1379
|
+
return a.kind < b.kind ? -1 : 1;
|
|
1380
|
+
if (a.filename !== b.filename)
|
|
1381
|
+
return a.filename < b.filename ? -1 : 1;
|
|
1382
|
+
return 0;
|
|
1383
|
+
});
|
|
1384
|
+
const kinds = new Set(sorted.map((l) => l.kind));
|
|
1385
|
+
if (kinds.size > 1) {
|
|
1386
|
+
return {
|
|
1387
|
+
ok: false,
|
|
1388
|
+
reason: `address ${JSON.stringify(bucket.decodedAddress)} message ${JSON.stringify(messageId)} appears in multiple queue states ${JSON.stringify(Array.from(kinds).sort())}; at most one of inbox/processing/consumed is permitted`,
|
|
1389
|
+
};
|
|
1390
|
+
}
|
|
1391
|
+
const kind = sorted[0]?.kind;
|
|
1392
|
+
if (kind === undefined)
|
|
1393
|
+
throw new Error("unreachable");
|
|
1394
|
+
return {
|
|
1395
|
+
ok: false,
|
|
1396
|
+
reason: `address ${JSON.stringify(bucket.decodedAddress)} message ${JSON.stringify(messageId)} appears at multiple ${kind} positions ${JSON.stringify(sorted.map((l) => l.filename))}; at most one entry per messageId is permitted`,
|
|
1397
|
+
};
|
|
1398
|
+
}
|
|
1399
|
+
}
|
|
1400
|
+
// Consumed entries are immutable. Compare the git blob OID the
|
|
1401
|
+
// enumeration surfaced: a consumed entry present in the prior tree
|
|
1402
|
+
// at the same path must carry the same OID (git trees are
|
|
1403
|
+
// content-addressed, so equal OID proves byte-equality without
|
|
1404
|
+
// reading either blob). A diverging OID is an immutability
|
|
1405
|
+
// violation. Immutability is load-bearing for exactly-once: a
|
|
1406
|
+
// mutated `receivedAt` on a retained consumed entry could fake it
|
|
1407
|
+
// below the watermark, get it pruned, and let a re-submission miss
|
|
1408
|
+
// dedup -- so this compare is not optional.
|
|
1409
|
+
const priorConsumedOidByPath = new Map();
|
|
1410
|
+
for (const e of priorBucket?.consumed ?? []) {
|
|
1411
|
+
if (e.oid === undefined) {
|
|
1412
|
+
throw new Error(`delta claim-check: prior consumed entry ${e.blobPath} was enumerated without an OID`);
|
|
1413
|
+
}
|
|
1414
|
+
priorConsumedOidByPath.set(e.blobPath, e.oid);
|
|
1415
|
+
}
|
|
1416
|
+
for (const entry of bucket.consumed) {
|
|
1417
|
+
const priorOid = priorConsumedOidByPath.get(entry.blobPath);
|
|
1418
|
+
if (priorOid === undefined)
|
|
1419
|
+
continue; // newly added; validated below
|
|
1420
|
+
if (entry.oid === undefined) {
|
|
1421
|
+
throw new Error(`delta claim-check: prospective consumed entry ${entry.blobPath} was enumerated without an OID`);
|
|
1422
|
+
}
|
|
1423
|
+
if (entry.oid !== priorOid) {
|
|
1424
|
+
return {
|
|
1425
|
+
ok: false,
|
|
1426
|
+
reason: `consumed ${entry.blobPath} bytes diverge from the prior tree (blob OID ${entry.oid} vs ${priorOid}); consumed entries are immutable once written`,
|
|
1427
|
+
};
|
|
1428
|
+
}
|
|
1429
|
+
}
|
|
1430
|
+
const prospectiveConsumedPaths = new Set(bucket.consumed.map((e) => e.blobPath));
|
|
1431
|
+
const prospectiveProcessingPaths = new Set(bucket.processing.map((e) => e.blobPath));
|
|
1432
|
+
const prospectiveInboxByFilename = new Map();
|
|
1433
|
+
const prospectiveInboxPaths = new Set();
|
|
1434
|
+
for (const e of bucket.inbox) {
|
|
1435
|
+
prospectiveInboxByFilename.set(e.filename, e);
|
|
1436
|
+
prospectiveInboxPaths.add(e.blobPath);
|
|
1437
|
+
}
|
|
1438
|
+
const prospectiveProcessingByFilename = new Map();
|
|
1439
|
+
for (const e of bucket.processing)
|
|
1440
|
+
prospectiveProcessingByFilename.set(e.filename, e);
|
|
1441
|
+
const prospectiveConsumedByMessageId = new Map();
|
|
1442
|
+
for (const e of bucket.consumed)
|
|
1443
|
+
prospectiveConsumedByMessageId.set(e.messageIdFromFilename, e);
|
|
1444
|
+
// Deletion-direction guards: walk every entry the prior tree
|
|
1445
|
+
// carried under `consumed/`, `processing/`, and `inbox/` and reject
|
|
1446
|
+
// any prior path that vanishes from the prospective tree except via
|
|
1447
|
+
// a permitted transition (or, for consumed, a watermark-passed
|
|
1448
|
+
// retention prune). Without this walk a prospective tree that
|
|
1449
|
+
// simply omits a prior entry would slip past the prospective-tree
|
|
1450
|
+
// by-presence checks above.
|
|
1451
|
+
//
|
|
1452
|
+
// Retention-watermark contract for the consumed dedup index. The
|
|
1453
|
+
// watermark is a monotonic `receivedAt` horizon; a `markConsumed`
|
|
1454
|
+
// commit may drop the oldest consumed tail (entries strictly below
|
|
1455
|
+
// the watermark) and the watermark may only advance. Resolve both
|
|
1456
|
+
// the prospective and prior watermark up front so the consumed
|
|
1457
|
+
// deletion check below can bind every drop to the watermark.
|
|
1458
|
+
let prospectiveWatermark = 0;
|
|
1459
|
+
if (bucket.watermarkPath !== null) {
|
|
1460
|
+
const wm = await readWatermark(bucket.watermarkPath, (p) => readBlob(p));
|
|
1461
|
+
if (!wm.ok)
|
|
1462
|
+
return wm;
|
|
1463
|
+
prospectiveWatermark = wm.watermark;
|
|
1464
|
+
}
|
|
1465
|
+
let priorWatermark = 0;
|
|
1466
|
+
if (priorBucket?.watermarkPath != null) {
|
|
1467
|
+
const wm = await readWatermark(priorBucket.watermarkPath, priorReadBlob);
|
|
1468
|
+
if (!wm.ok)
|
|
1469
|
+
return wm;
|
|
1470
|
+
priorWatermark = wm.watermark;
|
|
1471
|
+
}
|
|
1472
|
+
if (prospectiveWatermark < priorWatermark) {
|
|
1473
|
+
return {
|
|
1474
|
+
ok: false,
|
|
1475
|
+
reason: `address ${JSON.stringify(decodedAddress)} retention watermark regressed from ${String(priorWatermark)} to ${String(prospectiveWatermark)}; the watermark is monotonically non-decreasing`,
|
|
1476
|
+
};
|
|
1477
|
+
}
|
|
1478
|
+
if (priorBucket !== undefined) {
|
|
1479
|
+
// The consumed dedup index may shrink only by a watermark-passed
|
|
1480
|
+
// prune: a consumed entry dropped from the prior tree must have a
|
|
1481
|
+
// receivedAt strictly below the prospective watermark (you may
|
|
1482
|
+
// prune only what the watermark passed). Combined with the
|
|
1483
|
+
// already-verified watermark monotonicity, this is the whole of
|
|
1484
|
+
// the exactly-once retention contract: pruning is bound to the
|
|
1485
|
+
// watermark and the watermark only advances.
|
|
1486
|
+
//
|
|
1487
|
+
// The suffix relation (dropped entries older than every retained
|
|
1488
|
+
// entry) is deliberately NOT enforced. A RETAINED entry is NOT
|
|
1489
|
+
// required to sit at or above the watermark: a message consumed
|
|
1490
|
+
// long after receipt (or replayed back in-flight) may
|
|
1491
|
+
// legitimately carry a below-watermark receivedAt and survive
|
|
1492
|
+
// until a later commit prunes it. Holding it gives EXTRA dedup (a
|
|
1493
|
+
// re-submission at or above the watermark still hits the retained
|
|
1494
|
+
// entry; one below is stale-rejected at enqueue), so a hole left
|
|
1495
|
+
// by an out-of-order prune weakens nothing.
|
|
1496
|
+
//
|
|
1497
|
+
// Only the dropped entries are read. A retained entry (present in
|
|
1498
|
+
// both trees) is proven byte-identical by the OID compare above,
|
|
1499
|
+
// so its receivedAt is unchanged and need not be read. The
|
|
1500
|
+
// receivedAt lives in the body; read it from the prior tree
|
|
1501
|
+
// (retained bytes are immutable, so prior and prospective agree).
|
|
1502
|
+
for (const e of priorBucket.consumed) {
|
|
1503
|
+
if (prospectiveConsumedPaths.has(e.blobPath))
|
|
1504
|
+
continue;
|
|
1505
|
+
const priorParsed = await parseConsumedBlobFrom(e, priorReadBlob);
|
|
1506
|
+
if (!priorParsed.ok)
|
|
1507
|
+
return priorParsed;
|
|
1508
|
+
const receivedAt = priorParsed.body.receivedAt;
|
|
1509
|
+
if (receivedAt >= prospectiveWatermark) {
|
|
1510
|
+
return {
|
|
1511
|
+
ok: false,
|
|
1512
|
+
reason: `consumed ${e.blobPath} present in the prior tree is missing from the prospective tree but its receivedAt ${String(receivedAt)} is not below the retention watermark ${String(prospectiveWatermark)}; consumed entries may be pruned only once the watermark has passed them`,
|
|
1513
|
+
};
|
|
1514
|
+
}
|
|
1515
|
+
}
|
|
1516
|
+
for (const e of priorBucket.processing) {
|
|
1517
|
+
if (prospectiveProcessingPaths.has(e.blobPath))
|
|
1518
|
+
continue;
|
|
1519
|
+
// A processing entry may legitimately disappear in two
|
|
1520
|
+
// shapes: (1) markConsumed wrote a matching consumed entry
|
|
1521
|
+
// keyed by the same messageId, or (2) replayProcessingToInbox
|
|
1522
|
+
// moved the entry back to inbox preserving the
|
|
1523
|
+
// `<receivedAt>-<messageId>.json` filename. Anything else is
|
|
1524
|
+
// an in-flight loss.
|
|
1525
|
+
const consumedMatch = prospectiveConsumedByMessageId.get(e.messageIdFromFilename);
|
|
1526
|
+
const inboxMatch = prospectiveInboxByFilename.get(e.filename);
|
|
1527
|
+
if (consumedMatch !== undefined || inboxMatch !== undefined)
|
|
1528
|
+
continue;
|
|
1529
|
+
return {
|
|
1530
|
+
ok: false,
|
|
1531
|
+
reason: `processing ${e.blobPath} present in the prior tree is missing from the prospective tree without a matching consumed or inbox transition; in-flight processing entries cannot be silently dropped`,
|
|
1532
|
+
};
|
|
1533
|
+
}
|
|
1534
|
+
for (const e of priorBucket.inbox) {
|
|
1535
|
+
if (prospectiveInboxPaths.has(e.blobPath))
|
|
1536
|
+
continue;
|
|
1537
|
+
// A prior inbox entry may legitimately disappear when it
|
|
1538
|
+
// transitions to processing (same `<receivedAt>-<messageId>`
|
|
1539
|
+
// filename) or directly to consumed (matching messageId).
|
|
1540
|
+
// Anything else is an inbound-mail loss — the FIFO claim-check
|
|
1541
|
+
// contract requires the entry to reappear somewhere.
|
|
1542
|
+
const processingMatch = prospectiveProcessingByFilename.get(e.filename);
|
|
1543
|
+
const consumedMatch = prospectiveConsumedByMessageId.get(e.messageIdFromFilename);
|
|
1544
|
+
if (processingMatch !== undefined || consumedMatch !== undefined)
|
|
1545
|
+
continue;
|
|
1546
|
+
return {
|
|
1547
|
+
ok: false,
|
|
1548
|
+
reason: `inbox ${e.blobPath} present in the prior tree is missing from the prospective tree without a matching processing or consumed transition; pending inbox entries cannot be silently dropped`,
|
|
1549
|
+
};
|
|
1550
|
+
}
|
|
1551
|
+
}
|
|
1552
|
+
const priorInboxByFilename = new Map();
|
|
1553
|
+
const priorProcessingByMessageId = new Map();
|
|
1554
|
+
if (priorBucket !== undefined) {
|
|
1555
|
+
for (const e of priorBucket.inbox)
|
|
1556
|
+
priorInboxByFilename.set(e.filename, e);
|
|
1557
|
+
for (const e of priorBucket.processing)
|
|
1558
|
+
priorProcessingByMessageId.set(e.messageIdFromFilename, e);
|
|
1559
|
+
}
|
|
1560
|
+
const priorProcessingPaths = new Set((priorBucket?.processing ?? []).map((e) => e.blobPath));
|
|
1561
|
+
const priorConsumedPaths = new Set((priorBucket?.consumed ?? []).map((e) => e.blobPath));
|
|
1562
|
+
// Newly-added processing entries must match an inbox entry that
|
|
1563
|
+
// existed in the prior tree at the same `<receivedAt>-<messageId>`
|
|
1564
|
+
// filename. This makes inbox→processing the only legal way to
|
|
1565
|
+
// grow processing/.
|
|
1566
|
+
for (const entry of bucket.processing) {
|
|
1567
|
+
if (priorProcessingPaths.has(entry.blobPath))
|
|
1568
|
+
continue;
|
|
1569
|
+
const priorInbox = priorInboxByFilename.get(entry.filename);
|
|
1570
|
+
if (priorInbox === undefined) {
|
|
1571
|
+
return {
|
|
1572
|
+
ok: false,
|
|
1573
|
+
reason: `processing ${entry.blobPath} is newly added but the prior tree has no matching inbox entry ${JSON.stringify(`${WORKFLOW_RUN_ADDRESSES_PREFIX}/${segment}/${WORKFLOW_RUN_INBOX_DIR}/${entry.filename}`)}; processing entries must originate from a prior-tree inbox entry`,
|
|
1574
|
+
};
|
|
1575
|
+
}
|
|
1576
|
+
}
|
|
1577
|
+
// Newly-added consumed entries must match a processing entry that
|
|
1578
|
+
// existed in the prior tree at the same address+messageId, and
|
|
1579
|
+
// the receivedAt carried in the consumed envelope must equal the
|
|
1580
|
+
// receivedAt the processing entry's filename carried.
|
|
1581
|
+
for (const entry of bucket.consumed) {
|
|
1582
|
+
if (priorConsumedPaths.has(entry.blobPath))
|
|
1583
|
+
continue;
|
|
1584
|
+
const priorProcessing = priorProcessingByMessageId.get(entry.messageIdFromFilename);
|
|
1585
|
+
if (priorProcessing === undefined) {
|
|
1586
|
+
return {
|
|
1587
|
+
ok: false,
|
|
1588
|
+
reason: `consumed ${entry.blobPath} is newly added but the prior tree has no matching processing entry for messageId ${JSON.stringify(entry.messageIdFromFilename)}; consumed entries must originate from a prior-tree processing entry`,
|
|
1589
|
+
};
|
|
1590
|
+
}
|
|
1591
|
+
const parsed = await parseConsumedBlob(entry, readBlob);
|
|
1592
|
+
if (!parsed.ok)
|
|
1593
|
+
return parsed;
|
|
1594
|
+
const consumedBody = parsed.body;
|
|
1595
|
+
if (consumedBody.receivedAt !== priorProcessing.receivedAtFromFilename) {
|
|
1596
|
+
return {
|
|
1597
|
+
ok: false,
|
|
1598
|
+
reason: `consumed ${entry.blobPath} body.receivedAt ${String(consumedBody.receivedAt)} does not match the prior processing entry's receivedAt ${String(priorProcessing.receivedAtFromFilename)} for messageId ${JSON.stringify(entry.messageIdFromFilename)}`,
|
|
1599
|
+
};
|
|
1600
|
+
}
|
|
1601
|
+
}
|
|
1602
|
+
}
|
|
1603
|
+
return { ok: true };
|
|
1604
|
+
}
|
|
1605
|
+
/**
|
|
1606
|
+
* Enforce the Q3 principal-vs-origin map for a parsed
|
|
1607
|
+
* `CancelRequested` event. The principal kind is matched against the
|
|
1608
|
+
* origin's required-signer kind; a mismatch rejects with both the
|
|
1609
|
+
* declared origin and the actual principal kind in the message so a
|
|
1610
|
+
* misconfigured writer surfaces concretely at the push boundary.
|
|
1611
|
+
*/
|
|
1612
|
+
function checkCancelOriginPrincipal(blobPath, origin, principal) {
|
|
1613
|
+
const required = CANCEL_ORIGIN_TO_PRINCIPAL_KIND.get(origin);
|
|
1614
|
+
if (required === undefined) {
|
|
1615
|
+
return {
|
|
1616
|
+
ok: false,
|
|
1617
|
+
reason: `event ${blobPath} CancelRequested origin ${JSON.stringify(origin)} has no principal-kind binding`,
|
|
1618
|
+
};
|
|
1619
|
+
}
|
|
1620
|
+
if (principal.kind !== required) {
|
|
1621
|
+
return {
|
|
1622
|
+
ok: false,
|
|
1623
|
+
reason: `event ${blobPath} CancelRequested origin ${JSON.stringify(origin)} requires principal.kind=${JSON.stringify(required)} but the push was signed by principal.kind=${JSON.stringify(principal.kind)}`,
|
|
1624
|
+
};
|
|
1625
|
+
}
|
|
1626
|
+
return { ok: true };
|
|
1627
|
+
}
|
|
1628
|
+
/**
|
|
1629
|
+
* Path-scoping for the `workflow-process` principal. A workflow-process
|
|
1630
|
+
* proxies writes for the workflow-run repo's `runs/<runId>/` subtree
|
|
1631
|
+
* only; the supervisor owns the `addresses/...` claim-check subtree.
|
|
1632
|
+
* If the principal carries a `runId`, every prospective `runs/<X>/`
|
|
1633
|
+
* subtree must use `X === principal.runId`. A workflow-process that
|
|
1634
|
+
* touches the `addresses/...` subtree is rejected outright so the
|
|
1635
|
+
* single-writer contract on inbox/processing/consumed holds at the
|
|
1636
|
+
* substrate boundary.
|
|
1637
|
+
*
|
|
1638
|
+
* The check only fires for `workflow-process` principals; `hub` and
|
|
1639
|
+
* `supervisor` have broader write authority by design.
|
|
1640
|
+
*/
|
|
1641
|
+
async function enforceWorkflowProcessPathScope(principal, topLevelTreePaths, listDir) {
|
|
1642
|
+
if (principal.kind !== "workflow-process")
|
|
1643
|
+
return { ok: true };
|
|
1644
|
+
const parsed = WorkflowProcessPrincipal(principal);
|
|
1645
|
+
if (parsed instanceof type.errors) {
|
|
1646
|
+
// `workflowRunAuthorize` already rejects malformed
|
|
1647
|
+
// `workflow-process` principals at `gateAccess`, so this branch is
|
|
1648
|
+
// unreachable when the substrate is wired against the real
|
|
1649
|
+
// authorize callback. Fail closed so a future wiring that supplies
|
|
1650
|
+
// a permissive authorize (e.g. test substrates using `allowAll`)
|
|
1651
|
+
// cannot silently bypass the path-scope enforcement below.
|
|
1652
|
+
return {
|
|
1653
|
+
ok: false,
|
|
1654
|
+
reason: `workflow-process principal is malformed: ${parsed.summary}`,
|
|
1655
|
+
};
|
|
1656
|
+
}
|
|
1657
|
+
if (topLevelTreePaths.includes(WORKFLOW_RUN_ADDRESSES_PREFIX)) {
|
|
1658
|
+
return {
|
|
1659
|
+
ok: false,
|
|
1660
|
+
reason: `workflow-process principal may not write under ${WORKFLOW_RUN_ADDRESSES_PREFIX}/; the supervisor owns the claim-check subtree`,
|
|
1661
|
+
};
|
|
1662
|
+
}
|
|
1663
|
+
if (parsed.runId !== undefined &&
|
|
1664
|
+
topLevelTreePaths.includes(WORKFLOW_RUN_RUNS_PREFIX)) {
|
|
1665
|
+
const runIds = await listDir(WORKFLOW_RUN_RUNS_PREFIX);
|
|
1666
|
+
for (const runId of runIds) {
|
|
1667
|
+
if (runId !== parsed.runId) {
|
|
1668
|
+
return {
|
|
1669
|
+
ok: false,
|
|
1670
|
+
reason: `workflow-process principal scoped to runId ${JSON.stringify(parsed.runId)} may not write under ${WORKFLOW_RUN_RUNS_PREFIX}/${runId}/`,
|
|
1671
|
+
};
|
|
1672
|
+
}
|
|
1673
|
+
}
|
|
1674
|
+
}
|
|
1675
|
+
return { ok: true };
|
|
1676
|
+
}
|
|
1677
|
+
/**
|
|
1678
|
+
* Validate the `agent-state/` subtree shape (design §3c). The subtree
|
|
1679
|
+
* holds one MUTABLE per-agent conversation snapshot directory per agent
|
|
1680
|
+
* below the prefix; each entry directly under `agent-state/` must be a
|
|
1681
|
+
* `<agentKey>/` DIRECTORY (not a dangling blob), and each `<agentKey>`
|
|
1682
|
+
* segment must round-trip URL-encoding so a reader can recover the
|
|
1683
|
+
* agent's identity from the path. The conversation blobs inside a
|
|
1684
|
+
* `<agentKey>/` directory are opaque to the substrate (the warm agent's
|
|
1685
|
+
* ContextStore owns their shape), so no file-level shape is enforced
|
|
1686
|
+
* here.
|
|
1687
|
+
*
|
|
1688
|
+
* A blob written DIRECTLY at `agent-state/<name>` (with no `<agentKey>/`
|
|
1689
|
+
* layer) is rejected: it would not be keyed by an agent and would not be
|
|
1690
|
+
* recoverable by any reader walking the per-agent layout.
|
|
1691
|
+
*/
|
|
1692
|
+
async function validateAgentStateSubtree(topLevelTreePaths, listDir) {
|
|
1693
|
+
if (!topLevelTreePaths.includes(WORKFLOW_RUN_AGENT_STATE_PREFIX)) {
|
|
1694
|
+
return { ok: true };
|
|
1695
|
+
}
|
|
1696
|
+
const segments = await listDir(WORKFLOW_RUN_AGENT_STATE_PREFIX);
|
|
1697
|
+
for (const segment of segments) {
|
|
1698
|
+
const roundTrip = checkAddressSegmentRoundTrip(segment);
|
|
1699
|
+
if (!roundTrip.ok) {
|
|
1700
|
+
return {
|
|
1701
|
+
ok: false,
|
|
1702
|
+
reason: `agent-state segment ${JSON.stringify(segment)} does not round-trip URL-encoding; ${roundTrip.reason}`,
|
|
1703
|
+
};
|
|
1704
|
+
}
|
|
1705
|
+
// Reject a blob dangling directly at `agent-state/<segment>`: every
|
|
1706
|
+
// entry under the prefix must be a `<agentKey>/` directory carrying
|
|
1707
|
+
// the agent's snapshot files. A directory has children under
|
|
1708
|
+
// `agent-state/<segment>/`; a direct blob has none.
|
|
1709
|
+
const children = await listDir(`${WORKFLOW_RUN_AGENT_STATE_PREFIX}/${segment}`);
|
|
1710
|
+
if (children.length === 0) {
|
|
1711
|
+
return {
|
|
1712
|
+
ok: false,
|
|
1713
|
+
reason: `agent-state entry ${JSON.stringify(segment)} is a blob directly under ${WORKFLOW_RUN_AGENT_STATE_PREFIX}/; entries must be a <agentKey>/ directory carrying the agent's snapshot files`,
|
|
1714
|
+
};
|
|
1715
|
+
}
|
|
1716
|
+
}
|
|
1717
|
+
return { ok: true };
|
|
1718
|
+
}
|
|
1719
|
+
export const workflowRunKindHandler = {
|
|
1720
|
+
kind: "workflow-run",
|
|
1721
|
+
directoryPrefix: "workflow-runs",
|
|
1722
|
+
async validatePush({ repoId, ref, principal, topLevelTreePaths, readBlob, listDir, listDirOids, priorReadBlob, priorListDir, priorListDirOids, changedPathPrefixes, }) {
|
|
1723
|
+
// Bound the per-run event/blob walks to the runs this commit could
|
|
1724
|
+
// have touched. The substrate guarantees a prefix-preserving commit
|
|
1725
|
+
// mutates only paths under `changedPathPrefixes`; every run outside
|
|
1726
|
+
// them is carried forward byte-identical, so its per-run invariants
|
|
1727
|
+
// (seq-contiguity, terminal-lock, append-only, blob-immutability)
|
|
1728
|
+
// cannot change and were already validated when the run was last
|
|
1729
|
+
// written. `scopeRunIds` is the set of run ids under a
|
|
1730
|
+
// `runs/<runId>/` change prefix. It stays `undefined` -- validate
|
|
1731
|
+
// every run -- whenever the substrate could not bound the change set
|
|
1732
|
+
// (`changedPathPrefixes` is undefined) OR a change prefix touches the
|
|
1733
|
+
// `runs/` subtree at a coarser-than-per-run granularity (a bare
|
|
1734
|
+
// `runs/` prefix, which cannot identify which run changed), so the
|
|
1735
|
+
// scoping never narrows below what the substrate can prove.
|
|
1736
|
+
const scopeRunIds = runScopeFromChangedPrefixes(changedPathPrefixes);
|
|
1737
|
+
for (const entry of topLevelTreePaths) {
|
|
1738
|
+
if (entry.startsWith(`${WORKFLOW_RUN_CONTROL_PREFIX}/`) ||
|
|
1739
|
+
entry === WORKFLOW_RUN_CONTROL_PREFIX) {
|
|
1740
|
+
return {
|
|
1741
|
+
ok: false,
|
|
1742
|
+
reason: `top-level entry ${JSON.stringify(entry)} is under the unsupported ${WORKFLOW_RUN_CONTROL_PREFIX}/ subtree`,
|
|
1743
|
+
};
|
|
1744
|
+
}
|
|
1745
|
+
if (!ALLOWED_TOP_LEVEL.has(entry)) {
|
|
1746
|
+
return {
|
|
1747
|
+
ok: false,
|
|
1748
|
+
reason: `unexpected top-level entry ${JSON.stringify(entry)}; allowed: "${WORKFLOW_RUN_RUNS_PREFIX}", "${WORKFLOW_RUN_ADDRESSES_PREFIX}", "${WORKFLOW_RUN_AGENT_STATE_PREFIX}", "${WORKFLOW_RUN_GITIGNORE_PATH}"`,
|
|
1749
|
+
};
|
|
1750
|
+
}
|
|
1751
|
+
}
|
|
1752
|
+
const scopingCheck = await enforceWorkflowProcessPathScope(principal, topLevelTreePaths, listDir);
|
|
1753
|
+
if (!scopingCheck.ok) {
|
|
1754
|
+
logger.debug `workflow-run validatePush rejected ${repoId.kind}/${repoId.id} on ${ref}: ${scopingCheck.reason}`;
|
|
1755
|
+
return scopingCheck;
|
|
1756
|
+
}
|
|
1757
|
+
const agentStateCheck = await validateAgentStateSubtree(topLevelTreePaths, listDir);
|
|
1758
|
+
if (!agentStateCheck.ok) {
|
|
1759
|
+
logger.debug `workflow-run validatePush rejected ${repoId.kind}/${repoId.id} on ${ref}: ${agentStateCheck.reason}`;
|
|
1760
|
+
return agentStateCheck;
|
|
1761
|
+
}
|
|
1762
|
+
const priorTopLevels = await priorListDir("");
|
|
1763
|
+
const addressesPresent = topLevelTreePaths.includes(WORKFLOW_RUN_ADDRESSES_PREFIX) ||
|
|
1764
|
+
priorTopLevels.includes(WORKFLOW_RUN_ADDRESSES_PREFIX);
|
|
1765
|
+
if (addressesPresent) {
|
|
1766
|
+
// Enter claim-check validation when the prospective OR prior
|
|
1767
|
+
// tree carries an `addresses/` subtree. A prospective tree that
|
|
1768
|
+
// omits `addresses/` while the prior tree had consumed or
|
|
1769
|
+
// processing entries must still go through the subtree walk so
|
|
1770
|
+
// those prior entries' deletion-direction invariants fire.
|
|
1771
|
+
const claimCheck = await validateClaimCheckSubtree(listDir, readBlob, priorReadBlob, priorListDir, priorListDirOids, listDirOids);
|
|
1772
|
+
if (!claimCheck.ok) {
|
|
1773
|
+
logger.debug `workflow-run validatePush rejected ${repoId.kind}/${repoId.id} on ${ref}: ${claimCheck.reason}`;
|
|
1774
|
+
return claimCheck;
|
|
1775
|
+
}
|
|
1776
|
+
}
|
|
1777
|
+
const runsPresent = topLevelTreePaths.includes(WORKFLOW_RUN_RUNS_PREFIX) ||
|
|
1778
|
+
priorTopLevels.includes(WORKFLOW_RUN_RUNS_PREFIX);
|
|
1779
|
+
if (!runsPresent) {
|
|
1780
|
+
// A workflow-run repo without any `runs/` directory in either
|
|
1781
|
+
// the prior or the prospective tree is a genesis state for the
|
|
1782
|
+
// events subtree — `.gitignore`-only or claim-check-only trees
|
|
1783
|
+
// are accepted so the asset routes' init can land before any
|
|
1784
|
+
// run has produced an event.
|
|
1785
|
+
return { ok: true };
|
|
1786
|
+
}
|
|
1787
|
+
const enumerated = await enumerateEventBlobs(listDir, scopeRunIds);
|
|
1788
|
+
if (!enumerated.ok) {
|
|
1789
|
+
logger.debug `workflow-run validatePush rejected ${repoId.kind}/${repoId.id} on ${ref}: ${enumerated.reason}`;
|
|
1790
|
+
return { ok: false, reason: enumerated.reason };
|
|
1791
|
+
}
|
|
1792
|
+
const newlyTerminalRuns = [];
|
|
1793
|
+
for (const [runId, entries] of enumerated.runs) {
|
|
1794
|
+
if (entries.length === 0) {
|
|
1795
|
+
return {
|
|
1796
|
+
ok: false,
|
|
1797
|
+
reason: `run ${runId} has an empty events directory`,
|
|
1798
|
+
};
|
|
1799
|
+
}
|
|
1800
|
+
// Sequence contiguity: per-run events must run contiguously
|
|
1801
|
+
// through the tip from whatever seq the first entry uses. Without
|
|
1802
|
+
// this, a downstream consumer that iterates the log by seq would
|
|
1803
|
+
// skip past a gap silently. `entries` is sorted by filenameSeq
|
|
1804
|
+
// above. The first seq is not pinned to 0 because the runtime
|
|
1805
|
+
// body's emptyState carries `lastSeq = 0` and emits its first
|
|
1806
|
+
// event at `seq = lastSeq + 1 = 1`, while the supervisor's
|
|
1807
|
+
// self-signed CancelRequested path lands seq=0 against an empty
|
|
1808
|
+
// events tree.
|
|
1809
|
+
const firstEntry = entries[0];
|
|
1810
|
+
if (firstEntry === undefined)
|
|
1811
|
+
throw new Error("unreachable");
|
|
1812
|
+
const baseSeq = firstEntry.filenameSeq;
|
|
1813
|
+
for (let i = 0; i < entries.length; i++) {
|
|
1814
|
+
const e = entries[i];
|
|
1815
|
+
if (e === undefined)
|
|
1816
|
+
throw new Error("unreachable");
|
|
1817
|
+
const expectedSeq = baseSeq + i;
|
|
1818
|
+
if (e.filenameSeq !== expectedSeq) {
|
|
1819
|
+
const expectedPath = `${WORKFLOW_RUN_RUNS_PREFIX}/${runId}/${WORKFLOW_RUN_EVENTS_DIR}/${String(expectedSeq)}.json`;
|
|
1820
|
+
return {
|
|
1821
|
+
ok: false,
|
|
1822
|
+
reason: `run ${runId} events have a sequence gap: ${expectedPath} is missing (next observed is ${e.blobPath})`,
|
|
1823
|
+
};
|
|
1824
|
+
}
|
|
1825
|
+
}
|
|
1826
|
+
let terminalSeq = null;
|
|
1827
|
+
let terminalType = null;
|
|
1828
|
+
for (const entry of entries) {
|
|
1829
|
+
const priorCheck = await checkPriorByteEquality(entry.blobPath, readBlob, priorReadBlob);
|
|
1830
|
+
if (!priorCheck.ok) {
|
|
1831
|
+
logger.debug `workflow-run validatePush rejected ${repoId.kind}/${repoId.id} on ${ref}: ${priorCheck.reason}`;
|
|
1832
|
+
return priorCheck;
|
|
1833
|
+
}
|
|
1834
|
+
const parsed = await parseEventBlob(entry, readBlob);
|
|
1835
|
+
if (!parsed.ok) {
|
|
1836
|
+
logger.debug `workflow-run validatePush rejected ${repoId.kind}/${repoId.id} on ${ref}: ${parsed.reason}`;
|
|
1837
|
+
return { ok: false, reason: parsed.reason };
|
|
1838
|
+
}
|
|
1839
|
+
if (parsed.parsed.body.type === "CancelRequested") {
|
|
1840
|
+
const origin = parsed.parsed.body.origin;
|
|
1841
|
+
if (typeof origin !== "string") {
|
|
1842
|
+
return {
|
|
1843
|
+
ok: false,
|
|
1844
|
+
reason: `event ${entry.blobPath} CancelRequested origin must be a string`,
|
|
1845
|
+
};
|
|
1846
|
+
}
|
|
1847
|
+
// Enforce the cancel-origin principal only for a NEWLY-ADDED blob.
|
|
1848
|
+
// A CancelRequested's origin-vs-signer rule is a write-time
|
|
1849
|
+
// authorization: it belongs to the commit that authors the event. A
|
|
1850
|
+
// later commit that merely carries the event forward -- e.g. the
|
|
1851
|
+
// run's own workflow-process cascade write of CancelPropagated /
|
|
1852
|
+
// RunCancelled, which re-lists the whole events prefix -- must not be
|
|
1853
|
+
// rejected because the carried-forward cancel was authored under a
|
|
1854
|
+
// different (supervisor) signer. Re-checking it protects nothing: the
|
|
1855
|
+
// byte-equality check above already proves a carried-forward blob is
|
|
1856
|
+
// unchanged, and the deletion-direction check proves it cannot be
|
|
1857
|
+
// dropped. A tampered (byte-diverged) blob never reaches here --
|
|
1858
|
+
// checkPriorByteEquality rejects it first. Mirrors the newly-terminal
|
|
1859
|
+
// gate below, which likewise acts only on a blob absent from the
|
|
1860
|
+
// prior tree.
|
|
1861
|
+
if ((await priorReadBlob(entry.blobPath)) === null) {
|
|
1862
|
+
const principalCheck = checkCancelOriginPrincipal(entry.blobPath, origin, principal);
|
|
1863
|
+
if (!principalCheck.ok) {
|
|
1864
|
+
logger.debug `workflow-run validatePush rejected ${repoId.kind}/${repoId.id} on ${ref}: ${principalCheck.reason}`;
|
|
1865
|
+
return principalCheck;
|
|
1866
|
+
}
|
|
1867
|
+
}
|
|
1868
|
+
}
|
|
1869
|
+
if (terminalSeq !== null) {
|
|
1870
|
+
return {
|
|
1871
|
+
ok: false,
|
|
1872
|
+
reason: `run ${runId} has event at seq ${String(entry.filenameSeq)} after terminal ${terminalType} at seq ${String(terminalSeq)}`,
|
|
1873
|
+
};
|
|
1874
|
+
}
|
|
1875
|
+
if (TERMINAL_EVENT_TYPES.has(parsed.parsed.body.type)) {
|
|
1876
|
+
terminalSeq = entry.filenameSeq;
|
|
1877
|
+
terminalType = parsed.parsed.body.type;
|
|
1878
|
+
// Surface the run as newly terminal only when this commit is
|
|
1879
|
+
// the one that ADDS the terminal event -- i.e. the terminal
|
|
1880
|
+
// blob is absent from the prior tree. A commit that carries an
|
|
1881
|
+
// already-terminal run forward unchanged (a later compaction
|
|
1882
|
+
// commit folding the per-event files into one) finds the
|
|
1883
|
+
// terminal blob already present in the prior tree and emits no
|
|
1884
|
+
// signal, so a downstream consumer keyed on the signal does
|
|
1885
|
+
// not double-fire.
|
|
1886
|
+
if ((await priorReadBlob(entry.blobPath)) === null) {
|
|
1887
|
+
const status = TERMINAL_EVENT_STATUS.get(parsed.parsed.body.type);
|
|
1888
|
+
if (status === undefined) {
|
|
1889
|
+
throw new Error(`terminal event type ${parsed.parsed.body.type} has no workflow_run.status mapping`);
|
|
1890
|
+
}
|
|
1891
|
+
const terminalBytes = await readBlob(entry.blobPath);
|
|
1892
|
+
newlyTerminalRuns.push({
|
|
1893
|
+
runId,
|
|
1894
|
+
status,
|
|
1895
|
+
terminalEventJson: new TextDecoder().decode(terminalBytes),
|
|
1896
|
+
});
|
|
1897
|
+
}
|
|
1898
|
+
}
|
|
1899
|
+
}
|
|
1900
|
+
}
|
|
1901
|
+
const combinedRuns = await validateCombinedEventRuns(listDir, readBlob, priorListDir, priorReadBlob, scopeRunIds);
|
|
1902
|
+
if (!combinedRuns.ok) {
|
|
1903
|
+
logger.debug `workflow-run validatePush rejected ${repoId.kind}/${repoId.id} on ${ref}: ${combinedRuns.reason}`;
|
|
1904
|
+
return { ok: false, reason: combinedRuns.reason };
|
|
1905
|
+
}
|
|
1906
|
+
const blobsEnumerated = await enumerateRunBlobs(listDir, scopeRunIds);
|
|
1907
|
+
if (!blobsEnumerated.ok) {
|
|
1908
|
+
logger.debug `workflow-run validatePush rejected ${repoId.kind}/${repoId.id} on ${ref}: ${blobsEnumerated.reason}`;
|
|
1909
|
+
return { ok: false, reason: blobsEnumerated.reason };
|
|
1910
|
+
}
|
|
1911
|
+
for (const blob of blobsEnumerated.blobs) {
|
|
1912
|
+
const immutability = await checkBlobPriorByteEquality(blob.blobPath, readBlob, priorReadBlob);
|
|
1913
|
+
if (!immutability.ok) {
|
|
1914
|
+
logger.debug `workflow-run validatePush rejected ${repoId.kind}/${repoId.id} on ${ref}: ${immutability.reason}`;
|
|
1915
|
+
return immutability;
|
|
1916
|
+
}
|
|
1917
|
+
}
|
|
1918
|
+
// Append-only / immutability extended to the deletion direction
|
|
1919
|
+
// for the runs subtree. The prospective-tree walks above only
|
|
1920
|
+
// see paths PRESENT in the prospective tree; a prospective tree
|
|
1921
|
+
// that omits a prior `runs/<runId>/events/<seq>.json` or
|
|
1922
|
+
// `runs/<runId>/blobs/<sha>` slips past those iterations
|
|
1923
|
+
// entirely. Enumerate the prior tree's runs subtree under the
|
|
1924
|
+
// same shapes and reject any prior path that does not reappear.
|
|
1925
|
+
const priorEnumerated = await enumerateEventBlobs(priorListDir, scopeRunIds);
|
|
1926
|
+
if (!priorEnumerated.ok) {
|
|
1927
|
+
return {
|
|
1928
|
+
ok: false,
|
|
1929
|
+
reason: `prior tree's runs subtree is structurally invalid: ${priorEnumerated.reason}`,
|
|
1930
|
+
};
|
|
1931
|
+
}
|
|
1932
|
+
const prospectiveEventPaths = new Set();
|
|
1933
|
+
for (const entries of enumerated.runs.values()) {
|
|
1934
|
+
for (const e of entries)
|
|
1935
|
+
prospectiveEventPaths.add(e.blobPath);
|
|
1936
|
+
}
|
|
1937
|
+
for (const entries of priorEnumerated.runs.values()) {
|
|
1938
|
+
for (const e of entries) {
|
|
1939
|
+
if (prospectiveEventPaths.has(e.blobPath))
|
|
1940
|
+
continue;
|
|
1941
|
+
// A run sealed into its combined events.jsonl by this commit
|
|
1942
|
+
// legitimately drops its per-event files; the fold was validated
|
|
1943
|
+
// byte-for-byte against these same prior blobs above.
|
|
1944
|
+
if (combinedRuns.combinedRunIds.has(e.runId))
|
|
1945
|
+
continue;
|
|
1946
|
+
return {
|
|
1947
|
+
ok: false,
|
|
1948
|
+
reason: `event ${e.blobPath} present in the prior tree is missing from the prospective tree; event blobs are append-only`,
|
|
1949
|
+
};
|
|
1950
|
+
}
|
|
1951
|
+
}
|
|
1952
|
+
const priorBlobsEnumerated = await enumerateRunBlobs(priorListDir, scopeRunIds);
|
|
1953
|
+
if (!priorBlobsEnumerated.ok) {
|
|
1954
|
+
return {
|
|
1955
|
+
ok: false,
|
|
1956
|
+
reason: `prior tree's blobs subtree is structurally invalid: ${priorBlobsEnumerated.reason}`,
|
|
1957
|
+
};
|
|
1958
|
+
}
|
|
1959
|
+
const prospectiveBlobPaths = new Set(blobsEnumerated.blobs.map((b) => b.blobPath));
|
|
1960
|
+
for (const b of priorBlobsEnumerated.blobs) {
|
|
1961
|
+
if (prospectiveBlobPaths.has(b.blobPath))
|
|
1962
|
+
continue;
|
|
1963
|
+
return {
|
|
1964
|
+
ok: false,
|
|
1965
|
+
reason: `blob ${b.blobPath} present in the prior tree is missing from the prospective tree; blob entries are immutable once written`,
|
|
1966
|
+
};
|
|
1967
|
+
}
|
|
1968
|
+
return { ok: true, newlyTerminalRuns };
|
|
1969
|
+
},
|
|
1970
|
+
onRefUpdated() {
|
|
1971
|
+
// No cached index today. Consumers read events through the
|
|
1972
|
+
// substrate's subscribe / blob-read API.
|
|
1973
|
+
},
|
|
1974
|
+
};
|
|
1975
|
+
export const workflowRunAuthorize = (principal, repoId, ref, action) => {
|
|
1976
|
+
if (repoId.kind !== "workflow-run") {
|
|
1977
|
+
return {
|
|
1978
|
+
allowed: false,
|
|
1979
|
+
reason: `workflow-run authorize received non-workflow-run repo ${repoId.kind}/${repoId.id}`,
|
|
1980
|
+
};
|
|
1981
|
+
}
|
|
1982
|
+
if (principal.kind === "hub") {
|
|
1983
|
+
return { allowed: true };
|
|
1984
|
+
}
|
|
1985
|
+
if (principal.kind === "workflow-process") {
|
|
1986
|
+
const parsed = WorkflowProcessPrincipal(principal);
|
|
1987
|
+
if (parsed instanceof type.errors) {
|
|
1988
|
+
return {
|
|
1989
|
+
allowed: false,
|
|
1990
|
+
reason: `workflow-process principal is malformed: ${parsed.summary}`,
|
|
1991
|
+
};
|
|
1992
|
+
}
|
|
1993
|
+
if (parsed.anchorRunId !== repoId.id) {
|
|
1994
|
+
return {
|
|
1995
|
+
allowed: false,
|
|
1996
|
+
reason: `workflow-process deployment ${parsed.anchorRunId} cannot access workflow-run ${repoId.id}`,
|
|
1997
|
+
};
|
|
1998
|
+
}
|
|
1999
|
+
switch (action) {
|
|
2000
|
+
case "init":
|
|
2001
|
+
case "writeTree":
|
|
2002
|
+
case "receivePack":
|
|
2003
|
+
case "createPack":
|
|
2004
|
+
case "resolveRef":
|
|
2005
|
+
return { allowed: true };
|
|
2006
|
+
default: {
|
|
2007
|
+
const _exhaustive = action;
|
|
2008
|
+
return {
|
|
2009
|
+
allowed: false,
|
|
2010
|
+
reason: `unhandled action: ${String(_exhaustive)}`,
|
|
2011
|
+
};
|
|
2012
|
+
}
|
|
2013
|
+
}
|
|
2014
|
+
}
|
|
2015
|
+
if (principal.kind === "supervisor") {
|
|
2016
|
+
const parsed = SupervisorPrincipal(principal);
|
|
2017
|
+
if (parsed instanceof type.errors) {
|
|
2018
|
+
return {
|
|
2019
|
+
allowed: false,
|
|
2020
|
+
reason: `supervisor principal is malformed: ${parsed.summary}`,
|
|
2021
|
+
};
|
|
2022
|
+
}
|
|
2023
|
+
if (parsed.anchorRunId !== repoId.id) {
|
|
2024
|
+
return {
|
|
2025
|
+
allowed: false,
|
|
2026
|
+
reason: `supervisor deployment ${parsed.anchorRunId} cannot access workflow-run ${repoId.id}`,
|
|
2027
|
+
};
|
|
2028
|
+
}
|
|
2029
|
+
switch (action) {
|
|
2030
|
+
case "init":
|
|
2031
|
+
case "writeTree":
|
|
2032
|
+
case "receivePack":
|
|
2033
|
+
case "createPack":
|
|
2034
|
+
case "resolveRef":
|
|
2035
|
+
return { allowed: true };
|
|
2036
|
+
default: {
|
|
2037
|
+
const _exhaustive = action;
|
|
2038
|
+
return {
|
|
2039
|
+
allowed: false,
|
|
2040
|
+
reason: `unhandled action: ${String(_exhaustive)}`,
|
|
2041
|
+
};
|
|
2042
|
+
}
|
|
2043
|
+
}
|
|
2044
|
+
}
|
|
2045
|
+
if (principal.kind === "sidecar") {
|
|
2046
|
+
const parsed = SidecarPrincipal(principal);
|
|
2047
|
+
if (parsed instanceof type.errors) {
|
|
2048
|
+
return {
|
|
2049
|
+
allowed: false,
|
|
2050
|
+
reason: `sidecar principal is malformed: ${parsed.summary}`,
|
|
2051
|
+
};
|
|
2052
|
+
}
|
|
2053
|
+
switch (action) {
|
|
2054
|
+
case "createPack":
|
|
2055
|
+
case "resolveRef":
|
|
2056
|
+
return { allowed: true };
|
|
2057
|
+
case "init":
|
|
2058
|
+
case "writeTree":
|
|
2059
|
+
case "receivePack":
|
|
2060
|
+
return {
|
|
2061
|
+
allowed: false,
|
|
2062
|
+
reason: `sidecars may only read workflow-run repos, not ${action}`,
|
|
2063
|
+
};
|
|
2064
|
+
default: {
|
|
2065
|
+
const _exhaustive = action;
|
|
2066
|
+
return {
|
|
2067
|
+
allowed: false,
|
|
2068
|
+
reason: `unhandled action: ${String(_exhaustive)}`,
|
|
2069
|
+
};
|
|
2070
|
+
}
|
|
2071
|
+
}
|
|
2072
|
+
}
|
|
2073
|
+
if (principal.kind === "user") {
|
|
2074
|
+
// The route layer has already pre-resolved the grant verdict and
|
|
2075
|
+
// attached it as `authz`. The substrate does NOT re-query the
|
|
2076
|
+
// grant store here; it (a) checks the bearer-token's claims
|
|
2077
|
+
// bound the requested (ref, action) and have not expired, and
|
|
2078
|
+
// (b) sanity-checks that the pre-resolved verdict targets this
|
|
2079
|
+
// exact resource and grant verb. Both gates must pass before the
|
|
2080
|
+
// verdict's `effect` is honoured.
|
|
2081
|
+
const parsed = UserPrincipal(principal);
|
|
2082
|
+
if (parsed instanceof type.errors) {
|
|
2083
|
+
return {
|
|
2084
|
+
allowed: false,
|
|
2085
|
+
reason: `user principal is malformed: ${parsed.summary}`,
|
|
2086
|
+
};
|
|
2087
|
+
}
|
|
2088
|
+
if (!parsed.tokenClaims.actions.includes(action)) {
|
|
2089
|
+
return {
|
|
2090
|
+
allowed: false,
|
|
2091
|
+
reason: `token does not grant action ${action}`,
|
|
2092
|
+
};
|
|
2093
|
+
}
|
|
2094
|
+
// `ref === "*"` is the substrate's sentinel for the bulk read
|
|
2095
|
+
// performed by `listRefs`. Per-ref filtering is the advertise-refs
|
|
2096
|
+
// layer's responsibility, so the bulk read is gated on action and
|
|
2097
|
+
// expiry alone.
|
|
2098
|
+
if (ref !== "*" && !glob.match(parsed.tokenClaims.refPattern, ref)) {
|
|
2099
|
+
return {
|
|
2100
|
+
allowed: false,
|
|
2101
|
+
reason: `token refPattern ${parsed.tokenClaims.refPattern} does not match ${ref}`,
|
|
2102
|
+
};
|
|
2103
|
+
}
|
|
2104
|
+
if (Date.now() >= parsed.tokenClaims.expiresAt) {
|
|
2105
|
+
return {
|
|
2106
|
+
allowed: false,
|
|
2107
|
+
reason: `token expired at ${parsed.tokenClaims.expiresAt}`,
|
|
2108
|
+
};
|
|
2109
|
+
}
|
|
2110
|
+
const expectedResource = `workflow-run:${repoId.id}`;
|
|
2111
|
+
if (parsed.authz.resource !== expectedResource) {
|
|
2112
|
+
return {
|
|
2113
|
+
allowed: false,
|
|
2114
|
+
reason: `authz verdict resource ${parsed.authz.resource} does not match ${expectedResource}`,
|
|
2115
|
+
};
|
|
2116
|
+
}
|
|
2117
|
+
const expectedGrantVerb = repoActionToGrantVerb(action);
|
|
2118
|
+
if (parsed.authz.grantVerb !== expectedGrantVerb) {
|
|
2119
|
+
return {
|
|
2120
|
+
allowed: false,
|
|
2121
|
+
reason: `authz verdict grantVerb ${parsed.authz.grantVerb} does not match ${expectedGrantVerb}`,
|
|
2122
|
+
};
|
|
2123
|
+
}
|
|
2124
|
+
if (parsed.authz.effect === "allow") {
|
|
2125
|
+
return { allowed: true };
|
|
2126
|
+
}
|
|
2127
|
+
return {
|
|
2128
|
+
allowed: false,
|
|
2129
|
+
reason: `authz verdict denied for ${expectedResource} ${expectedGrantVerb}`,
|
|
2130
|
+
};
|
|
2131
|
+
}
|
|
2132
|
+
// Fail closed on any kind not handled above. The tenant-level
|
|
2133
|
+
// `workflow` principal kind (`@intx/types` principalKinds) is a
|
|
2134
|
+
// grant owner, not a workflow-run repo bearer, and never carries a
|
|
2135
|
+
// workflow-run push here -- so it is intentionally left denied.
|
|
2136
|
+
return {
|
|
2137
|
+
allowed: false,
|
|
2138
|
+
reason: `unknown principal kind: ${principal.kind}`,
|
|
2139
|
+
};
|
|
2140
|
+
};
|
|
2141
|
+
// ---------------------------------------------------------------------
|
|
2142
|
+
// Claim-check API.
|
|
2143
|
+
//
|
|
2144
|
+
// Four operations layer on top of `RepoStore.writeTreeDelta` to give
|
|
2145
|
+
// the workflow runtime a FIFO claim-check queue per address:
|
|
2146
|
+
//
|
|
2147
|
+
// enqueueInbox — append a new inbox entry for an inbound
|
|
2148
|
+
// message.
|
|
2149
|
+
// dequeueToProcessing — pick the lexicographically-first inbox
|
|
2150
|
+
// entry and atomically move it to
|
|
2151
|
+
// processing.
|
|
2152
|
+
// markConsumed — atomically remove the processing entry
|
|
2153
|
+
// and write the canonical
|
|
2154
|
+
// consumed/<messageId>.json dedup index
|
|
2155
|
+
// entry.
|
|
2156
|
+
// replayProcessingToInbox — recovery path that moves every
|
|
2157
|
+
// processing entry back to inbox preserving
|
|
2158
|
+
// its `<receivedAt>-<messageId>` filename
|
|
2159
|
+
// key so FIFO ordering survives a crash.
|
|
2160
|
+
//
|
|
2161
|
+
// All four route through `writeTreeDelta`, scoped to the per-address
|
|
2162
|
+
// subtree via `changedPathPrefixes`. The substrate serializes concurrent
|
|
2163
|
+
// claim-check operations on the per-repo lock and invokes each
|
|
2164
|
+
// operation's `computeDelta` callback with a `prior` view of the
|
|
2165
|
+
// committed tree. The callback reads only what it needs directly --
|
|
2166
|
+
// `prior.listDirOids` for a directory's names and OIDs, and
|
|
2167
|
+
// `prior.readBlobByOid` for a specific entry's bytes -- and returns a
|
|
2168
|
+
// TARGETED delta (the `puts` and `deletes` for the paths that change),
|
|
2169
|
+
// not the full subtree. The substrate applies that delta atomically over
|
|
2170
|
+
// the prior tree, carrying every untouched entry forward by OID and
|
|
2171
|
+
// landing the whole delta in a single commit, which is the atomic-commit
|
|
2172
|
+
// guarantee these operations require.
|
|
2173
|
+
function claimCheckCommitRef() {
|
|
2174
|
+
// Every claim-check operation targets the same canonical ref used by
|
|
2175
|
+
// the workflow-run kind handler's event log so subscribers see a
|
|
2176
|
+
// single coherent commit stream.
|
|
2177
|
+
return "refs/heads/events";
|
|
2178
|
+
}
|
|
2179
|
+
function addressSegmentFor(address) {
|
|
2180
|
+
// The substrate boundary is the only place URL-encoding happens.
|
|
2181
|
+
// `validatePush` rejects non-round-trip segments; mirroring the same
|
|
2182
|
+
// encoder here is the only legitimate way to produce one.
|
|
2183
|
+
return encodeURIComponent(address);
|
|
2184
|
+
}
|
|
2185
|
+
function addressPrefix(addressSegment) {
|
|
2186
|
+
return `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/`;
|
|
2187
|
+
}
|
|
2188
|
+
function inboxPath(addressSegment, key) {
|
|
2189
|
+
return `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_INBOX_DIR}/${key}.json`;
|
|
2190
|
+
}
|
|
2191
|
+
function processingPath(addressSegment, key) {
|
|
2192
|
+
return `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_PROCESSING_DIR}/${key}.json`;
|
|
2193
|
+
}
|
|
2194
|
+
function consumedPath(addressSegment, messageId) {
|
|
2195
|
+
return `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_CONSUMED_DIR}/${messageId}.json`;
|
|
2196
|
+
}
|
|
2197
|
+
function watermarkPath(addressSegment) {
|
|
2198
|
+
return `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_WATERMARK_FILE}`;
|
|
2199
|
+
}
|
|
2200
|
+
function filenameKey(receivedAt, messageId) {
|
|
2201
|
+
return `${String(receivedAt)}-${messageId}`;
|
|
2202
|
+
}
|
|
2203
|
+
/**
|
|
2204
|
+
* Read one address's claim-check listing from the parent commit: the
|
|
2205
|
+
* filenames and blob OIDs directly under
|
|
2206
|
+
* `addresses/<addressSegment>/{inbox,processing,consumed}/` (NOT their
|
|
2207
|
+
* bytes), plus the retention watermark. The bytes of the single entry a
|
|
2208
|
+
* leg actually moves are read separately by OID via
|
|
2209
|
+
* `prior.readBlobByOid`, so the unbounded consumed/ dedup index is
|
|
2210
|
+
* enumerated (one `listDirOids` per bucket, names and OIDs only) but
|
|
2211
|
+
* never read blob-by-blob. Every read goes through the store's
|
|
2212
|
+
* cache-backed `prior` closures under the write lock. An empty listing
|
|
2213
|
+
* covers the repo/ref/address-absent first-write states -- all
|
|
2214
|
+
* legitimate for a brand-new operation.
|
|
2215
|
+
*/
|
|
2216
|
+
async function readAddressListing(prior, addressSegment) {
|
|
2217
|
+
const listing = {
|
|
2218
|
+
inbox: [],
|
|
2219
|
+
processing: [],
|
|
2220
|
+
consumed: [],
|
|
2221
|
+
watermark: 0,
|
|
2222
|
+
};
|
|
2223
|
+
const addrDir = `${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}`;
|
|
2224
|
+
for (const child of await prior.listDirOids(addrDir)) {
|
|
2225
|
+
if (child.name === WORKFLOW_RUN_WATERMARK_FILE) {
|
|
2226
|
+
const blob = await prior.readBlobByOid(child.oid);
|
|
2227
|
+
listing.watermark = parseWatermark(blob, watermarkPath(addressSegment));
|
|
2228
|
+
continue;
|
|
2229
|
+
}
|
|
2230
|
+
const bucket = child.name === WORKFLOW_RUN_INBOX_DIR
|
|
2231
|
+
? listing.inbox
|
|
2232
|
+
: child.name === WORKFLOW_RUN_PROCESSING_DIR
|
|
2233
|
+
? listing.processing
|
|
2234
|
+
: child.name === WORKFLOW_RUN_CONSUMED_DIR
|
|
2235
|
+
? listing.consumed
|
|
2236
|
+
: null;
|
|
2237
|
+
if (bucket === null)
|
|
2238
|
+
continue;
|
|
2239
|
+
for (const entry of await prior.listDirOids(`${addrDir}/${child.name}`)) {
|
|
2240
|
+
bucket.push({ name: entry.name, oid: entry.oid });
|
|
2241
|
+
}
|
|
2242
|
+
}
|
|
2243
|
+
return listing;
|
|
2244
|
+
}
|
|
2245
|
+
function utf8(s) {
|
|
2246
|
+
return new TextEncoder().encode(s);
|
|
2247
|
+
}
|
|
2248
|
+
function decodeQueueEnvelopeOrThrow(bytes, blobPath) {
|
|
2249
|
+
let parsed;
|
|
2250
|
+
try {
|
|
2251
|
+
parsed = JSON.parse(new TextDecoder().decode(bytes));
|
|
2252
|
+
}
|
|
2253
|
+
catch (cause) {
|
|
2254
|
+
throw new Error(`claim_check_corrupt_json: ${blobPath}`, { cause });
|
|
2255
|
+
}
|
|
2256
|
+
const validated = ClaimCheckEnvelope(parsed);
|
|
2257
|
+
if (validated instanceof type.errors) {
|
|
2258
|
+
throw new Error(`claim_check_envelope_invalid: ${blobPath}: ${validated.summary}`);
|
|
2259
|
+
}
|
|
2260
|
+
return validated;
|
|
2261
|
+
}
|
|
2262
|
+
function decodeConsumedReceivedAtOrThrow(bytes, blobPath) {
|
|
2263
|
+
let parsed;
|
|
2264
|
+
try {
|
|
2265
|
+
parsed = JSON.parse(new TextDecoder().decode(bytes));
|
|
2266
|
+
}
|
|
2267
|
+
catch (cause) {
|
|
2268
|
+
throw new Error(`claim_check_corrupt_json: ${blobPath}`, { cause });
|
|
2269
|
+
}
|
|
2270
|
+
const validated = ConsumedEnvelope(parsed);
|
|
2271
|
+
if (validated instanceof type.errors) {
|
|
2272
|
+
throw new Error(`claim_check_consumed_invalid: ${blobPath}: ${validated.summary}`);
|
|
2273
|
+
}
|
|
2274
|
+
return validated.receivedAt;
|
|
2275
|
+
}
|
|
2276
|
+
/**
|
|
2277
|
+
* Decode the per-address retention watermark from its blob bytes. The
|
|
2278
|
+
* caller treats an absent watermark blob as 0 (the address has never
|
|
2279
|
+
* pruned; nothing refused).
|
|
2280
|
+
*/
|
|
2281
|
+
function parseWatermark(bytes, watermarkFull) {
|
|
2282
|
+
let parsed;
|
|
2283
|
+
try {
|
|
2284
|
+
parsed = JSON.parse(new TextDecoder().decode(bytes));
|
|
2285
|
+
}
|
|
2286
|
+
catch (cause) {
|
|
2287
|
+
throw new Error(`claim_check_corrupt_json: ${watermarkFull}`, { cause });
|
|
2288
|
+
}
|
|
2289
|
+
const validated = WatermarkEnvelope(parsed);
|
|
2290
|
+
if (validated instanceof type.errors) {
|
|
2291
|
+
throw new Error(`claim_check_watermark_invalid: ${watermarkFull}: ${validated.summary}`);
|
|
2292
|
+
}
|
|
2293
|
+
return validated.watermark;
|
|
2294
|
+
}
|
|
2295
|
+
/**
|
|
2296
|
+
* Internal signal thrown from the `enqueueInbox` merge callback when the
|
|
2297
|
+
* messageId is already present in a queue state. Caught at the `enqueueInbox`
|
|
2298
|
+
* boundary and turned into an `already-present` outcome; never escapes. It
|
|
2299
|
+
* carries the specific `reason` so the boundary maps it without re-deriving.
|
|
2300
|
+
*/
|
|
2301
|
+
class InboxEntryAlreadyPresent extends Error {
|
|
2302
|
+
reason;
|
|
2303
|
+
constructor(reason, message) {
|
|
2304
|
+
super(message);
|
|
2305
|
+
this.reason = reason;
|
|
2306
|
+
this.name = "InboxEntryAlreadyPresent";
|
|
2307
|
+
}
|
|
2308
|
+
}
|
|
2309
|
+
/**
|
|
2310
|
+
* Thrown by `enqueueInbox` when the inbound's `receivedAt` is strictly below
|
|
2311
|
+
* the address's retention watermark. This is refusal under UNCERTAINTY, not
|
|
2312
|
+
* proof of prior receipt: the consumed dedup entry that would rule out a
|
|
2313
|
+
* duplicate may have been pruned, so the substrate can no longer tell a
|
|
2314
|
+
* duplicate from a never-processed message and refuses rather than risk
|
|
2315
|
+
* reprocessing. A caller gating a durable-receipt ack MUST NOT acknowledge on
|
|
2316
|
+
* this -- acking an "I cannot tell" would terminally drop a message that was
|
|
2317
|
+
* never written. It is its own type (not a generic Error) so that a caller,
|
|
2318
|
+
* and monitoring, can surface it as a distinct loud signal rather than
|
|
2319
|
+
* blending it into ordinary I/O-failure noise.
|
|
2320
|
+
*
|
|
2321
|
+
* Structurally unreachable on the mail-inbound path today: `enqueueInbox` is
|
|
2322
|
+
* only ever called with a freshly stamped `receivedAt` (a redelivery
|
|
2323
|
+
* re-stamps `Date.now()` rather than carrying the original), and the watermark
|
|
2324
|
+
* only ever advances to at most `consumedAt - retentionHorizonMs <= now`, so a
|
|
2325
|
+
* fresh `receivedAt` sits a full horizon above it. The sole path that carries
|
|
2326
|
+
* an original (old) `receivedAt` back into the queue is
|
|
2327
|
+
* `replayProcessingToInbox`, which writes straight to `inbox/` and bypasses
|
|
2328
|
+
* this gate entirely. If any redelivery source is ever changed to carry the
|
|
2329
|
+
* original `receivedAt` into `enqueueInbox`, this becomes reachable and its
|
|
2330
|
+
* withhold-not-ack handling becomes load-bearing.
|
|
2331
|
+
*/
|
|
2332
|
+
export class StaleInboxEnqueueError extends Error {
|
|
2333
|
+
constructor(message) {
|
|
2334
|
+
super(message);
|
|
2335
|
+
this.name = "StaleInboxEnqueueError";
|
|
2336
|
+
}
|
|
2337
|
+
}
|
|
2338
|
+
/**
|
|
2339
|
+
* Append a new inbox entry for `address`. The merge callback reads
|
|
2340
|
+
* the address subtree under the per-repo lock, augments the inbox
|
|
2341
|
+
* with the new entry, and returns the full set of address files. The
|
|
2342
|
+
* substrate replaces the address subtree wholesale.
|
|
2343
|
+
*
|
|
2344
|
+
* Rejects if a same-messageId entry already exists in any queue
|
|
2345
|
+
* state at the address — including a prior inbox entry at a
|
|
2346
|
+
* different `receivedAt`. The caller is expected to consult the
|
|
2347
|
+
* dedup index (consumed/) before calling, but enforcing the
|
|
2348
|
+
* invariant here also catches the concurrent-enqueue race that the
|
|
2349
|
+
* per-repo lock alone cannot surface.
|
|
2350
|
+
*/
|
|
2351
|
+
export async function enqueueInbox(store, principal, repoId, args) {
|
|
2352
|
+
const addressSegment = addressSegmentFor(args.address);
|
|
2353
|
+
const ref = claimCheckCommitRef();
|
|
2354
|
+
const inboxKey = filenameKey(args.receivedAt, args.messageId);
|
|
2355
|
+
const envelope = {
|
|
2356
|
+
messageId: args.messageId,
|
|
2357
|
+
receivedAt: args.receivedAt,
|
|
2358
|
+
address: args.address,
|
|
2359
|
+
mailAuditRef: args.mailAuditRef,
|
|
2360
|
+
...(args.rawMessage !== undefined ? { rawMessage: args.rawMessage } : {}),
|
|
2361
|
+
};
|
|
2362
|
+
const newInboxPath = inboxPath(addressSegment, inboxKey);
|
|
2363
|
+
const inboxFname = `${inboxKey}.json`;
|
|
2364
|
+
const consumedFname = `${args.messageId}.json`;
|
|
2365
|
+
const messageIdSuffix = `-${args.messageId}.json`;
|
|
2366
|
+
// The already-present cases throw `InboxEntryAlreadyPresent` from the merge
|
|
2367
|
+
// callback and are caught here into an `already-present` outcome; the
|
|
2368
|
+
// stale-refusal throws `StaleInboxEnqueueError`, and a substrate/I/O failure
|
|
2369
|
+
// throws a generic error -- both of those propagate. The return/throw split
|
|
2370
|
+
// is deliberate and load-bearing: it is the ack/withhold boundary for a
|
|
2371
|
+
// caller gating a durable-receipt ack (return = safe to ack, throw =
|
|
2372
|
+
// withhold), so `stale` sits with I/O on the throw side, NOT with the
|
|
2373
|
+
// already-present cases (see `StaleInboxEnqueueError`).
|
|
2374
|
+
let commitSha;
|
|
2375
|
+
try {
|
|
2376
|
+
({ commitSha } = await store.writeTreeDelta(principal, repoId, ref, {
|
|
2377
|
+
changedPathPrefixes: new Set([addressPrefix(addressSegment)]),
|
|
2378
|
+
message: `enqueue inbox ${args.address} ${args.messageId}`,
|
|
2379
|
+
computeDelta: async (_parentCommitSha, prior) => {
|
|
2380
|
+
const listing = await readAddressListing(prior, addressSegment);
|
|
2381
|
+
// Refuse a definitively-stale enqueue: a message whose receivedAt
|
|
2382
|
+
// is strictly below the retention watermark could have had its
|
|
2383
|
+
// consumed/ dedup entry pruned, so a duplicate can no longer be
|
|
2384
|
+
// ruled out. Reject it LOUDLY rather than risk reprocessing. This
|
|
2385
|
+
// is the second half of the exactly-once guarantee: above the
|
|
2386
|
+
// watermark the consumed/ index is authoritative; below it, refuse.
|
|
2387
|
+
if (args.receivedAt < listing.watermark) {
|
|
2388
|
+
throw new StaleInboxEnqueueError(`claim_check_stale_enqueue: address ${args.address} message ${args.messageId} receivedAt ${String(args.receivedAt)} is below the retention watermark ${String(listing.watermark)}; its dedup entry may have been pruned, so it is refused as definitively-stale`);
|
|
2389
|
+
}
|
|
2390
|
+
if (listing.inbox.some((e) => e.name === inboxFname)) {
|
|
2391
|
+
throw new InboxEntryAlreadyPresent("duplicate", `claim_check_duplicate_inbox: ${newInboxPath} already exists`);
|
|
2392
|
+
}
|
|
2393
|
+
// consumed/ is keyed by messageId alone, so this is an exact
|
|
2394
|
+
// filename lookup against the dedup index.
|
|
2395
|
+
if (listing.consumed.some((e) => e.name === consumedFname)) {
|
|
2396
|
+
throw new InboxEntryAlreadyPresent("consumed", `claim_check_already_consumed: address ${args.address} message ${args.messageId} is already in the consumed dedup index`);
|
|
2397
|
+
}
|
|
2398
|
+
if (listing.processing.some((e) => e.name.endsWith(messageIdSuffix))) {
|
|
2399
|
+
throw new InboxEntryAlreadyPresent("processing", `claim_check_already_processing: address ${args.address} message ${args.messageId} is currently in processing`);
|
|
2400
|
+
}
|
|
2401
|
+
// Reject a second inbox entry for the same messageId at a
|
|
2402
|
+
// different receivedAt. The validatePush atomicity check also
|
|
2403
|
+
// catches this on the commit path, but surfacing it here gives the
|
|
2404
|
+
// caller a precise error and keeps the bad tree off the substrate.
|
|
2405
|
+
const inboxDup = listing.inbox.find((e) => e.name.endsWith(messageIdSuffix));
|
|
2406
|
+
if (inboxDup !== undefined) {
|
|
2407
|
+
throw new InboxEntryAlreadyPresent("already_inbox", `claim_check_already_inbox: address ${args.address} message ${args.messageId} is already in the inbox at ${inboxPath(addressSegment, inboxDup.name.slice(0, -".json".length))}`);
|
|
2408
|
+
}
|
|
2409
|
+
return {
|
|
2410
|
+
puts: { [newInboxPath]: utf8(JSON.stringify(envelope)) },
|
|
2411
|
+
deletes: [],
|
|
2412
|
+
};
|
|
2413
|
+
},
|
|
2414
|
+
}));
|
|
2415
|
+
}
|
|
2416
|
+
catch (err) {
|
|
2417
|
+
if (err instanceof InboxEntryAlreadyPresent) {
|
|
2418
|
+
return { outcome: "already-present", reason: err.reason };
|
|
2419
|
+
}
|
|
2420
|
+
throw err;
|
|
2421
|
+
}
|
|
2422
|
+
return { outcome: "enqueued", commitSha, inboxKey, envelope };
|
|
2423
|
+
}
|
|
2424
|
+
/**
|
|
2425
|
+
* Move the FIFO-first inbox entry for `address` to processing.
|
|
2426
|
+
* Returns `null` when the inbox is empty so the caller can
|
|
2427
|
+
* distinguish "nothing to do" from "operation failed".
|
|
2428
|
+
*
|
|
2429
|
+
* FIFO is keyed on the parsed numeric `receivedAt` prefix of the
|
|
2430
|
+
* inbox filename, with a lexicographic messageId tiebreak. The
|
|
2431
|
+
* substrate does NOT rely on uniform digit widths — sorting raw
|
|
2432
|
+
* filenames would put `"100-…"` ahead of `"99-…"` since `'1' < '9'`,
|
|
2433
|
+
* which violates the FIFO invariant.
|
|
2434
|
+
*/
|
|
2435
|
+
export async function dequeueToProcessing(store, principal, repoId, address) {
|
|
2436
|
+
const addressSegment = addressSegmentFor(address);
|
|
2437
|
+
const ref = claimCheckCommitRef();
|
|
2438
|
+
let dequeued = null;
|
|
2439
|
+
const { commitSha } = await store.writeTreeDelta(principal, repoId, ref, {
|
|
2440
|
+
changedPathPrefixes: new Set([addressPrefix(addressSegment)]),
|
|
2441
|
+
message: `dequeue ${address}`,
|
|
2442
|
+
computeDelta: async (_parentCommitSha, prior) => {
|
|
2443
|
+
const listing = await readAddressListing(prior, addressSegment);
|
|
2444
|
+
const inboxDir = `${addressPrefix(addressSegment)}${WORKFLOW_RUN_INBOX_DIR}/`;
|
|
2445
|
+
const candidates = [];
|
|
2446
|
+
for (const entry of listing.inbox) {
|
|
2447
|
+
const m = QUEUE_FILENAME_RE.exec(entry.name);
|
|
2448
|
+
if (m === null || m[1] === undefined || m[2] === undefined) {
|
|
2449
|
+
throw new Error(`claim_check_invalid_inbox_filename: ${inboxDir}${entry.name}`);
|
|
2450
|
+
}
|
|
2451
|
+
candidates.push({
|
|
2452
|
+
entry,
|
|
2453
|
+
receivedAt: Number.parseInt(m[1], 10),
|
|
2454
|
+
messageId: m[2],
|
|
2455
|
+
});
|
|
2456
|
+
}
|
|
2457
|
+
candidates.sort((a, b) => {
|
|
2458
|
+
if (a.receivedAt !== b.receivedAt)
|
|
2459
|
+
return a.receivedAt - b.receivedAt;
|
|
2460
|
+
if (a.messageId < b.messageId)
|
|
2461
|
+
return -1;
|
|
2462
|
+
if (a.messageId > b.messageId)
|
|
2463
|
+
return 1;
|
|
2464
|
+
return 0;
|
|
2465
|
+
});
|
|
2466
|
+
const first = candidates[0];
|
|
2467
|
+
if (first === undefined) {
|
|
2468
|
+
// Empty inbox: nothing to move. The commit is a no-op rewrite of
|
|
2469
|
+
// the same tree; the caller reads `dequeued === null`.
|
|
2470
|
+
dequeued = null;
|
|
2471
|
+
return { puts: {}, deletes: [] };
|
|
2472
|
+
}
|
|
2473
|
+
const firstPath = `${inboxDir}${first.entry.name}`;
|
|
2474
|
+
const key = first.entry.name.slice(0, -".json".length);
|
|
2475
|
+
const bytes = await prior.readBlobByOid(first.entry.oid);
|
|
2476
|
+
const envelope = decodeQueueEnvelopeOrThrow(bytes, firstPath);
|
|
2477
|
+
dequeued = { key, envelope };
|
|
2478
|
+
return {
|
|
2479
|
+
puts: { [processingPath(addressSegment, key)]: bytes },
|
|
2480
|
+
deletes: [firstPath],
|
|
2481
|
+
};
|
|
2482
|
+
},
|
|
2483
|
+
});
|
|
2484
|
+
if (dequeued === null)
|
|
2485
|
+
return null;
|
|
2486
|
+
const captured = dequeued;
|
|
2487
|
+
return { commitSha, key: captured.key, envelope: captured.envelope };
|
|
2488
|
+
}
|
|
2489
|
+
/**
|
|
2490
|
+
* Read the processing-queue entry for `messageId` at `address` without
|
|
2491
|
+
* mutating the tree. Returns the decoded claim-check envelope (carrying
|
|
2492
|
+
* `mailAuditRef` and, when the enqueuer inlined them, the base64
|
|
2493
|
+
* `rawMessage` bytes) or `null` when no processing entry exists for the
|
|
2494
|
+
* messageId.
|
|
2495
|
+
*
|
|
2496
|
+
* This is the read half of mailbox ownership (§3a): the supervisor's
|
|
2497
|
+
* dispatch loop moves an inbox entry to processing and forwards a
|
|
2498
|
+
* `trigger.fired{messageId}` to the workflow-process child; the child
|
|
2499
|
+
* calls this to recover the inbound message bytes that become its step
|
|
2500
|
+
* input.
|
|
2501
|
+
*
|
|
2502
|
+
* The read is a flat working-tree read of
|
|
2503
|
+
* `addresses/<seg>/processing/`. The substrate materializes each
|
|
2504
|
+
* claim-check commit's touched paths into the repo's working tree (the
|
|
2505
|
+
* delta write removes each deleted path and writes each put after
|
|
2506
|
+
* validation passes), so a read issued after `dequeueToProcessing`
|
|
2507
|
+
* committed -- which is exactly when the supervisor forwards
|
|
2508
|
+
* `trigger.fired` -- observes the processing entry. Reading the working tree (rather than walking the
|
|
2509
|
+
* committed git tree) matches the workflow-process child's sibling
|
|
2510
|
+
* read of `runs/<runId>/events/`. Because the
|
|
2511
|
+
* read issues no commit it cannot race the supervisor's `markConsumed`
|
|
2512
|
+
* write; it returns a point-in-time snapshot of the directory.
|
|
2513
|
+
*/
|
|
2514
|
+
export async function readProcessingEntry(store, _principal, repoId, address, messageId) {
|
|
2515
|
+
const addressSegment = addressSegmentFor(address);
|
|
2516
|
+
const repoDir = store.getRepoDir(repoId);
|
|
2517
|
+
const processingDir = `${repoDir}/${WORKFLOW_RUN_ADDRESSES_PREFIX}/${addressSegment}/${WORKFLOW_RUN_PROCESSING_DIR}`;
|
|
2518
|
+
const suffix = `-${messageId}.json`;
|
|
2519
|
+
let filenames;
|
|
2520
|
+
try {
|
|
2521
|
+
filenames = await fs.promises.readdir(processingDir);
|
|
2522
|
+
}
|
|
2523
|
+
catch (cause) {
|
|
2524
|
+
// A missing processing directory is the legitimate "no entry yet"
|
|
2525
|
+
// state; any other failure surfaces.
|
|
2526
|
+
if (cause instanceof Error &&
|
|
2527
|
+
cause.code === "ENOENT") {
|
|
2528
|
+
return null;
|
|
2529
|
+
}
|
|
2530
|
+
throw cause;
|
|
2531
|
+
}
|
|
2532
|
+
for (const filename of filenames) {
|
|
2533
|
+
if (!filename.endsWith(suffix))
|
|
2534
|
+
continue;
|
|
2535
|
+
const blobPath = `${processingDir}/${filename}`;
|
|
2536
|
+
const bytes = await fs.promises.readFile(blobPath);
|
|
2537
|
+
const envelope = decodeQueueEnvelopeOrThrow(new Uint8Array(bytes), blobPath);
|
|
2538
|
+
return { envelope };
|
|
2539
|
+
}
|
|
2540
|
+
return null;
|
|
2541
|
+
}
|
|
2542
|
+
/**
|
|
2543
|
+
* Atomically remove the processing entry for `messageId` at `address`,
|
|
2544
|
+
* write the canonical `consumed/<messageId>.json` dedup index entry,
|
|
2545
|
+
* advance the per-address retention watermark, and prune consumed
|
|
2546
|
+
* entries the watermark has passed. The caller is expected to have
|
|
2547
|
+
* called `dequeueToProcessing` for this messageId; calling
|
|
2548
|
+
* `markConsumed` without a matching processing entry throws.
|
|
2549
|
+
*
|
|
2550
|
+
* The consumed envelope preserves the original `receivedAt` and
|
|
2551
|
+
* `mailAuditRef` from the processing entry so the dedup index doubles
|
|
2552
|
+
* as an audit record.
|
|
2553
|
+
*
|
|
2554
|
+
* Retention (the bounded-`consumed/` contract): the watermark advances
|
|
2555
|
+
* to `max(priorWatermark, min(consumedAt - retentionHorizonMs,
|
|
2556
|
+
* thisEntry.receivedAt))` -- monotonic, and never past the entry being
|
|
2557
|
+
* written so the new entry is always retained. Every consumed entry
|
|
2558
|
+
* whose `receivedAt` is strictly below the new watermark is dropped
|
|
2559
|
+
* (the oldest age-ordered tail). `consumed/` therefore reaches a
|
|
2560
|
+
* bounded steady state of roughly one horizon's worth of entries
|
|
2561
|
+
* instead of growing one entry per message forever.
|
|
2562
|
+
*/
|
|
2563
|
+
export async function markConsumed(store, principal, repoId, args) {
|
|
2564
|
+
const addressSegment = addressSegmentFor(args.address);
|
|
2565
|
+
const ref = claimCheckCommitRef();
|
|
2566
|
+
const retentionHorizonMs = args.retentionHorizonMs ?? DEFAULT_CONSUMED_RETENTION_MS;
|
|
2567
|
+
let consumedEnvelope = null;
|
|
2568
|
+
let advancedWatermark = 0;
|
|
2569
|
+
const prunedMessageIds = [];
|
|
2570
|
+
const { commitSha } = await store.writeTreeDelta(principal, repoId, ref, {
|
|
2571
|
+
changedPathPrefixes: new Set([addressPrefix(addressSegment)]),
|
|
2572
|
+
message: `consume ${args.address} ${args.messageId}`,
|
|
2573
|
+
computeDelta: async (_parentCommitSha, prior) => {
|
|
2574
|
+
const listing = await readAddressListing(prior, addressSegment);
|
|
2575
|
+
const consumedFull = consumedPath(addressSegment, args.messageId);
|
|
2576
|
+
const consumedFname = `${args.messageId}.json`;
|
|
2577
|
+
if (listing.consumed.some((e) => e.name === consumedFname)) {
|
|
2578
|
+
throw new Error(`claim_check_already_consumed: ${consumedFull} already in the dedup index`);
|
|
2579
|
+
}
|
|
2580
|
+
const processingDir = `${addressPrefix(addressSegment)}${WORKFLOW_RUN_PROCESSING_DIR}/`;
|
|
2581
|
+
const processingEntry = listing.processing.find((e) => e.name.endsWith(`-${args.messageId}.json`));
|
|
2582
|
+
if (processingEntry === undefined) {
|
|
2583
|
+
throw new Error(`claim_check_processing_not_found: address ${args.address} message ${args.messageId} has no processing entry`);
|
|
2584
|
+
}
|
|
2585
|
+
const processingFull = `${processingDir}${processingEntry.name}`;
|
|
2586
|
+
const processingBytes = await prior.readBlobByOid(processingEntry.oid);
|
|
2587
|
+
const processingEnvelope = decodeQueueEnvelopeOrThrow(processingBytes, processingFull);
|
|
2588
|
+
const envelope = {
|
|
2589
|
+
messageId: args.messageId,
|
|
2590
|
+
receivedAt: processingEnvelope.receivedAt,
|
|
2591
|
+
address: args.address,
|
|
2592
|
+
runId: args.runId,
|
|
2593
|
+
consumedAt: args.consumedAt,
|
|
2594
|
+
mailAuditRef: processingEnvelope.mailAuditRef,
|
|
2595
|
+
...(args.rejection !== undefined ? { rejection: args.rejection } : {}),
|
|
2596
|
+
};
|
|
2597
|
+
consumedEnvelope = envelope;
|
|
2598
|
+
// The watermark may only advance, and never past the entry this
|
|
2599
|
+
// commit writes (so the new entry is always retained -- a message
|
|
2600
|
+
// consumed long after receipt may legitimately sit below
|
|
2601
|
+
// `consumedAt - horizon`, and it is pruned on a later commit once
|
|
2602
|
+
// the watermark passes ITS receivedAt).
|
|
2603
|
+
const horizonBoundary = args.consumedAt - retentionHorizonMs;
|
|
2604
|
+
const newWatermark = Math.max(listing.watermark, Math.min(horizonBoundary, envelope.receivedAt));
|
|
2605
|
+
advancedWatermark = newWatermark;
|
|
2606
|
+
// Prune the oldest consumed tail: read each retained consumed
|
|
2607
|
+
// entry's receivedAt and drop any that has fallen strictly below
|
|
2608
|
+
// the new watermark. This is the one leg that must scan the
|
|
2609
|
+
// consumed index — its filenames carry only the messageId, so the
|
|
2610
|
+
// receivedAt lives in the bytes — and is the residual the
|
|
2611
|
+
// consumed-shard lever removes. The new entry (added via puts) is
|
|
2612
|
+
// never below the watermark by construction.
|
|
2613
|
+
const consumedDir = `${addressPrefix(addressSegment)}${WORKFLOW_RUN_CONSUMED_DIR}/`;
|
|
2614
|
+
const deletes = [processingFull];
|
|
2615
|
+
for (const entry of listing.consumed) {
|
|
2616
|
+
const blobPath = `${consumedDir}${entry.name}`;
|
|
2617
|
+
const bytes = await prior.readBlobByOid(entry.oid);
|
|
2618
|
+
const consumedReceivedAt = decodeConsumedReceivedAtOrThrow(bytes, blobPath);
|
|
2619
|
+
if (consumedReceivedAt < newWatermark) {
|
|
2620
|
+
prunedMessageIds.push(entry.name.slice(0, -".json".length));
|
|
2621
|
+
deletes.push(blobPath);
|
|
2622
|
+
}
|
|
2623
|
+
}
|
|
2624
|
+
return {
|
|
2625
|
+
puts: {
|
|
2626
|
+
[consumedFull]: utf8(JSON.stringify(envelope)),
|
|
2627
|
+
[watermarkPath(addressSegment)]: utf8(JSON.stringify({ watermark: newWatermark })),
|
|
2628
|
+
},
|
|
2629
|
+
deletes,
|
|
2630
|
+
};
|
|
2631
|
+
},
|
|
2632
|
+
});
|
|
2633
|
+
if (consumedEnvelope === null)
|
|
2634
|
+
throw new Error("unreachable");
|
|
2635
|
+
const captured = consumedEnvelope;
|
|
2636
|
+
return {
|
|
2637
|
+
commitSha,
|
|
2638
|
+
envelope: captured,
|
|
2639
|
+
watermark: advancedWatermark,
|
|
2640
|
+
prunedMessageIds,
|
|
2641
|
+
};
|
|
2642
|
+
}
|
|
2643
|
+
/**
|
|
2644
|
+
* Read the run event logs under `runs/` and return the set of
|
|
2645
|
+
* `consumedMessageId`s belonging to NON-terminal runs -- the messages a
|
|
2646
|
+
* live run still owns. The caller (the supervisor's spawn-time replay)
|
|
2647
|
+
* feeds this into `replayProcessingToInbox`'s `ownedMessageIds` so a
|
|
2648
|
+
* parked run's message is not re-admitted to inbox and dispatched a
|
|
2649
|
+
* second time while the run is recovered by re-driving its durable log.
|
|
2650
|
+
* Without this, the re-drive AND the re-triggered fresh run both re-park
|
|
2651
|
+
* the same awaitSignal gate on the same runId, and the two concurrent
|
|
2652
|
+
* runtime bodies race to a corrupt terminal.
|
|
2653
|
+
*
|
|
2654
|
+
* Reads the substrate's working tree via `getRepoDir`, mirroring the
|
|
2655
|
+
* child's `discoverInFlightRuns`. The working tree tracks the run-event
|
|
2656
|
+
* ref (`refs/heads/main`); the claim-check ref (`refs/heads/events`)
|
|
2657
|
+
* cannot see it, which is why this lives at the caller rather than inside
|
|
2658
|
+
* `replayProcessingToInbox`'s single-ref delta. A run whose log is sealed
|
|
2659
|
+
* (combined `events.json`, only permitted for a terminated run) or
|
|
2660
|
+
* carries a terminal event is excluded; an absent `runs/` directory
|
|
2661
|
+
* yields an empty set.
|
|
2662
|
+
*/
|
|
2663
|
+
export async function readOwnedMessageIds(store, repoId) {
|
|
2664
|
+
const fs = await import("node:fs/promises");
|
|
2665
|
+
const path = await import("node:path");
|
|
2666
|
+
const repoDir = store.getRepoDir(repoId);
|
|
2667
|
+
const runsDir = path.join(repoDir, WORKFLOW_RUN_RUNS_PREFIX);
|
|
2668
|
+
let runIds;
|
|
2669
|
+
try {
|
|
2670
|
+
runIds = await fs.readdir(runsDir);
|
|
2671
|
+
}
|
|
2672
|
+
catch (cause) {
|
|
2673
|
+
if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") {
|
|
2674
|
+
return new Set();
|
|
2675
|
+
}
|
|
2676
|
+
throw cause;
|
|
2677
|
+
}
|
|
2678
|
+
const owned = new Set();
|
|
2679
|
+
for (const runId of runIds) {
|
|
2680
|
+
const runDir = path.join(runsDir, runId);
|
|
2681
|
+
// A sealed run (combined events file) is terminal by the handler's
|
|
2682
|
+
// own invariant -- only a terminated run is sealed -- so it owns
|
|
2683
|
+
// nothing. Its presence also means the per-event directory is absent.
|
|
2684
|
+
let sealed = false;
|
|
2685
|
+
try {
|
|
2686
|
+
await fs.access(path.join(runDir, WORKFLOW_RUN_EVENTS_FILE));
|
|
2687
|
+
sealed = true;
|
|
2688
|
+
}
|
|
2689
|
+
catch {
|
|
2690
|
+
sealed = false;
|
|
2691
|
+
}
|
|
2692
|
+
if (sealed)
|
|
2693
|
+
continue;
|
|
2694
|
+
const eventsDir = path.join(runDir, WORKFLOW_RUN_EVENTS_DIR);
|
|
2695
|
+
let files;
|
|
2696
|
+
try {
|
|
2697
|
+
files = await fs.readdir(eventsDir);
|
|
2698
|
+
}
|
|
2699
|
+
catch {
|
|
2700
|
+
continue;
|
|
2701
|
+
}
|
|
2702
|
+
let terminal = false;
|
|
2703
|
+
let consumedMessageId;
|
|
2704
|
+
for (const file of files) {
|
|
2705
|
+
if (!file.endsWith(".json"))
|
|
2706
|
+
continue;
|
|
2707
|
+
let parsed;
|
|
2708
|
+
try {
|
|
2709
|
+
parsed = JSON.parse(await fs.readFile(path.join(eventsDir, file), "utf8"));
|
|
2710
|
+
}
|
|
2711
|
+
catch {
|
|
2712
|
+
continue;
|
|
2713
|
+
}
|
|
2714
|
+
if (typeof parsed !== "object" ||
|
|
2715
|
+
parsed === null ||
|
|
2716
|
+
!("type" in parsed)) {
|
|
2717
|
+
continue;
|
|
2718
|
+
}
|
|
2719
|
+
const type = parsed.type;
|
|
2720
|
+
if (typeof type !== "string")
|
|
2721
|
+
continue;
|
|
2722
|
+
if (TERMINAL_EVENT_TYPES.has(type)) {
|
|
2723
|
+
terminal = true;
|
|
2724
|
+
break;
|
|
2725
|
+
}
|
|
2726
|
+
if (type === "RunStarted" && "consumedMessageId" in parsed) {
|
|
2727
|
+
const mid = parsed.consumedMessageId;
|
|
2728
|
+
if (typeof mid === "string")
|
|
2729
|
+
consumedMessageId = mid;
|
|
2730
|
+
}
|
|
2731
|
+
}
|
|
2732
|
+
if (terminal)
|
|
2733
|
+
continue;
|
|
2734
|
+
if (consumedMessageId !== undefined)
|
|
2735
|
+
owned.add(consumedMessageId);
|
|
2736
|
+
}
|
|
2737
|
+
return owned;
|
|
2738
|
+
}
|
|
2739
|
+
/** Read one run's lifecycle from a committed workflow-run tree. */
|
|
2740
|
+
export async function readCommittedWorkflowRunLifecycle(reads, runId) {
|
|
2741
|
+
if (reads === null)
|
|
2742
|
+
return "absent";
|
|
2743
|
+
const runPath = `${WORKFLOW_RUN_RUNS_PREFIX}/${runId}`;
|
|
2744
|
+
const runChildren = await reads.listDir(runPath);
|
|
2745
|
+
if (runChildren.some((entry) => entry.type === "blob" && entry.name === WORKFLOW_RUN_EVENTS_FILE)) {
|
|
2746
|
+
return "terminal";
|
|
2747
|
+
}
|
|
2748
|
+
const eventsPath = `${runPath}/${WORKFLOW_RUN_EVENTS_DIR}`;
|
|
2749
|
+
const eventEntries = (await reads.listDir(eventsPath)).filter((entry) => entry.type === "blob" && parseEventSeq(entry.name) !== null);
|
|
2750
|
+
const latest = eventEntries.reduce((candidate, entry) => {
|
|
2751
|
+
if (candidate === undefined)
|
|
2752
|
+
return entry;
|
|
2753
|
+
const candidateSeq = parseEventSeq(candidate.name);
|
|
2754
|
+
const entrySeq = parseEventSeq(entry.name);
|
|
2755
|
+
return entrySeq !== null &&
|
|
2756
|
+
candidateSeq !== null &&
|
|
2757
|
+
entrySeq > candidateSeq
|
|
2758
|
+
? entry
|
|
2759
|
+
: candidate;
|
|
2760
|
+
}, undefined);
|
|
2761
|
+
if (latest !== undefined) {
|
|
2762
|
+
const eventPath = `${eventsPath}/${latest.name}`;
|
|
2763
|
+
let parsed;
|
|
2764
|
+
try {
|
|
2765
|
+
parsed = JSON.parse(new TextDecoder().decode(await reads.readBlobByOid(latest.oid)));
|
|
2766
|
+
}
|
|
2767
|
+
catch (cause) {
|
|
2768
|
+
throw new Error(`workflow_run_event_unreadable: ${eventPath}`, { cause });
|
|
2769
|
+
}
|
|
2770
|
+
if (typeof parsed === "object" &&
|
|
2771
|
+
parsed !== null &&
|
|
2772
|
+
"type" in parsed &&
|
|
2773
|
+
typeof parsed.type === "string" &&
|
|
2774
|
+
TERMINAL_EVENT_TYPES.has(parsed.type)) {
|
|
2775
|
+
return "terminal";
|
|
2776
|
+
}
|
|
2777
|
+
}
|
|
2778
|
+
return eventEntries.length === 0 ? "absent" : "live";
|
|
2779
|
+
}
|
|
2780
|
+
/**
|
|
2781
|
+
* Read the durable lifecycle of one run from the workflow-run working tree.
|
|
2782
|
+
* `grants.json` alone is still an absent run: grants are staged before the
|
|
2783
|
+
* first trigger, while the first event is the durable proof that the run was
|
|
2784
|
+
* fired. A sealed event log is terminal by the kind handler's compaction
|
|
2785
|
+
* invariant.
|
|
2786
|
+
*
|
|
2787
|
+
* The supervisor uses this when in-memory cohort membership is empty. That
|
|
2788
|
+
* happens both for a genuinely new deployment and briefly during recovery,
|
|
2789
|
+
* so treating both states as "fire" would start a second driver for a live
|
|
2790
|
+
* log or reuse a terminal run.
|
|
2791
|
+
*/
|
|
2792
|
+
export async function readWorkflowRunLifecycle(store, repoId, runId) {
|
|
2793
|
+
const fs = await import("node:fs/promises");
|
|
2794
|
+
const path = await import("node:path");
|
|
2795
|
+
const runDir = path.join(store.getRepoDir(repoId), WORKFLOW_RUN_RUNS_PREFIX, runId);
|
|
2796
|
+
try {
|
|
2797
|
+
await fs.access(path.join(runDir, WORKFLOW_RUN_EVENTS_FILE));
|
|
2798
|
+
return "terminal";
|
|
2799
|
+
}
|
|
2800
|
+
catch (cause) {
|
|
2801
|
+
if (!(cause instanceof Error) ||
|
|
2802
|
+
!("code" in cause) ||
|
|
2803
|
+
cause.code !== "ENOENT") {
|
|
2804
|
+
throw cause;
|
|
2805
|
+
}
|
|
2806
|
+
}
|
|
2807
|
+
const eventsDir = path.join(runDir, WORKFLOW_RUN_EVENTS_DIR);
|
|
2808
|
+
let files;
|
|
2809
|
+
try {
|
|
2810
|
+
files = await fs.readdir(eventsDir);
|
|
2811
|
+
}
|
|
2812
|
+
catch (cause) {
|
|
2813
|
+
if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") {
|
|
2814
|
+
return "absent";
|
|
2815
|
+
}
|
|
2816
|
+
throw cause;
|
|
2817
|
+
}
|
|
2818
|
+
const eventFiles = files.filter((file) => parseEventSeq(file) !== null);
|
|
2819
|
+
const latest = eventFiles.reduce((candidate, file) => {
|
|
2820
|
+
if (candidate === undefined)
|
|
2821
|
+
return file;
|
|
2822
|
+
const candidateSeq = parseEventSeq(candidate);
|
|
2823
|
+
const fileSeq = parseEventSeq(file);
|
|
2824
|
+
return fileSeq !== null && candidateSeq !== null && fileSeq > candidateSeq
|
|
2825
|
+
? file
|
|
2826
|
+
: candidate;
|
|
2827
|
+
}, undefined);
|
|
2828
|
+
if (latest !== undefined) {
|
|
2829
|
+
const eventPath = path.join(eventsDir, latest);
|
|
2830
|
+
let parsed;
|
|
2831
|
+
try {
|
|
2832
|
+
parsed = JSON.parse(await fs.readFile(eventPath, "utf8"));
|
|
2833
|
+
}
|
|
2834
|
+
catch (cause) {
|
|
2835
|
+
throw new Error(`workflow_run_event_unreadable: ${eventPath}`, { cause });
|
|
2836
|
+
}
|
|
2837
|
+
if (typeof parsed === "object" &&
|
|
2838
|
+
parsed !== null &&
|
|
2839
|
+
"type" in parsed &&
|
|
2840
|
+
typeof parsed.type === "string" &&
|
|
2841
|
+
TERMINAL_EVENT_TYPES.has(parsed.type)) {
|
|
2842
|
+
return "terminal";
|
|
2843
|
+
}
|
|
2844
|
+
}
|
|
2845
|
+
return eventFiles.length === 0 ? "absent" : "live";
|
|
2846
|
+
}
|
|
2847
|
+
/**
|
|
2848
|
+
* Recovery path: move every processing entry at `address` back to
|
|
2849
|
+
* inbox preserving the original `<receivedAt>-<messageId>` filename
|
|
2850
|
+
* key so FIFO ordering survives a workflow-process crash. Returns
|
|
2851
|
+
* the set of keys that were moved; when nothing was in processing
|
|
2852
|
+
* the returned `replayedKeys` is empty (and the commit is a no-op
|
|
2853
|
+
* rewrite of the same tree).
|
|
2854
|
+
*
|
|
2855
|
+
* The replay is atomic across all processing entries — a partial
|
|
2856
|
+
* replay that left some entries in processing would corrupt the
|
|
2857
|
+
* FIFO discipline (the next dequeue would pull the wrong entry).
|
|
2858
|
+
*
|
|
2859
|
+
* Watermark carve-out (load-bearing — do NOT "tighten" this): the
|
|
2860
|
+
* replay deliberately does NOT apply the `receivedAt < watermark`
|
|
2861
|
+
* stale-reject that `enqueueInbox` applies. A `processing/` entry was
|
|
2862
|
+
* already dequeued past the dedup index, so re-admitting it to
|
|
2863
|
+
* `inbox/` even when its `receivedAt` has fallen below an advanced
|
|
2864
|
+
* watermark is correct — the message is a legitimately in-flight one
|
|
2865
|
+
* recovered after a crash, not a fresh inbound that could be a
|
|
2866
|
+
* duplicate. Applying the stale-reject here would silently LOSE that
|
|
2867
|
+
* message. The watermark only ever gates fresh inbound at the enqueue
|
|
2868
|
+
* boundary; the recovery replay is exempt by design.
|
|
2869
|
+
*/
|
|
2870
|
+
export async function replayProcessingToInbox(store, principal, repoId, address, opts = {}) {
|
|
2871
|
+
const addressSegment = addressSegmentFor(address);
|
|
2872
|
+
const ref = claimCheckCommitRef();
|
|
2873
|
+
const ownedMessageIds = opts.ownedMessageIds ?? new Set();
|
|
2874
|
+
const replayedKeys = [];
|
|
2875
|
+
const { commitSha } = await store.writeTreeDelta(principal, repoId, ref, {
|
|
2876
|
+
changedPathPrefixes: new Set([addressPrefix(addressSegment)]),
|
|
2877
|
+
message: `replay processing ${address}`,
|
|
2878
|
+
computeDelta: async (_parentCommitSha, prior) => {
|
|
2879
|
+
const listing = await readAddressListing(prior, addressSegment);
|
|
2880
|
+
const processingDir = `${addressPrefix(addressSegment)}${WORKFLOW_RUN_PROCESSING_DIR}/`;
|
|
2881
|
+
const inboxDir = `${addressPrefix(addressSegment)}${WORKFLOW_RUN_INBOX_DIR}/`;
|
|
2882
|
+
const inboxNames = new Set(listing.inbox.map((e) => e.name));
|
|
2883
|
+
const puts = {};
|
|
2884
|
+
const deletes = [];
|
|
2885
|
+
for (const entry of listing.processing) {
|
|
2886
|
+
const bytes = await prior.readBlobByOid(entry.oid);
|
|
2887
|
+
// A processing entry whose run is still live (non-terminal durable
|
|
2888
|
+
// log) is owned by the recovery re-drive of that same run. Re-
|
|
2889
|
+
// admitting the message to inbox would dispatch a SECOND run for
|
|
2890
|
+
// it, colliding with the re-drive on the shared runId. Leave such
|
|
2891
|
+
// an entry in processing untouched; the run's eventual
|
|
2892
|
+
// `markConsumed` clears it. Only genuinely orphaned entries (no
|
|
2893
|
+
// run, or a terminal run) are replayed. The run logs live on a
|
|
2894
|
+
// different ref, so the caller precomputes the owned set.
|
|
2895
|
+
const envelope = decodeQueueEnvelopeOrThrow(bytes, `${processingDir}${entry.name}`);
|
|
2896
|
+
if (ownedMessageIds.has(envelope.messageId)) {
|
|
2897
|
+
continue;
|
|
2898
|
+
}
|
|
2899
|
+
const inboxFull = `${inboxDir}${entry.name}`;
|
|
2900
|
+
if (inboxNames.has(entry.name)) {
|
|
2901
|
+
throw new Error(`claim_check_replay_collision: ${inboxFull} already exists; cannot replay processing entry`);
|
|
2902
|
+
}
|
|
2903
|
+
// Re-admit the in-flight entry WITHOUT the watermark stale-reject
|
|
2904
|
+
// enqueueInbox applies: it was already past dedup, so a
|
|
2905
|
+
// below-watermark receivedAt is no reason to refuse it. Applying
|
|
2906
|
+
// the stale-check here would lose a legitimately in-flight
|
|
2907
|
+
// message after a crash. Do not tighten this.
|
|
2908
|
+
puts[inboxFull] = bytes;
|
|
2909
|
+
deletes.push(`${processingDir}${entry.name}`);
|
|
2910
|
+
replayedKeys.push(entry.name.slice(0, -".json".length));
|
|
2911
|
+
}
|
|
2912
|
+
return { puts, deletes };
|
|
2913
|
+
},
|
|
2914
|
+
});
|
|
2915
|
+
return { commitSha, replayedKeys };
|
|
2916
|
+
}
|