@namzu/sdk 20.4.0 → 21.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/CHANGELOG.md +179 -0
  2. package/dist/bridge/sse/mapper.d.ts +19 -0
  3. package/dist/bridge/sse/mapper.d.ts.map +1 -1
  4. package/dist/bridge/sse/mapper.js +9 -1
  5. package/dist/bridge/sse/mapper.js.map +1 -1
  6. package/dist/contracts/schemas.d.ts +10 -10
  7. package/dist/manager/run/persistence.d.ts +26 -0
  8. package/dist/manager/run/persistence.d.ts.map +1 -1
  9. package/dist/manager/run/persistence.js +40 -0
  10. package/dist/manager/run/persistence.js.map +1 -1
  11. package/dist/public-runtime.d.ts +4 -0
  12. package/dist/public-runtime.d.ts.map +1 -1
  13. package/dist/public-runtime.js +16 -0
  14. package/dist/public-runtime.js.map +1 -1
  15. package/dist/run/drain.d.ts +194 -0
  16. package/dist/run/drain.d.ts.map +1 -0
  17. package/dist/run/drain.js +228 -0
  18. package/dist/run/drain.js.map +1 -0
  19. package/dist/run/index.d.ts +2 -0
  20. package/dist/run/index.d.ts.map +1 -1
  21. package/dist/run/index.js +1 -0
  22. package/dist/run/index.js.map +1 -1
  23. package/dist/runtime/query/events.d.ts +12 -0
  24. package/dist/runtime/query/events.d.ts.map +1 -1
  25. package/dist/runtime/query/events.js +61 -3
  26. package/dist/runtime/query/events.js.map +1 -1
  27. package/dist/runtime/query/index.d.ts +36 -0
  28. package/dist/runtime/query/index.d.ts.map +1 -1
  29. package/dist/runtime/query/index.js +32 -0
  30. package/dist/runtime/query/index.js.map +1 -1
  31. package/dist/runtime/query/resume-run.d.ts +25 -0
  32. package/dist/runtime/query/resume-run.d.ts.map +1 -1
  33. package/dist/runtime/query/resume-run.js +15 -3
  34. package/dist/runtime/query/resume-run.js.map +1 -1
  35. package/dist/store/index.d.ts +1 -1
  36. package/dist/store/index.d.ts.map +1 -1
  37. package/dist/store/index.js +1 -1
  38. package/dist/store/index.js.map +1 -1
  39. package/dist/store/run/conformance.d.ts +159 -0
  40. package/dist/store/run/conformance.d.ts.map +1 -0
  41. package/dist/store/run/conformance.js +451 -0
  42. package/dist/store/run/conformance.js.map +1 -0
  43. package/dist/store/run/disk.d.ts +30 -2
  44. package/dist/store/run/disk.d.ts.map +1 -1
  45. package/dist/store/run/disk.js +101 -0
  46. package/dist/store/run/disk.js.map +1 -1
  47. package/dist/store/run/memory.d.ts +5 -4
  48. package/dist/store/run/memory.d.ts.map +1 -1
  49. package/dist/store/run/memory.js +30 -1
  50. package/dist/store/run/memory.js.map +1 -1
  51. package/dist/types/doctor/check.d.ts +28 -2
  52. package/dist/types/doctor/check.d.ts.map +1 -1
  53. package/dist/types/run/event-cursor.d.ts +90 -0
  54. package/dist/types/run/event-cursor.d.ts.map +1 -0
  55. package/dist/types/run/event-cursor.js +47 -0
  56. package/dist/types/run/event-cursor.js.map +1 -0
  57. package/dist/types/run/events.d.ts +71 -1
  58. package/dist/types/run/events.d.ts.map +1 -1
  59. package/dist/types/run/events.js.map +1 -1
  60. package/dist/types/run/index.d.ts +1 -0
  61. package/dist/types/run/index.d.ts.map +1 -1
  62. package/dist/types/run/index.js +1 -0
  63. package/dist/types/run/index.js.map +1 -1
  64. package/dist/types/run/store.d.ts +41 -1
  65. package/dist/types/run/store.d.ts.map +1 -1
  66. package/package.json +6 -1
  67. package/src/bridge/sse/mapper.ts +28 -1
  68. package/src/manager/run/persistence.ts +43 -0
  69. package/src/public-runtime.ts +22 -0
  70. package/src/run/drain.ts +393 -0
  71. package/src/run/index.ts +3 -0
  72. package/src/runtime/query/events.ts +68 -4
  73. package/src/runtime/query/index.ts +87 -0
  74. package/src/runtime/query/resume-run.ts +67 -12
  75. package/src/store/index.ts +1 -1
  76. package/src/store/run/conformance.ts +705 -0
  77. package/src/store/run/disk.ts +106 -2
  78. package/src/store/run/memory.ts +35 -5
  79. package/src/types/doctor/check.ts +28 -2
  80. package/src/types/run/event-cursor.ts +118 -0
  81. package/src/types/run/events.ts +80 -3
  82. package/src/types/run/index.ts +1 -0
  83. package/src/types/run/store.ts +43 -1
@@ -0,0 +1,393 @@
1
+ /**
2
+ * One pass over a queue of durable runs: list what nobody holds, take it,
3
+ * hand it to a worker, give it back.
4
+ *
5
+ * Every primitive this composes already shipped —
6
+ * {@link import('../store/run/listing.js').listDurableRuns} enumerates runs
7
+ * above a run id, `claimRun` arbitrates between processes, `releaseRun`
8
+ * returns a run to the queue, and `resumeRun` carries a fence into every
9
+ * durable write. Nothing composed them, so the two things the claim was
10
+ * built for — an approval inbox and a crash sweeper — still required a host
11
+ * to write the loop, and writing it correctly means getting the release
12
+ * into a `finally` and the `null` claim out of the error path. Both are the
13
+ * kind of thing a host gets wrong once, quietly.
14
+ *
15
+ * ## What this deliberately is NOT
16
+ *
17
+ * A supervisor, a daemon, or a scheduler. There is no timer here, no
18
+ * process spawn, no retry backoff and no `while (true)`. `drainRuns` makes
19
+ * ONE bounded pass and returns what happened; running it again is the
20
+ * caller's decision, made wherever that caller already has a scheduler. A
21
+ * per-platform supervisor is the same trade the deployment-adapter matrix
22
+ * was rejected for: one seam beats N adapters.
23
+ *
24
+ * The unit of work is a callback, so this module never needs a provider, a
25
+ * tool registry or a sandbox — the half of a run that cannot be serialized
26
+ * stays with the caller, exactly as `resumeRun` already splits it.
27
+ */
28
+
29
+ import { claimRun, listDurableRuns, releaseRun, summarizePark } from '../store/run/listing.js'
30
+ import type { NamzuErrorCode } from '../types/errors/index.js'
31
+ import { NamzuError } from '../types/errors/index.js'
32
+ import type { RunId } from '../types/ids/index.js'
33
+ import type {
34
+ CheckpointListingScope,
35
+ CheckpointStore,
36
+ DurableRunEntry,
37
+ ParkState,
38
+ RunClaim,
39
+ } from '../types/run/checkpoint-store.js'
40
+
41
+ /** Runs handled per pass when the caller names no page size. */
42
+ export const DEFAULT_DRAIN_PAGE_SIZE = 100
43
+
44
+ /**
45
+ * What a drainer does with one run it successfully took.
46
+ *
47
+ * Receives the claim, not just its fence, because the holder and expiry are
48
+ * what a worker needs to decide whether it still has time to start — and
49
+ * because a caller that only ever sees a number tends to forget the lease
50
+ * can lapse under it.
51
+ *
52
+ * The intended body is a resume:
53
+ *
54
+ * ```ts
55
+ * onRun: (entry, claim) =>
56
+ * resumeRun({
57
+ * ...yourQueryParams,
58
+ * scope: { ...entry, threadId },
59
+ * checkpointStore: store,
60
+ * claimFence: claim.fence,
61
+ * })
62
+ * ```
63
+ *
64
+ * `claimFence` is the whole reason the claim is handed over: a write that
65
+ * does not carry it is unfenced, so a worker stalled past its lease would
66
+ * still be able to overwrite the record of whoever took the run over.
67
+ *
68
+ * A throw is recorded against that run and the pass continues. A drainer
69
+ * that died on the first bad run would leave the rest of the queue
70
+ * untouched, which is the failure a queue exists to spread out.
71
+ */
72
+ export type DrainRun = (entry: DurableRunEntry, claim: RunClaim) => void | Promise<void>
73
+
74
+ export interface DrainRunsParams {
75
+ /** Backend to list, claim and release against. Must support all three. */
76
+ readonly store: CheckpointStore
77
+ /** Contiguous prefix — `tenantId` required. See {@link CheckpointListingScope}. */
78
+ readonly scope: CheckpointListingScope
79
+ /**
80
+ * Who is taking the runs. Per-PROCESS, never per-deployment: `holder` is
81
+ * the only thing that distinguishes a renewal from a theft, so two
82
+ * drainers sharing a string take live claims from each other instantly.
83
+ */
84
+ readonly holder: string
85
+ /** Lease length in ms. Long enough that the slowest run finishes inside it. */
86
+ readonly ttlMs: number
87
+ /** The work. See {@link DrainRun}. */
88
+ readonly onRun: DrainRun
89
+
90
+ /**
91
+ * Keep only runs whose park is in one of these states.
92
+ *
93
+ * **Absent means every run with durable state, parked or not**, and that
94
+ * is not a placeholder default — it is what a crash sweep wants, because
95
+ * a run that died mid-flight never parked and would be invisible under
96
+ * any park filter. An approval inbox passes `['outstanding']`; a
97
+ * reclamation sweep passes `['expired']`.
98
+ */
99
+ readonly park?: readonly ParkState[]
100
+
101
+ /**
102
+ * Stop taking new runs. Work already in flight is NOT interrupted — this
103
+ * module owns no run and cannot cancel one; a caller that needs to abort
104
+ * the work itself passes the same signal into whatever `onRun` starts.
105
+ */
106
+ readonly signal?: AbortSignal
107
+
108
+ /**
109
+ * How many runs may be in flight at once. Defaults to 1.
110
+ *
111
+ * Bounded on purpose. The obvious implementation — claim everything, then
112
+ * `Promise.all` — holds N leases while doing one run's worth of work, so
113
+ * the runs at the back of the batch expire before they are started and
114
+ * are taken by somebody else mid-flight.
115
+ */
116
+ readonly maxConcurrent?: number
117
+
118
+ /** Listing page size. See {@link DEFAULT_DRAIN_PAGE_SIZE}. */
119
+ readonly pageSize?: number
120
+
121
+ /**
122
+ * Clock for expiry, so one pass judges every claim against one instant
123
+ * and a test does not have to wait out a lease.
124
+ */
125
+ readonly now?: number
126
+ }
127
+
128
+ /** A run a pass could not finish, and why. */
129
+ export interface DrainFailure {
130
+ readonly runId: RunId
131
+ readonly error: string
132
+ }
133
+
134
+ /** What one pass did. */
135
+ export interface DrainRunsResult {
136
+ /** Rows the listing returned, before any of them were contended for. */
137
+ readonly listed: number
138
+ /** Runs whose `onRun` returned. */
139
+ readonly drained: readonly RunId[]
140
+ /**
141
+ * Runs another worker held. Not failures: "somebody got there first" is
142
+ * the ordinary outcome of a queue with more than one reader.
143
+ */
144
+ readonly skipped: readonly RunId[]
145
+ /**
146
+ * Runs that stopped matching {@link DrainRunsParams.park} between the
147
+ * listing and the claim, and were given straight back.
148
+ *
149
+ * Separate from {@link DrainRunsResult.skipped} because the cause is
150
+ * different and so is what an operator should do about a lot of them: a
151
+ * skip means another drainer is holding runs right now, a stale entry
152
+ * means another drainer already FINISHED one. Empty on a pass with no
153
+ * park filter, which has nothing to re-check against.
154
+ */
155
+ readonly stale: readonly RunId[]
156
+ /** Runs whose `onRun` threw. */
157
+ readonly failed: readonly DrainFailure[]
158
+ /**
159
+ * Runs that finished but whose lease could not be handed back.
160
+ *
161
+ * Separate from {@link DrainRunsResult.failed} because it is a different
162
+ * fact with a different consequence: the work is done and the record is
163
+ * written; the run is merely unavailable to the next reader until the
164
+ * lease lapses. Reported rather than swallowed — a release that quietly
165
+ * did nothing is how a queue silently loses throughput.
166
+ */
167
+ readonly unreleased: readonly DrainFailure[]
168
+ /** Whether the pass stopped early because the signal aborted. */
169
+ readonly stopped: boolean
170
+ }
171
+
172
+ function refuse(code: NamzuErrorCode, message: string, details: Record<string, unknown>): never {
173
+ throw new NamzuError({ code, message, details })
174
+ }
175
+
176
+ function toMessage(err: unknown): string {
177
+ return err instanceof Error ? err.message : String(err)
178
+ }
179
+
180
+ /**
181
+ * Refuse a store that cannot do the whole job, BEFORE anything is listed.
182
+ *
183
+ * Checked up front rather than at the first call that needs each method, so
184
+ * that a store missing only `releaseRun` cannot resume half a queue and then
185
+ * discover it has no way to give the runs back. The optional-capability rule
186
+ * on `CheckpointStore` says a caller refuses rather than degrades; a drainer
187
+ * that degraded would be the worst instance of it, because "claimed by
188
+ * default" here means every worker proceeds on every run.
189
+ */
190
+ function assertDrainable(store: CheckpointStore): void {
191
+ const missing = (['listDurableRuns', 'claimRun', 'releaseRun'] as const).filter(
192
+ (m) => typeof store[m] !== 'function',
193
+ )
194
+ if (missing.length === 0) return
195
+ refuse(
196
+ 'capability_unavailable',
197
+ `drainRuns: the injected checkpoint store does not implement ${missing.map((m) => `\`${m}\``).join(', ')}, so it cannot arbitrate a queue. Refusing before anything is claimed rather than draining what it can — a drainer that proceeded without a claim would let two workers restore one checkpoint, both execute its tools and both write under one run id. Supply a store that implements all three (the built-in disk and in-memory stores do), or run a single writer per run.`,
198
+ { missing },
199
+ )
200
+ }
201
+
202
+ /**
203
+ * Take every unclaimed run under a scope, one bounded pass, and give each
204
+ * one back when its work returns.
205
+ *
206
+ * The shape is: list parked-and-unclaimed → claim → work → release in a
207
+ * `finally`. The `finally` is the part a host writes wrong: a worker that
208
+ * returns without releasing leaves the run stuck until the lease lapses,
209
+ * and a worker that releases only on success leaves a FAILED run stuck for
210
+ * the same duration — so a queue quietly loses its throughput to the runs
211
+ * that need retrying most.
212
+ *
213
+ * `claimed: false` is not a parameter. A drainer never wants work somebody
214
+ * else holds; that is what makes it a drainer rather than a listing. An
215
+ * expired claim counts as unheld, which is what makes a dead worker's runs
216
+ * recoverable at all.
217
+ *
218
+ * ## What "exactly once" does and does not mean here
219
+ *
220
+ * Two drainers never hold one run at the same time — that is the claim, and
221
+ * it is absolute. **Exactly-once over a whole pass is a weaker promise, and
222
+ * where it holds it comes from the FILTER, not from the claim.** A listing
223
+ * is a snapshot; between paging a row and claiming it, another drainer can
224
+ * finish that run and release it, and the claim then succeeds on work
225
+ * already done. So a claimed row is re-read against
226
+ * {@link DrainRunsParams.park} before any work starts, and one that no
227
+ * longer matches is given straight back as {@link DrainRunsResult.stale}.
228
+ * An inbox drain (`park: ['outstanding']`) whose work answers the park is
229
+ * therefore exactly-once, because doing the work is what removes the run
230
+ * from the queue.
231
+ *
232
+ * With NO park filter there is nothing to re-check, and two drainers can
233
+ * both process one run. That is not an omission: a checkpoint store holds no
234
+ * run STATUS by design — nothing in it distinguishes a run that finished
235
+ * from one that died — so "already done" is a fact only the host's own run
236
+ * records carry. A crash sweep intersects with those records inside
237
+ * `onRun`, which is the shape {@link DurableRunEntry} already prescribes.
238
+ *
239
+ * @throws NamzuError `capability_unavailable` when the store cannot list,
240
+ * claim or release — before any run is touched.
241
+ * @throws NamzuError `invalid_config` on a lease or concurrency that cannot
242
+ * mean what it says.
243
+ */
244
+ export async function drainRuns(params: DrainRunsParams): Promise<DrainRunsResult> {
245
+ const { store, scope, holder, ttlMs, onRun, park, signal, now } = params
246
+
247
+ assertDrainable(store)
248
+
249
+ if (holder.trim().length === 0) {
250
+ refuse(
251
+ 'invalid_config',
252
+ 'drainRuns: `holder` is empty. It is the only thing that distinguishes a renewal from a theft, so two drainers sharing one string take live claims from each other instantly. Use something per-process — a worker id, a pod name plus a pid.',
253
+ { holder },
254
+ )
255
+ }
256
+ if (!Number.isFinite(ttlMs) || ttlMs <= 0) {
257
+ refuse(
258
+ 'invalid_config',
259
+ `drainRuns: ttlMs must be a positive number of milliseconds, got ${String(ttlMs)}. A lease that expires immediately is a lease every worker can take at once, which is the condition a claim exists to prevent.`,
260
+ { ttlMs },
261
+ )
262
+ }
263
+ const maxConcurrent = params.maxConcurrent ?? 1
264
+ if (!Number.isInteger(maxConcurrent) || maxConcurrent < 1) {
265
+ refuse(
266
+ 'invalid_config',
267
+ `drainRuns: maxConcurrent must be a positive integer, got ${String(params.maxConcurrent)}. Zero would drain nothing while reporting a successful pass.`,
268
+ { maxConcurrent: params.maxConcurrent },
269
+ )
270
+ }
271
+ const pageSize = params.pageSize ?? DEFAULT_DRAIN_PAGE_SIZE
272
+
273
+ const drained: RunId[] = []
274
+ const skipped: RunId[] = []
275
+ const stale: RunId[] = []
276
+ const failed: DrainFailure[] = []
277
+ const unreleased: DrainFailure[] = []
278
+ let listed = 0
279
+ let stopped = false
280
+
281
+ const giveBack = async (entry: DurableRunEntry, fence: number): Promise<void> => {
282
+ try {
283
+ await releaseRun(store, entry, fence)
284
+ } catch (err) {
285
+ // Never rethrown: on the work path this runs inside a `finally`
286
+ // unwinding the caller's error, and replacing it would send the
287
+ // operator to debug the disk instead of the run.
288
+ unreleased.push({ runId: entry.runId, error: toMessage(err) })
289
+ }
290
+ }
291
+
292
+ /**
293
+ * Is this row still the row the listing described?
294
+ *
295
+ * A listing is a SNAPSHOT, and a claim taken against a stale snapshot is
296
+ * a claim on work somebody already did. The window is real and small:
297
+ * drainer B pages the queue, drainer A takes a run, finishes it, answers
298
+ * its park and releases — and B's claim then succeeds on a run that is no
299
+ * longer outstanding. Mutual exclusion cannot close that; only re-reading
300
+ * after the claim can, which is why this is here and not in the store.
301
+ *
302
+ * Only the park is re-checked, because it is the only predicate this loop
303
+ * was given. **Two drainers with no park filter can both process one
304
+ * run**, and no amount of claiming prevents it: a checkpoint store holds
305
+ * no run STATUS by design — see the note on {@link DurableRunEntry} — so
306
+ * "already done" is a fact only the host's own run records carry. A crash
307
+ * sweep intersects with those records inside `onRun`.
308
+ */
309
+ const stillMatches = async (entry: DurableRunEntry): Promise<boolean> => {
310
+ if (!park) return true
311
+ const fresh = summarizePark(await store.listCheckpoints(entry), now ?? Date.now())
312
+ return fresh !== undefined && park.includes(fresh.state)
313
+ }
314
+
315
+ /**
316
+ * One run: take it, work it, give it back.
317
+ *
318
+ * **No cancellation check here, and its absence is deliberate.** One was
319
+ * written, and a mutation test found nothing could kill it: the batch
320
+ * below dispatches with `.map(handle)`, which calls every handler
321
+ * synchronously before any of them awaits, so a signal that aborts during
322
+ * a batch cannot be observed at the top of a handler that has already
323
+ * been entered — and a signal that aborts BETWEEN batches is caught by
324
+ * the check in the loop, which runs first. A branch nothing can reach is
325
+ * a declaration nothing drives, so it is gone rather than covered by a
326
+ * test that would have proved nothing
327
+ * (`docs/conventions/declared-but-undriven.md`).
328
+ */
329
+ const handle = async (entry: DurableRunEntry): Promise<void> => {
330
+ const claim = await claimRun(store, entry, {
331
+ holder,
332
+ ttlMs,
333
+ ...(now !== undefined ? { now } : {}),
334
+ })
335
+ // `null` is not an error. Another worker got there first, which is the
336
+ // ordinary outcome of two readers on one queue.
337
+ if (!claim) {
338
+ skipped.push(entry.runId)
339
+ return
340
+ }
341
+ // Checked with the claim in hand rather than before taking it: only
342
+ // under the claim is the answer stable, because nobody else can change
343
+ // it while this drainer holds the run.
344
+ if (!(await stillMatches(entry))) {
345
+ stale.push(entry.runId)
346
+ await giveBack(entry, claim.fence)
347
+ return
348
+ }
349
+ try {
350
+ await onRun(entry, claim)
351
+ drained.push(entry.runId)
352
+ } catch (err) {
353
+ failed.push({ runId: entry.runId, error: toMessage(err) })
354
+ } finally {
355
+ await giveBack(entry, claim.fence)
356
+ }
357
+ }
358
+
359
+ let cursor: string | undefined
360
+ do {
361
+ if (signal?.aborted) {
362
+ stopped = true
363
+ break
364
+ }
365
+ const page = await listDurableRuns(store, scope, {
366
+ // Not a parameter. See the note above.
367
+ claimed: false,
368
+ ...(park ? { park } : {}),
369
+ limit: pageSize,
370
+ ...(cursor !== undefined ? { cursor } : {}),
371
+ ...(now !== undefined ? { now } : {}),
372
+ })
373
+ listed += page.entries.length
374
+
375
+ // Windowed rather than `Promise.all` over the page: the leases are taken
376
+ // as the work starts, so a page bigger than the pass can finish inside
377
+ // one TTL does not hand the tail of it to somebody else mid-flight.
378
+ for (let i = 0; i < page.entries.length; i += maxConcurrent) {
379
+ if (signal?.aborted) {
380
+ stopped = true
381
+ break
382
+ }
383
+ await Promise.all(page.entries.slice(i, i + maxConcurrent).map(handle))
384
+ }
385
+
386
+ // A drained run is released, so it is unclaimed again — but the cursor
387
+ // is a position in a total order and has already passed it, so the pass
388
+ // cannot see it twice and cannot fail to terminate.
389
+ cursor = stopped ? undefined : page.cursor
390
+ } while (cursor !== undefined)
391
+
392
+ return { listed, drained, skipped, stale, failed, unreleased, stopped }
393
+ }
package/src/run/index.ts CHANGED
@@ -5,5 +5,8 @@ export { RunDiskStore } from '../store/run/disk.js'
5
5
  export { createRunReporter } from './reporter.js'
6
6
  export type { RunReporter } from './reporter.js'
7
7
 
8
+ export { DEFAULT_DRAIN_PAGE_SIZE, drainRuns } from './drain.js'
9
+ export type { DrainFailure, DrainRun, DrainRunsParams, DrainRunsResult } from './drain.js'
10
+
8
11
  export { checkLimitsDetailed, buildLimitConfig } from './LimitChecker.js'
9
12
  export type { LimitCheckerState, LimitCheckResult } from './LimitChecker.js'
@@ -4,6 +4,7 @@ import { buildProbeContext } from '../../probe/context.js'
4
4
  import { type ProbeRegistry, probe as defaultProbeRegistry } from '../../probe/registry.js'
5
5
  import type { ActivityEvent, ActivityStore } from '../../store/activity/memory.js'
6
6
  import type { RunId } from '../../types/ids/index.js'
7
+ import type { ClaimFence } from '../../types/run/checkpoint-store.js'
7
8
  import { isEphemeralEvent } from '../../types/run/events.js'
8
9
  import type { RunEvent } from '../../types/run/index.js'
9
10
  import type { TaskEvent, TaskStore } from '../../types/task/index.js'
@@ -37,6 +38,22 @@ export class EventTranslator {
37
38
  this.probes = probeRegistry
38
39
  }
39
40
 
41
+ /**
42
+ * The claim this run is being written under, when it holds one.
43
+ *
44
+ * Stamped on every durable event as its `generation`, so a consumer whose
45
+ * cursor predates a takeover is told its sequence space changed instead of
46
+ * being handed a splice from a different writer's log.
47
+ */
48
+ private generation: ClaimFence | undefined
49
+
50
+ /** Serializes sequence assignment against the append. See {@link emitEvent}. */
51
+ private appendChain: Promise<void> = Promise.resolve()
52
+
53
+ setGeneration(fence: ClaimFence | undefined): void {
54
+ this.generation = fence
55
+ }
56
+
40
57
  readonly emitEvent: EmitEvent = async (event: RunEvent): Promise<void> => {
41
58
  this.probes.dispatch(event, buildProbeContext({ runId: event.runId }))
42
59
 
@@ -64,15 +81,62 @@ export class EventTranslator {
64
81
  // briefly than to drop a state transition.
65
82
  }
66
83
 
67
- this.pendingEvents.push(event)
68
-
69
84
  // D1 middle path: ephemeral events never enter `transcript.jsonl`.
70
85
  // They live only on the in-memory bus for live UI rendering.
71
86
  // Replay (`runtime/query/replay/prepare.ts`) reads checkpoints
72
87
  // not transcripts, so this preserves replay fidelity while
73
88
  // eliminating the durable bloat review flagged.
74
- if (!isEphemeralEvent(event)) {
75
- await this.runMgr.getRunStore().appendEvent(event)
89
+ if (isEphemeralEvent(event)) {
90
+ // No number, and that is the honest statement: nothing will
91
+ // persist this, so a consumer must never advance a cursor to it.
92
+ this.pendingEvents.push(event)
93
+ return
94
+ }
95
+
96
+ // One appender at a time, and this is not a precaution — it is the fix
97
+ // for a measured defect. Taking the number, awaiting the write and then
98
+ // committing is a read-modify-write, and emits genuinely interleave:
99
+ // the task store, the plan manager and a batch of parallel tools all
100
+ // emit into this one funnel. Measured on a two-tool run, three events
101
+ // took the number 15 and two took 12. A duplicated sequence is worse
102
+ // than a missing one — a consumer asking for everything above 15 is
103
+ // handed part of the run it already had, spliced in as if it were new.
104
+ const previous = this.appendChain
105
+ let release!: () => void
106
+ this.appendChain = new Promise<void>((resolve) => {
107
+ release = resolve
108
+ })
109
+
110
+ try {
111
+ await previous
112
+
113
+ // The number is a claim that the event is IN the log, so it is taken
114
+ // against the append and not before it. The candidate goes to the
115
+ // store first; only a write that landed advances the counter and
116
+ // reaches the live stream carrying it.
117
+ //
118
+ // The failure path still delivers the event — unstamped. A store
119
+ // that cannot record a `run_failed` must not also swallow it, and an
120
+ // unstamped event says exactly what is true of it: it happened, and
121
+ // it is not recoverable.
122
+ const seq = this.runMgr.nextEventSeq()
123
+ const stamped = {
124
+ ...event,
125
+ seq,
126
+ ...(this.generation !== undefined ? { generation: this.generation } : {}),
127
+ } as RunEvent
128
+
129
+ try {
130
+ await this.runMgr.getRunStore().appendEvent(stamped)
131
+ } catch (err) {
132
+ this.pendingEvents.push(event)
133
+ throw err
134
+ }
135
+
136
+ this.runMgr.commitEventSeq(seq)
137
+ this.pendingEvents.push(stamped)
138
+ } finally {
139
+ release()
76
140
  }
77
141
  };
78
142
 
@@ -15,6 +15,7 @@ import { restoreWorkingState, snapshotWorkingState } from '../../compaction/wire
15
15
  import type { CompactionConfig } from '../../config/runtime.js'
16
16
  import { TOOL_OUTPUT_DIR_NAME } from '../../constants/tools/index.js'
17
17
  import { EmergencySaveManager } from '../../manager/run/emergency.js'
18
+ import type { RunPersistence } from '../../manager/run/persistence.js'
18
19
  import { resolveProviderCapabilities } from '../../provider/capabilities.js'
19
20
  import {
20
21
  type ProviderChainMember,
@@ -59,6 +60,8 @@ import type { LLMProvider } from '../../types/provider/index.js'
59
60
  import type { TaskRouterConfig } from '../../types/router/index.js'
60
61
  import type { ReviewAnswer } from '../../types/run/answer-review.js'
61
62
  import type { CheckpointStore, ClaimFence } from '../../types/run/checkpoint-store.js'
63
+ import type { RunEventCursor, RunEventReplay } from '../../types/run/event-cursor.js'
64
+ import { resolveRunEventReplay } from '../../types/run/event-cursor.js'
62
65
  import type {
63
66
  AgentRunConfig,
64
67
  PrepareStepChain,
@@ -445,6 +448,43 @@ export interface QueryParams {
445
448
  */
446
449
  runStore?: RunStore
447
450
 
451
+ /**
452
+ * Where a reconnecting consumer left off, so this run's stream can start by
453
+ * handing back what it missed.
454
+ *
455
+ * The case this serves is the one that exists without a network hop: the
456
+ * process holding the run died, and the consumer watching it is coming back
457
+ * to a run that has to be resumed. Pair it with `resumeFromCheckpoint` — or
458
+ * reach it through {@link import('./resume-run.js').resumeRun}, which is the
459
+ * surface that does both — and the missed durable events are yielded, in
460
+ * order, before the resumed run emits anything of its own.
461
+ *
462
+ * On a run with no log to catch up on the cursor is answered honestly rather
463
+ * than ignored: a `sinceSeq` above what exists is `cursor_ahead`, not
464
+ * silence.
465
+ *
466
+ * What comes back is message-granular. Streaming deltas are never persisted
467
+ * — see {@link import('../../types/run/store.js').RunStore.appendEvent} —
468
+ * so a late subscriber recovers the assistant text, the tool results and the
469
+ * lifecycle, not the keystroke cadence that produced them.
470
+ */
471
+ eventCursor?: RunEventCursor
472
+
473
+ /**
474
+ * What became of {@link QueryParams.eventCursor}.
475
+ *
476
+ * A callback rather than an event on the stream, because the answer is about
477
+ * the SUBSCRIPTION and not about the run — and rather than a throw, because
478
+ * a stale cursor is a client's problem and must not be able to stop a run
479
+ * from continuing. A host that receives `unavailable` re-derives from the
480
+ * transcript; one that receives nothing at all would splice a hole into its
481
+ * state and never know.
482
+ *
483
+ * Called once, before the run's first event, and only when a cursor was
484
+ * supplied.
485
+ */
486
+ onEventReplay?: (replay: RunEventReplay) => void
487
+
448
488
  runId?: RunId
449
489
 
450
490
  parentRunId?: RunId
@@ -1038,6 +1078,10 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
1038
1078
  // fence exists, the refusal exists, and no checkpoint a RUN writes ever
1039
1079
  // carries a number — so a stalled worker is refused nowhere.
1040
1080
  checkpointMgr.setClaimFence(params.claimFence)
1081
+ // And every EVENT it records carries the same fence as its generation,
1082
+ // so a consumer whose cursor was minted under an older holding is told
1083
+ // the sequence space changed rather than handed a splice from it.
1084
+ eventTranslator.setGeneration(params.claimFence)
1041
1085
 
1042
1086
  // A question raised from inside a tool becomes a real checkpoint
1043
1087
  // here. It used to park under a synthetic id nothing ever wrote, so
@@ -1115,6 +1159,20 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
1115
1159
  try {
1116
1160
  await ctx.runMgr.init()
1117
1161
 
1162
+ // A consumer coming back gets what it missed BEFORE the run says
1163
+ // anything new, which is the only order that lets it fold one
1164
+ // stream into one state. It has to follow `init()` — that is what
1165
+ // binds the store and reads the log's head — and precede every
1166
+ // emit below.
1167
+ if (params.eventCursor) {
1168
+ yield* catchUpFromCursor(
1169
+ ctx.runMgr,
1170
+ params.eventCursor,
1171
+ params.onEventReplay,
1172
+ params.claimFence,
1173
+ )
1174
+ }
1175
+
1118
1176
  // Handed over here, and the position is load-bearing in BOTH
1119
1177
  // directions. It has to follow `wirePlanManager`, or a host that
1120
1178
  // builds its plan in this callback — which is what the callback is
@@ -1594,6 +1652,35 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
1594
1652
  })()
1595
1653
  }
1596
1654
 
1655
+ /**
1656
+ * Hand a returning consumer what it missed, or tell it why it cannot have it.
1657
+ *
1658
+ * Yields NOTHING on a refusal. A partial catch-up is the failure this exists to
1659
+ * prevent: a consumer that receives some of the gap folds it into its state and
1660
+ * cannot tell the state is wrong, where one that receives an explicit
1661
+ * `unavailable` re-derives from the transcript and is right. The run continues
1662
+ * either way — a stale cursor belongs to the client, and must not be able to
1663
+ * stop the work.
1664
+ */
1665
+ async function* catchUpFromCursor(
1666
+ runMgr: RunPersistence,
1667
+ cursor: RunEventCursor,
1668
+ onEventReplay: ((replay: RunEventReplay) => void) | undefined,
1669
+ generation: ClaimFence | undefined,
1670
+ ): AsyncGenerator<RunEvent, void> {
1671
+ const missed = await runMgr.getRunStore().readEvents({ sinceSeq: cursor.sinceSeq })
1672
+ const replay = resolveRunEventReplay(
1673
+ cursor,
1674
+ { lastSeq: runMgr.lastEventSeq, ...(generation !== undefined ? { generation } : {}) },
1675
+ missed,
1676
+ )
1677
+
1678
+ onEventReplay?.(replay)
1679
+
1680
+ if (replay.status !== 'replayed') return
1681
+ for (const event of replay.events) yield event
1682
+ }
1683
+
1597
1684
  export async function drainQuery(
1598
1685
  params: Omit<QueryParams, 'resumeHandler'> & { resumeHandler?: ResumeHandler },
1599
1686
  listener?: RunEventListener,