@namzu/sdk 20.3.0 → 21.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/CHANGELOG.md +321 -0
  2. package/dist/bridge/sse/mapper.d.ts +19 -0
  3. package/dist/bridge/sse/mapper.d.ts.map +1 -1
  4. package/dist/bridge/sse/mapper.js +9 -1
  5. package/dist/bridge/sse/mapper.js.map +1 -1
  6. package/dist/contracts/schemas.d.ts +10 -10
  7. package/dist/manager/run/persistence.d.ts +26 -0
  8. package/dist/manager/run/persistence.d.ts.map +1 -1
  9. package/dist/manager/run/persistence.js +40 -0
  10. package/dist/manager/run/persistence.js.map +1 -1
  11. package/dist/public-runtime.d.ts +5 -0
  12. package/dist/public-runtime.d.ts.map +1 -1
  13. package/dist/public-runtime.js +21 -0
  14. package/dist/public-runtime.js.map +1 -1
  15. package/dist/run/drain.d.ts +194 -0
  16. package/dist/run/drain.d.ts.map +1 -0
  17. package/dist/run/drain.js +228 -0
  18. package/dist/run/drain.js.map +1 -0
  19. package/dist/run/index.d.ts +2 -0
  20. package/dist/run/index.d.ts.map +1 -1
  21. package/dist/run/index.js +1 -0
  22. package/dist/run/index.js.map +1 -1
  23. package/dist/runtime/query/checkpoint.d.ts +27 -1
  24. package/dist/runtime/query/checkpoint.d.ts.map +1 -1
  25. package/dist/runtime/query/checkpoint.js +34 -4
  26. package/dist/runtime/query/checkpoint.js.map +1 -1
  27. package/dist/runtime/query/events.d.ts +12 -0
  28. package/dist/runtime/query/events.d.ts.map +1 -1
  29. package/dist/runtime/query/events.js +61 -3
  30. package/dist/runtime/query/events.js.map +1 -1
  31. package/dist/runtime/query/index.d.ts +57 -1
  32. package/dist/runtime/query/index.d.ts.map +1 -1
  33. package/dist/runtime/query/index.js +36 -0
  34. package/dist/runtime/query/index.js.map +1 -1
  35. package/dist/runtime/query/resume-run.d.ts +34 -1
  36. package/dist/runtime/query/resume-run.d.ts.map +1 -1
  37. package/dist/runtime/query/resume-run.js +16 -3
  38. package/dist/runtime/query/resume-run.js.map +1 -1
  39. package/dist/store/index.d.ts +2 -2
  40. package/dist/store/index.d.ts.map +1 -1
  41. package/dist/store/index.js +2 -2
  42. package/dist/store/index.js.map +1 -1
  43. package/dist/store/run/checkpoint-disk.d.ts +18 -2
  44. package/dist/store/run/checkpoint-disk.d.ts.map +1 -1
  45. package/dist/store/run/checkpoint-disk.js +50 -5
  46. package/dist/store/run/checkpoint-disk.js.map +1 -1
  47. package/dist/store/run/checkpoint-memory.d.ts +22 -2
  48. package/dist/store/run/checkpoint-memory.d.ts.map +1 -1
  49. package/dist/store/run/checkpoint-memory.js +99 -4
  50. package/dist/store/run/checkpoint-memory.js.map +1 -1
  51. package/dist/store/run/claim-disk.d.ts +130 -0
  52. package/dist/store/run/claim-disk.d.ts.map +1 -0
  53. package/dist/store/run/claim-disk.js +550 -0
  54. package/dist/store/run/claim-disk.js.map +1 -0
  55. package/dist/store/run/conformance.d.ts +159 -0
  56. package/dist/store/run/conformance.d.ts.map +1 -0
  57. package/dist/store/run/conformance.js +451 -0
  58. package/dist/store/run/conformance.js.map +1 -0
  59. package/dist/store/run/disk.d.ts +30 -2
  60. package/dist/store/run/disk.d.ts.map +1 -1
  61. package/dist/store/run/disk.js +101 -0
  62. package/dist/store/run/disk.js.map +1 -1
  63. package/dist/store/run/listing.d.ts +44 -1
  64. package/dist/store/run/listing.d.ts.map +1 -1
  65. package/dist/store/run/listing.js +92 -1
  66. package/dist/store/run/listing.js.map +1 -1
  67. package/dist/store/run/memory.d.ts +5 -4
  68. package/dist/store/run/memory.d.ts.map +1 -1
  69. package/dist/store/run/memory.js +30 -1
  70. package/dist/store/run/memory.js.map +1 -1
  71. package/dist/types/doctor/check.d.ts +28 -2
  72. package/dist/types/doctor/check.d.ts.map +1 -1
  73. package/dist/types/run/checkpoint-store.d.ts +178 -2
  74. package/dist/types/run/checkpoint-store.d.ts.map +1 -1
  75. package/dist/types/run/event-cursor.d.ts +90 -0
  76. package/dist/types/run/event-cursor.d.ts.map +1 -0
  77. package/dist/types/run/event-cursor.js +47 -0
  78. package/dist/types/run/event-cursor.js.map +1 -0
  79. package/dist/types/run/events.d.ts +71 -1
  80. package/dist/types/run/events.d.ts.map +1 -1
  81. package/dist/types/run/events.js.map +1 -1
  82. package/dist/types/run/index.d.ts +1 -0
  83. package/dist/types/run/index.d.ts.map +1 -1
  84. package/dist/types/run/index.js +1 -0
  85. package/dist/types/run/index.js.map +1 -1
  86. package/dist/types/run/store.d.ts +41 -1
  87. package/dist/types/run/store.d.ts.map +1 -1
  88. package/package.json +6 -1
  89. package/src/bridge/sse/mapper.ts +28 -1
  90. package/src/manager/run/persistence.ts +43 -0
  91. package/src/public-runtime.ts +27 -0
  92. package/src/run/drain.ts +393 -0
  93. package/src/run/index.ts +3 -0
  94. package/src/runtime/query/checkpoint.ts +42 -5
  95. package/src/runtime/query/events.ts +68 -4
  96. package/src/runtime/query/index.ts +113 -1
  97. package/src/runtime/query/resume-run.ts +77 -12
  98. package/src/store/index.ts +5 -1
  99. package/src/store/run/checkpoint-disk.ts +70 -5
  100. package/src/store/run/checkpoint-memory.ts +118 -3
  101. package/src/store/run/claim-disk.ts +593 -0
  102. package/src/store/run/conformance.ts +705 -0
  103. package/src/store/run/disk.ts +106 -2
  104. package/src/store/run/listing.ts +116 -1
  105. package/src/store/run/memory.ts +35 -5
  106. package/src/types/doctor/check.ts +28 -2
  107. package/src/types/run/checkpoint-store.ts +189 -2
  108. package/src/types/run/event-cursor.ts +118 -0
  109. package/src/types/run/events.ts +80 -3
  110. package/src/types/run/index.ts +1 -0
  111. package/src/types/run/store.ts +43 -1
@@ -5,6 +5,25 @@ import type { RunEvent } from '../../types/run/events.js'
5
5
  export interface MappedStreamEvent {
6
6
  wire: StreamEventType
7
7
  data: Record<string, unknown>
8
+ /**
9
+ * The cursor a client resubscribes at, as `<runId>:<seq>`.
10
+ *
11
+ * Not a bare number, and the reason is structural: a parent's stream also
12
+ * carries its children's events, each numbered in its OWN run's log, so one
13
+ * scalar over a mixed stream would compare positions from two different
14
+ * sequences. The run id is what makes the position addressable — a client
15
+ * keeps one cursor per run id and sends the right one back.
16
+ *
17
+ * This is what an SSE `id:` line should carry, which is why it sits beside
18
+ * the payload rather than inside it: a framer writes it without having to
19
+ * understand what kind of event it is.
20
+ *
21
+ * Absent when the event is not recoverable — every ephemeral event, every
22
+ * event whose durable write failed, and every delegation-lifecycle event
23
+ * that never passed through the run's log at all. A client must not advance
24
+ * its cursor on one, and the absence is how it knows.
25
+ */
26
+ id?: string
8
27
  }
9
28
 
10
29
  type EventTransform<K extends RunEvent['type']> = {
@@ -525,7 +544,15 @@ export function mapRunToStreamEvent(event: RunEvent, runId: RunId): MappedStream
525
544
  data.parent_task_id = annotated.parentTaskId
526
545
  }
527
546
 
528
- return { wire: mapping.wire, data }
547
+ // Keyed on the event's OWN run id, not the stream's. A child's event
548
+ // arriving on a parent's stream is numbered in the child's log, so stamping
549
+ // the enclosing run here would produce a cursor that addresses the wrong
550
+ // sequence — and it would look right.
551
+ return {
552
+ wire: mapping.wire,
553
+ data,
554
+ ...(event.seq !== undefined ? { id: `${event.runId}:${event.seq}` } : {}),
555
+ }
529
556
  }
530
557
 
531
558
  /** @deprecated Use mapRunToStreamEvent */
@@ -174,11 +174,54 @@ export class RunPersistence {
174
174
  return this.runStore.getRunDir()
175
175
  }
176
176
 
177
+ /**
178
+ * Highest sequence already in this run's durable event log.
179
+ *
180
+ * Seeded by {@link init} and advanced by {@link commitEventSeq}. Held here
181
+ * rather than on the event translator because the translator is built
182
+ * before `init` has run and would need a second, forgettable hop to learn
183
+ * the seed — where `init` is already mandatory, already awaited, and
184
+ * already the thing that binds the store.
185
+ */
186
+ private _lastEventSeq = 0
187
+
188
+ /** Highest sequence this run's log holds. Zero before anything is recorded. */
189
+ get lastEventSeq(): number {
190
+ return this._lastEventSeq
191
+ }
192
+
177
193
  async init(): Promise<void> {
178
194
  await this.runStore.initRun(this.run.id, this.run.parentRunId)
195
+ // A resume reuses the run id and therefore the same log, so the counter
196
+ // has to continue that log rather than start a second sequence inside
197
+ // it. Without this, a run recalled after a crash re-numbers from 1 and a
198
+ // consumer holding a cursor at 40 is told, truthfully and uselessly,
199
+ // that there is nothing above it.
200
+ const existing = await this.runStore.readEvents()
201
+ this._lastEventSeq = existing.at(-1)?.seq ?? 0
179
202
  await this.runStore.writeRunMeta(this.run)
180
203
  }
181
204
 
205
+ /**
206
+ * The number the NEXT durable event would take. Not yet taken — see
207
+ * {@link commitEventSeq}.
208
+ */
209
+ nextEventSeq(): number {
210
+ return this._lastEventSeq + 1
211
+ }
212
+
213
+ /**
214
+ * Take the candidate number, once its event is actually in the log.
215
+ *
216
+ * Split from {@link nextEventSeq} so a failed append consumes nothing: a
217
+ * `seq` is a claim that the event is recoverable, and advancing the counter
218
+ * before the write lands would leave a hole no event fills and hand a live
219
+ * consumer a cursor pointing at it.
220
+ */
221
+ commitEventSeq(seq: number): void {
222
+ if (seq > this._lastEventSeq) this._lastEventSeq = seq
223
+ }
224
+
182
225
  markRunning(): void {
183
226
  this.run.status = 'running'
184
227
  }
@@ -128,6 +128,19 @@ export {
128
128
  checkLimitsDetailed,
129
129
  createRunReporter,
130
130
  } from './run/index.js'
131
+ // One bounded pass over a queue of durable runs: list what nobody holds,
132
+ // claim it, work it, release it in a `finally`. Every primitive it composes
133
+ // already shipped and nothing composed them, so an approval inbox and a
134
+ // crash sweeper each still needed a host to write the loop — including the
135
+ // two parts a host writes wrong, the release on the failure path and the
136
+ // `null` claim that is not one. Not a daemon: it makes one pass and returns.
137
+ export { DEFAULT_DRAIN_PAGE_SIZE, drainRuns } from './run/index.js'
138
+ export type {
139
+ DrainFailure,
140
+ DrainRun,
141
+ DrainRunsParams,
142
+ DrainRunsResult,
143
+ } from './run/index.js'
131
144
 
132
145
  // ─── personas, skills, advisory ──────────────────────────────────────────
133
146
 
@@ -218,6 +231,20 @@ export {
218
231
  paginateDurableRuns,
219
232
  toDurableRunEntry,
220
233
  } from './store/index.js'
234
+ // Cross-process possession of a run. `claimRun` REFUSES on a store that
235
+ // cannot arbitrate rather than proceeding unclaimed, because proceeding lets
236
+ // two workers restore one checkpoint, both run its tools and both write under
237
+ // one run id — which loses half the work and reports nothing.
238
+ export { claimRun, fencedOut, releaseRun, toClaimSummary } from './store/index.js'
239
+ // Reading a run's durable event log back — what a consumer that lost its
240
+ // connection catches up through. `readRunEventsIn` takes a directory rather
241
+ // than a bound store because binding one CREATES the run directory, and a read
242
+ // that mints an empty run then reports it as having no events is worse than an
243
+ // error. `resolveRunEventReplay` decides what a cursor is owed, and REFUSES
244
+ // rather than delivering a partial catch-up a consumer would fold into its
245
+ // state without knowing it had a hole in it.
246
+ export { readRunEventsIn } from './store/index.js'
247
+ export { resolveRunEventReplay } from './types/run/event-cursor.js'
221
248
 
222
249
  export {
223
250
  AgentRegistry,
@@ -0,0 +1,393 @@
1
+ /**
2
+ * One pass over a queue of durable runs: list what nobody holds, take it,
3
+ * hand it to a worker, give it back.
4
+ *
5
+ * Every primitive this composes already shipped —
6
+ * {@link import('../store/run/listing.js').listDurableRuns} enumerates runs
7
+ * above a run id, `claimRun` arbitrates between processes, `releaseRun`
8
+ * returns a run to the queue, and `resumeRun` carries a fence into every
9
+ * durable write. Nothing composed them, so the two things the claim was
10
+ * built for — an approval inbox and a crash sweeper — still required a host
11
+ * to write the loop, and writing it correctly means getting the release
12
+ * into a `finally` and the `null` claim out of the error path. Both are the
13
+ * kind of thing a host gets wrong once, quietly.
14
+ *
15
+ * ## What this deliberately is NOT
16
+ *
17
+ * A supervisor, a daemon, or a scheduler. There is no timer here, no
18
+ * process spawn, no retry backoff and no `while (true)`. `drainRuns` makes
19
+ * ONE bounded pass and returns what happened; running it again is the
20
+ * caller's decision, made wherever that caller already has a scheduler. A
21
+ * per-platform supervisor is the same trade the deployment-adapter matrix
22
+ * was rejected for: one seam beats N adapters.
23
+ *
24
+ * The unit of work is a callback, so this module never needs a provider, a
25
+ * tool registry or a sandbox — the half of a run that cannot be serialized
26
+ * stays with the caller, exactly as `resumeRun` already splits it.
27
+ */
28
+
29
+ import { claimRun, listDurableRuns, releaseRun, summarizePark } from '../store/run/listing.js'
30
+ import type { NamzuErrorCode } from '../types/errors/index.js'
31
+ import { NamzuError } from '../types/errors/index.js'
32
+ import type { RunId } from '../types/ids/index.js'
33
+ import type {
34
+ CheckpointListingScope,
35
+ CheckpointStore,
36
+ DurableRunEntry,
37
+ ParkState,
38
+ RunClaim,
39
+ } from '../types/run/checkpoint-store.js'
40
+
41
+ /** Runs handled per pass when the caller names no page size. */
42
+ export const DEFAULT_DRAIN_PAGE_SIZE = 100
43
+
44
+ /**
45
+ * What a drainer does with one run it successfully took.
46
+ *
47
+ * Receives the claim, not just its fence, because the holder and expiry are
48
+ * what a worker needs to decide whether it still has time to start — and
49
+ * because a caller that only ever sees a number tends to forget the lease
50
+ * can lapse under it.
51
+ *
52
+ * The intended body is a resume:
53
+ *
54
+ * ```ts
55
+ * onRun: (entry, claim) =>
56
+ * resumeRun({
57
+ * ...yourQueryParams,
58
+ * scope: { ...entry, threadId },
59
+ * checkpointStore: store,
60
+ * claimFence: claim.fence,
61
+ * })
62
+ * ```
63
+ *
64
+ * `claimFence` is the whole reason the claim is handed over: a write that
65
+ * does not carry it is unfenced, so a worker stalled past its lease would
66
+ * still be able to overwrite the record of whoever took the run over.
67
+ *
68
+ * A throw is recorded against that run and the pass continues. A drainer
69
+ * that died on the first bad run would leave the rest of the queue
70
+ * untouched, which is the failure a queue exists to spread out.
71
+ */
72
+ export type DrainRun = (entry: DurableRunEntry, claim: RunClaim) => void | Promise<void>
73
+
74
+ export interface DrainRunsParams {
75
+ /** Backend to list, claim and release against. Must support all three. */
76
+ readonly store: CheckpointStore
77
+ /** Contiguous prefix — `tenantId` required. See {@link CheckpointListingScope}. */
78
+ readonly scope: CheckpointListingScope
79
+ /**
80
+ * Who is taking the runs. Per-PROCESS, never per-deployment: `holder` is
81
+ * the only thing that distinguishes a renewal from a theft, so two
82
+ * drainers sharing a string take live claims from each other instantly.
83
+ */
84
+ readonly holder: string
85
+ /** Lease length in ms. Long enough that the slowest run finishes inside it. */
86
+ readonly ttlMs: number
87
+ /** The work. See {@link DrainRun}. */
88
+ readonly onRun: DrainRun
89
+
90
+ /**
91
+ * Keep only runs whose park is in one of these states.
92
+ *
93
+ * **Absent means every run with durable state, parked or not**, and that
94
+ * is not a placeholder default — it is what a crash sweep wants, because
95
+ * a run that died mid-flight never parked and would be invisible under
96
+ * any park filter. An approval inbox passes `['outstanding']`; a
97
+ * reclamation sweep passes `['expired']`.
98
+ */
99
+ readonly park?: readonly ParkState[]
100
+
101
+ /**
102
+ * Stop taking new runs. Work already in flight is NOT interrupted — this
103
+ * module owns no run and cannot cancel one; a caller that needs to abort
104
+ * the work itself passes the same signal into whatever `onRun` starts.
105
+ */
106
+ readonly signal?: AbortSignal
107
+
108
+ /**
109
+ * How many runs may be in flight at once. Defaults to 1.
110
+ *
111
+ * Bounded on purpose. The obvious implementation — claim everything, then
112
+ * `Promise.all` — holds N leases while doing one run's worth of work, so
113
+ * the runs at the back of the batch expire before they are started and
114
+ * are taken by somebody else mid-flight.
115
+ */
116
+ readonly maxConcurrent?: number
117
+
118
+ /** Listing page size. See {@link DEFAULT_DRAIN_PAGE_SIZE}. */
119
+ readonly pageSize?: number
120
+
121
+ /**
122
+ * Clock for expiry, so one pass judges every claim against one instant
123
+ * and a test does not have to wait out a lease.
124
+ */
125
+ readonly now?: number
126
+ }
127
+
128
+ /** A run a pass could not finish, and why. */
129
+ export interface DrainFailure {
130
+ readonly runId: RunId
131
+ readonly error: string
132
+ }
133
+
134
+ /** What one pass did. */
135
+ export interface DrainRunsResult {
136
+ /** Rows the listing returned, before any of them were contended for. */
137
+ readonly listed: number
138
+ /** Runs whose `onRun` returned. */
139
+ readonly drained: readonly RunId[]
140
+ /**
141
+ * Runs another worker held. Not failures: "somebody got there first" is
142
+ * the ordinary outcome of a queue with more than one reader.
143
+ */
144
+ readonly skipped: readonly RunId[]
145
+ /**
146
+ * Runs that stopped matching {@link DrainRunsParams.park} between the
147
+ * listing and the claim, and were given straight back.
148
+ *
149
+ * Separate from {@link DrainRunsResult.skipped} because the cause is
150
+ * different and so is what an operator should do about a lot of them: a
151
+ * skip means another drainer is holding runs right now, a stale entry
152
+ * means another drainer already FINISHED one. Empty on a pass with no
153
+ * park filter, which has nothing to re-check against.
154
+ */
155
+ readonly stale: readonly RunId[]
156
+ /** Runs whose `onRun` threw. */
157
+ readonly failed: readonly DrainFailure[]
158
+ /**
159
+ * Runs that finished but whose lease could not be handed back.
160
+ *
161
+ * Separate from {@link DrainRunsResult.failed} because it is a different
162
+ * fact with a different consequence: the work is done and the record is
163
+ * written; the run is merely unavailable to the next reader until the
164
+ * lease lapses. Reported rather than swallowed — a release that quietly
165
+ * did nothing is how a queue silently loses throughput.
166
+ */
167
+ readonly unreleased: readonly DrainFailure[]
168
+ /** Whether the pass stopped early because the signal aborted. */
169
+ readonly stopped: boolean
170
+ }
171
+
172
+ function refuse(code: NamzuErrorCode, message: string, details: Record<string, unknown>): never {
173
+ throw new NamzuError({ code, message, details })
174
+ }
175
+
176
+ function toMessage(err: unknown): string {
177
+ return err instanceof Error ? err.message : String(err)
178
+ }
179
+
180
+ /**
181
+ * Refuse a store that cannot do the whole job, BEFORE anything is listed.
182
+ *
183
+ * Checked up front rather than at the first call that needs each method, so
184
+ * that a store missing only `releaseRun` cannot resume half a queue and then
185
+ * discover it has no way to give the runs back. The optional-capability rule
186
+ * on `CheckpointStore` says a caller refuses rather than degrades; a drainer
187
+ * that degraded would be the worst instance of it, because "claimed by
188
+ * default" here means every worker proceeds on every run.
189
+ */
190
+ function assertDrainable(store: CheckpointStore): void {
191
+ const missing = (['listDurableRuns', 'claimRun', 'releaseRun'] as const).filter(
192
+ (m) => typeof store[m] !== 'function',
193
+ )
194
+ if (missing.length === 0) return
195
+ refuse(
196
+ 'capability_unavailable',
197
+ `drainRuns: the injected checkpoint store does not implement ${missing.map((m) => `\`${m}\``).join(', ')}, so it cannot arbitrate a queue. Refusing before anything is claimed rather than draining what it can — a drainer that proceeded without a claim would let two workers restore one checkpoint, both execute its tools and both write under one run id. Supply a store that implements all three (the built-in disk and in-memory stores do), or run a single writer per run.`,
198
+ { missing },
199
+ )
200
+ }
201
+
202
+ /**
203
+ * Take every unclaimed run under a scope, one bounded pass, and give each
204
+ * one back when its work returns.
205
+ *
206
+ * The shape is: list parked-and-unclaimed → claim → work → release in a
207
+ * `finally`. The `finally` is the part a host writes wrong: a worker that
208
+ * returns without releasing leaves the run stuck until the lease lapses,
209
+ * and a worker that releases only on success leaves a FAILED run stuck for
210
+ * the same duration — so a queue quietly loses its throughput to the runs
211
+ * that need retrying most.
212
+ *
213
+ * `claimed: false` is not a parameter. A drainer never wants work somebody
214
+ * else holds; that is what makes it a drainer rather than a listing. An
215
+ * expired claim counts as unheld, which is what makes a dead worker's runs
216
+ * recoverable at all.
217
+ *
218
+ * ## What "exactly once" does and does not mean here
219
+ *
220
+ * Two drainers never hold one run at the same time — that is the claim, and
221
+ * it is absolute. **Exactly-once over a whole pass is a weaker promise, and
222
+ * where it holds it comes from the FILTER, not from the claim.** A listing
223
+ * is a snapshot; between paging a row and claiming it, another drainer can
224
+ * finish that run and release it, and the claim then succeeds on work
225
+ * already done. So a claimed row is re-read against
226
+ * {@link DrainRunsParams.park} before any work starts, and one that no
227
+ * longer matches is given straight back as {@link DrainRunsResult.stale}.
228
+ * An inbox drain (`park: ['outstanding']`) whose work answers the park is
229
+ * therefore exactly-once, because doing the work is what removes the run
230
+ * from the queue.
231
+ *
232
+ * With NO park filter there is nothing to re-check, and two drainers can
233
+ * both process one run. That is not an omission: a checkpoint store holds no
234
+ * run STATUS by design — nothing in it distinguishes a run that finished
235
+ * from one that died — so "already done" is a fact only the host's own run
236
+ * records carry. A crash sweep intersects with those records inside
237
+ * `onRun`, which is the shape {@link DurableRunEntry} already prescribes.
238
+ *
239
+ * @throws NamzuError `capability_unavailable` when the store cannot list,
240
+ * claim or release — before any run is touched.
241
+ * @throws NamzuError `invalid_config` on a lease or concurrency that cannot
242
+ * mean what it says.
243
+ */
244
+ export async function drainRuns(params: DrainRunsParams): Promise<DrainRunsResult> {
245
+ const { store, scope, holder, ttlMs, onRun, park, signal, now } = params
246
+
247
+ assertDrainable(store)
248
+
249
+ if (holder.trim().length === 0) {
250
+ refuse(
251
+ 'invalid_config',
252
+ 'drainRuns: `holder` is empty. It is the only thing that distinguishes a renewal from a theft, so two drainers sharing one string take live claims from each other instantly. Use something per-process — a worker id, a pod name plus a pid.',
253
+ { holder },
254
+ )
255
+ }
256
+ if (!Number.isFinite(ttlMs) || ttlMs <= 0) {
257
+ refuse(
258
+ 'invalid_config',
259
+ `drainRuns: ttlMs must be a positive number of milliseconds, got ${String(ttlMs)}. A lease that expires immediately is a lease every worker can take at once, which is the condition a claim exists to prevent.`,
260
+ { ttlMs },
261
+ )
262
+ }
263
+ const maxConcurrent = params.maxConcurrent ?? 1
264
+ if (!Number.isInteger(maxConcurrent) || maxConcurrent < 1) {
265
+ refuse(
266
+ 'invalid_config',
267
+ `drainRuns: maxConcurrent must be a positive integer, got ${String(params.maxConcurrent)}. Zero would drain nothing while reporting a successful pass.`,
268
+ { maxConcurrent: params.maxConcurrent },
269
+ )
270
+ }
271
+ const pageSize = params.pageSize ?? DEFAULT_DRAIN_PAGE_SIZE
272
+
273
+ const drained: RunId[] = []
274
+ const skipped: RunId[] = []
275
+ const stale: RunId[] = []
276
+ const failed: DrainFailure[] = []
277
+ const unreleased: DrainFailure[] = []
278
+ let listed = 0
279
+ let stopped = false
280
+
281
+ const giveBack = async (entry: DurableRunEntry, fence: number): Promise<void> => {
282
+ try {
283
+ await releaseRun(store, entry, fence)
284
+ } catch (err) {
285
+ // Never rethrown: on the work path this runs inside a `finally`
286
+ // unwinding the caller's error, and replacing it would send the
287
+ // operator to debug the disk instead of the run.
288
+ unreleased.push({ runId: entry.runId, error: toMessage(err) })
289
+ }
290
+ }
291
+
292
+ /**
293
+ * Is this row still the row the listing described?
294
+ *
295
+ * A listing is a SNAPSHOT, and a claim taken against a stale snapshot is
296
+ * a claim on work somebody already did. The window is real and small:
297
+ * drainer B pages the queue, drainer A takes a run, finishes it, answers
298
+ * its park and releases — and B's claim then succeeds on a run that is no
299
+ * longer outstanding. Mutual exclusion cannot close that; only re-reading
300
+ * after the claim can, which is why this is here and not in the store.
301
+ *
302
+ * Only the park is re-checked, because it is the only predicate this loop
303
+ * was given. **Two drainers with no park filter can both process one
304
+ * run**, and no amount of claiming prevents it: a checkpoint store holds
305
+ * no run STATUS by design — see the note on {@link DurableRunEntry} — so
306
+ * "already done" is a fact only the host's own run records carry. A crash
307
+ * sweep intersects with those records inside `onRun`.
308
+ */
309
+ const stillMatches = async (entry: DurableRunEntry): Promise<boolean> => {
310
+ if (!park) return true
311
+ const fresh = summarizePark(await store.listCheckpoints(entry), now ?? Date.now())
312
+ return fresh !== undefined && park.includes(fresh.state)
313
+ }
314
+
315
+ /**
316
+ * One run: take it, work it, give it back.
317
+ *
318
+ * **No cancellation check here, and its absence is deliberate.** One was
319
+ * written, and a mutation test found nothing could kill it: the batch
320
+ * below dispatches with `.map(handle)`, which calls every handler
321
+ * synchronously before any of them awaits, so a signal that aborts during
322
+ * a batch cannot be observed at the top of a handler that has already
323
+ * been entered — and a signal that aborts BETWEEN batches is caught by
324
+ * the check in the loop, which runs first. A branch nothing can reach is
325
+ * a declaration nothing drives, so it is gone rather than covered by a
326
+ * test that would have proved nothing
327
+ * (`docs/conventions/declared-but-undriven.md`).
328
+ */
329
+ const handle = async (entry: DurableRunEntry): Promise<void> => {
330
+ const claim = await claimRun(store, entry, {
331
+ holder,
332
+ ttlMs,
333
+ ...(now !== undefined ? { now } : {}),
334
+ })
335
+ // `null` is not an error. Another worker got there first, which is the
336
+ // ordinary outcome of two readers on one queue.
337
+ if (!claim) {
338
+ skipped.push(entry.runId)
339
+ return
340
+ }
341
+ // Checked with the claim in hand rather than before taking it: only
342
+ // under the claim is the answer stable, because nobody else can change
343
+ // it while this drainer holds the run.
344
+ if (!(await stillMatches(entry))) {
345
+ stale.push(entry.runId)
346
+ await giveBack(entry, claim.fence)
347
+ return
348
+ }
349
+ try {
350
+ await onRun(entry, claim)
351
+ drained.push(entry.runId)
352
+ } catch (err) {
353
+ failed.push({ runId: entry.runId, error: toMessage(err) })
354
+ } finally {
355
+ await giveBack(entry, claim.fence)
356
+ }
357
+ }
358
+
359
+ let cursor: string | undefined
360
+ do {
361
+ if (signal?.aborted) {
362
+ stopped = true
363
+ break
364
+ }
365
+ const page = await listDurableRuns(store, scope, {
366
+ // Not a parameter. See the note above.
367
+ claimed: false,
368
+ ...(park ? { park } : {}),
369
+ limit: pageSize,
370
+ ...(cursor !== undefined ? { cursor } : {}),
371
+ ...(now !== undefined ? { now } : {}),
372
+ })
373
+ listed += page.entries.length
374
+
375
+ // Windowed rather than `Promise.all` over the page: the leases are taken
376
+ // as the work starts, so a page bigger than the pass can finish inside
377
+ // one TTL does not hand the tail of it to somebody else mid-flight.
378
+ for (let i = 0; i < page.entries.length; i += maxConcurrent) {
379
+ if (signal?.aborted) {
380
+ stopped = true
381
+ break
382
+ }
383
+ await Promise.all(page.entries.slice(i, i + maxConcurrent).map(handle))
384
+ }
385
+
386
+ // A drained run is released, so it is unclaimed again — but the cursor
387
+ // is a position in a total order and has already passed it, so the pass
388
+ // cannot see it twice and cannot fail to terminate.
389
+ cursor = stopped ? undefined : page.cursor
390
+ } while (cursor !== undefined)
391
+
392
+ return { listed, drained, skipped, stale, failed, unreleased, stopped }
393
+ }
package/src/run/index.ts CHANGED
@@ -5,5 +5,8 @@ export { RunDiskStore } from '../store/run/disk.js'
5
5
  export { createRunReporter } from './reporter.js'
6
6
  export type { RunReporter } from './reporter.js'
7
7
 
8
+ export { DEFAULT_DRAIN_PAGE_SIZE, drainRuns } from './drain.js'
9
+ export type { DrainFailure, DrainRun, DrainRunsParams, DrainRunsResult } from './drain.js'
10
+
8
11
  export { checkLimitsDetailed, buildLimitConfig } from './LimitChecker.js'
9
12
  export type { LimitCheckerState, LimitCheckResult } from './LimitChecker.js'
@@ -11,7 +11,11 @@ import type {
11
11
  PendingDecision,
12
12
  } from '../../types/hitl/index.js'
13
13
  import type { AssistantMessage } from '../../types/message/index.js'
14
- import type { CheckpointRunScope, CheckpointStore } from '../../types/run/checkpoint-store.js'
14
+ import type {
15
+ CheckpointRunScope,
16
+ CheckpointStore,
17
+ ClaimFence,
18
+ } from '../../types/run/checkpoint-store.js'
15
19
  import type { EmergencySaveData } from '../../types/run/emergency.js'
16
20
  import type { CheckpointListEntry } from '../../types/run/replay.js'
17
21
  import { ZERO_COST } from '../../utils/cost.js'
@@ -154,6 +158,22 @@ export class CheckpointManager {
154
158
  /** See {@link setParkTtl}. */
155
159
  private parkTtlMs?: number
156
160
 
161
+ /**
162
+ * The claim this run holds, presented on every checkpoint write.
163
+ *
164
+ * Unset means unfenced, which is correct for a single-writer deployment
165
+ * and is what every run did before claims existed. Set it and a write from
166
+ * a superseded holding is refused by the store.
167
+ *
168
+ * This existed nowhere for one release, and the omission was invisible in
169
+ * the worst way: the claim, the fence and the refusal were all built and
170
+ * tested, and no code path between a run and its store carried the number,
171
+ * so every checkpoint a RUN wrote went out unfenced. A capability that is
172
+ * complete except for the wire between its halves reads exactly like a
173
+ * working one.
174
+ */
175
+ private claimFence?: ClaimFence
176
+
157
177
  /**
158
178
  * The run's attribution instant, stamped onto every checkpoint this
159
179
  * manager writes.
@@ -234,7 +254,7 @@ export class CheckpointManager {
234
254
  traceContext: this.traceSource?.(),
235
255
  }
236
256
 
237
- await this.store.writeCheckpoint(this.scope, checkpoint)
257
+ await this.store.writeCheckpoint(this.scope, checkpoint, this.claimFence)
238
258
  this.lastCreatedId = checkpoint.id
239
259
  return checkpoint
240
260
  }
@@ -298,10 +318,27 @@ export class CheckpointManager {
298
318
  ...(ttl !== undefined && ttl > 0 ? { deadlineAt: parkedAt + ttl } : {}),
299
319
  },
300
320
  }
301
- await this.store.writeCheckpoint(this.scope, parked)
321
+ await this.store.writeCheckpoint(this.scope, parked, this.claimFence)
302
322
  return parked
303
323
  }
304
324
 
325
+ /**
326
+ * Present this claim on every subsequent write. See {@link claimFence}.
327
+ *
328
+ * A setter rather than a constructor argument because a run is claimed at
329
+ * a different moment than it is constructed — a worker draining a queue
330
+ * takes the run, then builds the pipeline around it — and because a
331
+ * renewal mints a NEW fence mid-run that has to replace the old one.
332
+ */
333
+ setClaimFence(fence: ClaimFence | undefined): void {
334
+ this.claimFence = fence
335
+ }
336
+
337
+ /** The claim currently presented on writes, if any. */
338
+ get presentedFence(): ClaimFence | undefined {
339
+ return this.claimFence
340
+ }
341
+
305
342
  /** Default time-to-live applied to every park this manager records. */
306
343
  setParkTtl(ttlMs: number | undefined): void {
307
344
  this.parkTtlMs = ttlMs
@@ -329,7 +366,7 @@ export class CheckpointManager {
329
366
  decision: { action: 'pause', reason: 'The approval request expired without an answer.' },
330
367
  },
331
368
  }
332
- await this.store.writeCheckpoint(this.scope, expired)
369
+ await this.store.writeCheckpoint(this.scope, expired, this.claimFence)
333
370
  return expired
334
371
  }
335
372
 
@@ -352,7 +389,7 @@ export class CheckpointManager {
352
389
  ...checkpoint,
353
390
  pending: { ...checkpoint.pending, resolvedAt: Date.now(), decision },
354
391
  }
355
- await this.store.writeCheckpoint(this.scope, resolved)
392
+ await this.store.writeCheckpoint(this.scope, resolved, this.claimFence)
356
393
  return resolved
357
394
  }
358
395