@namzu/sdk 20.3.0 → 21.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/CHANGELOG.md +321 -0
  2. package/dist/bridge/sse/mapper.d.ts +19 -0
  3. package/dist/bridge/sse/mapper.d.ts.map +1 -1
  4. package/dist/bridge/sse/mapper.js +9 -1
  5. package/dist/bridge/sse/mapper.js.map +1 -1
  6. package/dist/contracts/schemas.d.ts +10 -10
  7. package/dist/manager/run/persistence.d.ts +26 -0
  8. package/dist/manager/run/persistence.d.ts.map +1 -1
  9. package/dist/manager/run/persistence.js +40 -0
  10. package/dist/manager/run/persistence.js.map +1 -1
  11. package/dist/public-runtime.d.ts +5 -0
  12. package/dist/public-runtime.d.ts.map +1 -1
  13. package/dist/public-runtime.js +21 -0
  14. package/dist/public-runtime.js.map +1 -1
  15. package/dist/run/drain.d.ts +194 -0
  16. package/dist/run/drain.d.ts.map +1 -0
  17. package/dist/run/drain.js +228 -0
  18. package/dist/run/drain.js.map +1 -0
  19. package/dist/run/index.d.ts +2 -0
  20. package/dist/run/index.d.ts.map +1 -1
  21. package/dist/run/index.js +1 -0
  22. package/dist/run/index.js.map +1 -1
  23. package/dist/runtime/query/checkpoint.d.ts +27 -1
  24. package/dist/runtime/query/checkpoint.d.ts.map +1 -1
  25. package/dist/runtime/query/checkpoint.js +34 -4
  26. package/dist/runtime/query/checkpoint.js.map +1 -1
  27. package/dist/runtime/query/events.d.ts +12 -0
  28. package/dist/runtime/query/events.d.ts.map +1 -1
  29. package/dist/runtime/query/events.js +61 -3
  30. package/dist/runtime/query/events.js.map +1 -1
  31. package/dist/runtime/query/index.d.ts +57 -1
  32. package/dist/runtime/query/index.d.ts.map +1 -1
  33. package/dist/runtime/query/index.js +36 -0
  34. package/dist/runtime/query/index.js.map +1 -1
  35. package/dist/runtime/query/resume-run.d.ts +34 -1
  36. package/dist/runtime/query/resume-run.d.ts.map +1 -1
  37. package/dist/runtime/query/resume-run.js +16 -3
  38. package/dist/runtime/query/resume-run.js.map +1 -1
  39. package/dist/store/index.d.ts +2 -2
  40. package/dist/store/index.d.ts.map +1 -1
  41. package/dist/store/index.js +2 -2
  42. package/dist/store/index.js.map +1 -1
  43. package/dist/store/run/checkpoint-disk.d.ts +18 -2
  44. package/dist/store/run/checkpoint-disk.d.ts.map +1 -1
  45. package/dist/store/run/checkpoint-disk.js +50 -5
  46. package/dist/store/run/checkpoint-disk.js.map +1 -1
  47. package/dist/store/run/checkpoint-memory.d.ts +22 -2
  48. package/dist/store/run/checkpoint-memory.d.ts.map +1 -1
  49. package/dist/store/run/checkpoint-memory.js +99 -4
  50. package/dist/store/run/checkpoint-memory.js.map +1 -1
  51. package/dist/store/run/claim-disk.d.ts +130 -0
  52. package/dist/store/run/claim-disk.d.ts.map +1 -0
  53. package/dist/store/run/claim-disk.js +550 -0
  54. package/dist/store/run/claim-disk.js.map +1 -0
  55. package/dist/store/run/conformance.d.ts +159 -0
  56. package/dist/store/run/conformance.d.ts.map +1 -0
  57. package/dist/store/run/conformance.js +451 -0
  58. package/dist/store/run/conformance.js.map +1 -0
  59. package/dist/store/run/disk.d.ts +30 -2
  60. package/dist/store/run/disk.d.ts.map +1 -1
  61. package/dist/store/run/disk.js +101 -0
  62. package/dist/store/run/disk.js.map +1 -1
  63. package/dist/store/run/listing.d.ts +44 -1
  64. package/dist/store/run/listing.d.ts.map +1 -1
  65. package/dist/store/run/listing.js +92 -1
  66. package/dist/store/run/listing.js.map +1 -1
  67. package/dist/store/run/memory.d.ts +5 -4
  68. package/dist/store/run/memory.d.ts.map +1 -1
  69. package/dist/store/run/memory.js +30 -1
  70. package/dist/store/run/memory.js.map +1 -1
  71. package/dist/types/doctor/check.d.ts +28 -2
  72. package/dist/types/doctor/check.d.ts.map +1 -1
  73. package/dist/types/run/checkpoint-store.d.ts +178 -2
  74. package/dist/types/run/checkpoint-store.d.ts.map +1 -1
  75. package/dist/types/run/event-cursor.d.ts +90 -0
  76. package/dist/types/run/event-cursor.d.ts.map +1 -0
  77. package/dist/types/run/event-cursor.js +47 -0
  78. package/dist/types/run/event-cursor.js.map +1 -0
  79. package/dist/types/run/events.d.ts +71 -1
  80. package/dist/types/run/events.d.ts.map +1 -1
  81. package/dist/types/run/events.js.map +1 -1
  82. package/dist/types/run/index.d.ts +1 -0
  83. package/dist/types/run/index.d.ts.map +1 -1
  84. package/dist/types/run/index.js +1 -0
  85. package/dist/types/run/index.js.map +1 -1
  86. package/dist/types/run/store.d.ts +41 -1
  87. package/dist/types/run/store.d.ts.map +1 -1
  88. package/package.json +6 -1
  89. package/src/bridge/sse/mapper.ts +28 -1
  90. package/src/manager/run/persistence.ts +43 -0
  91. package/src/public-runtime.ts +27 -0
  92. package/src/run/drain.ts +393 -0
  93. package/src/run/index.ts +3 -0
  94. package/src/runtime/query/checkpoint.ts +42 -5
  95. package/src/runtime/query/events.ts +68 -4
  96. package/src/runtime/query/index.ts +113 -1
  97. package/src/runtime/query/resume-run.ts +77 -12
  98. package/src/store/index.ts +5 -1
  99. package/src/store/run/checkpoint-disk.ts +70 -5
  100. package/src/store/run/checkpoint-memory.ts +118 -3
  101. package/src/store/run/claim-disk.ts +593 -0
  102. package/src/store/run/conformance.ts +705 -0
  103. package/src/store/run/disk.ts +106 -2
  104. package/src/store/run/listing.ts +116 -1
  105. package/src/store/run/memory.ts +35 -5
  106. package/src/types/doctor/check.ts +28 -2
  107. package/src/types/run/checkpoint-store.ts +189 -2
  108. package/src/types/run/event-cursor.ts +118 -0
  109. package/src/types/run/events.ts +80 -3
  110. package/src/types/run/index.ts +1 -0
  111. package/src/types/run/store.ts +43 -1
@@ -1,8 +1,8 @@
1
1
  import { appendFile, mkdir, readFile, readdir, unlink } from 'node:fs/promises'
2
2
  import { join } from 'node:path'
3
3
  import type { CheckpointId, IterationCheckpoint } from '../../types/hitl/index.js'
4
- import type { Run, RunEvent, RunStoreConfig } from '../../types/run/index.js'
5
- import type { CompletedToolRecord, RunStore } from '../../types/run/store.js'
4
+ import type { PersistedRunEvent, Run, RunEvent, RunStoreConfig } from '../../types/run/index.js'
5
+ import type { CompletedToolRecord, ReadRunEventsOptions, RunStore } from '../../types/run/store.js'
6
6
  import { atomicWriteFile } from '../../utils/atomic-write.js'
7
7
  import { type Logger, getRootLogger } from '../../utils/logger.js'
8
8
  import { defineSchema, migrate, stamp } from '../schema.js'
@@ -51,6 +51,7 @@ export class RunDiskStore implements RunStore {
51
51
  this.runDir = join(this.baseDir, runId)
52
52
  }
53
53
  await mkdir(this.runDir, { recursive: true })
54
+ await healTornTranscript(this.runDir)
54
55
  this.log.info(`Run directory created: ${this.runDir}`)
55
56
  return this.runDir
56
57
  }
@@ -66,6 +67,10 @@ export class RunDiskStore implements RunStore {
66
67
  await appendFile(join(dir, 'transcript.jsonl'), line, 'utf-8')
67
68
  }
68
69
 
70
+ async readEvents(options?: ReadRunEventsOptions): Promise<readonly PersistedRunEvent[]> {
71
+ return readRunEventsIn(this.requireInit(), options)
72
+ }
73
+
69
74
  /**
70
75
  * Every tool call this run has already finished, keyed by `toolUseId`.
71
76
  *
@@ -296,6 +301,105 @@ export class RunDiskStore implements RunStore {
296
301
  }
297
302
  }
298
303
 
304
+ /**
305
+ * Every durable event under one run directory, oldest first.
306
+ *
307
+ * A free function for the same reason {@link readCheckpointsIn} is one: a
308
+ * caller catching up on a run this process never started would otherwise have
309
+ * to bind a {@link RunDiskStore} to read it, and binding one CREATES the
310
+ * directory. A read that mints an empty run directory then answers "no events"
311
+ * is indistinguishable from a run that genuinely has none.
312
+ *
313
+ * ## Unsequenced lines take their position
314
+ *
315
+ * A transcript written before events were numbered carries no `seq` at all.
316
+ * Numbering those lines by their 1-based position is what keeps their evidence
317
+ * reachable: a legacy run of five lines reads back as 1..5, seeds the emitter
318
+ * at 5, and its next event is 6 — continuous, and stable on every later read.
319
+ * Skipping them instead would erase a run's whole history from a catch-up, and
320
+ * synthesising nothing at all would put the emitter back at 1 on top of a log
321
+ * that already has five entries.
322
+ *
323
+ * A damaged line is skipped rather than refused, which is the one place this
324
+ * differs from the checkpoint reader next door, and deliberately: a checkpoint
325
+ * is read to RESUME from, so a damaged one must stop the resume, while the
326
+ * transcript is read to REPORT from, and dropping every event after a torn line
327
+ * would be a larger loss than the torn line itself. The position count still
328
+ * advances over it, so the numbering of the events after it is unchanged.
329
+ */
330
+ export async function readRunEventsIn(
331
+ runDir: string,
332
+ options?: ReadRunEventsOptions,
333
+ ): Promise<readonly PersistedRunEvent[]> {
334
+ let raw: string
335
+ try {
336
+ raw = await readFile(join(runDir, 'transcript.jsonl'), 'utf-8')
337
+ } catch (err) {
338
+ if (isFileNotFound(err)) return []
339
+ throw err
340
+ }
341
+
342
+ const sinceSeq = options?.sinceSeq ?? 0
343
+ const events: PersistedRunEvent[] = []
344
+ let position = 0
345
+
346
+ for (const line of raw.split('\n')) {
347
+ if (line.length === 0) continue
348
+ position += 1
349
+
350
+ let parsed: Record<string, unknown>
351
+ try {
352
+ parsed = JSON.parse(line) as Record<string, unknown>
353
+ } catch {
354
+ continue
355
+ }
356
+ if (parsed === null || typeof parsed !== 'object' || typeof parsed.type !== 'string') continue
357
+
358
+ const seq = typeof parsed.seq === 'number' ? parsed.seq : position
359
+ if (seq <= sinceSeq) continue
360
+
361
+ events.push({
362
+ ...parsed,
363
+ seq,
364
+ // Stamped by `appendEvent` since long before it was declared. A line
365
+ // that predates even that gets the only honest answer available:
366
+ // zero, which sorts before every real moment and cannot be mistaken
367
+ // for one.
368
+ timestamp: typeof parsed.timestamp === 'number' ? parsed.timestamp : 0,
369
+ } as unknown as PersistedRunEvent)
370
+ }
371
+
372
+ return events
373
+ }
374
+
375
+ /**
376
+ * Terminate a transcript whose last line was cut off mid-write.
377
+ *
378
+ * A process killed during `appendFile` leaves a fragment with no newline. The
379
+ * next append lands on the same line, so the fragment and a WHOLE, correct
380
+ * event merge into one unparsable line — and the reader skips it. The event was
381
+ * written, the emitter counted it as durable, and it is gone.
382
+ *
383
+ * Ending the fragment is enough. It stays unreadable and is skipped as it
384
+ * always was; everything appended after it survives, which is the difference
385
+ * between losing one event and losing one event plus the next.
386
+ *
387
+ * Called from `initRun`, which is the only moment the store knows nothing is
388
+ * mid-write.
389
+ */
390
+ async function healTornTranscript(runDir: string): Promise<void> {
391
+ const path = join(runDir, 'transcript.jsonl')
392
+ let raw: string
393
+ try {
394
+ raw = await readFile(path, 'utf-8')
395
+ } catch (err) {
396
+ if (isFileNotFound(err)) return
397
+ throw err
398
+ }
399
+ if (raw.length === 0 || raw.endsWith('\n')) return
400
+ await appendFile(path, '\n', 'utf-8')
401
+ }
402
+
299
403
  /**
300
404
  * Every checkpoint stored under one run directory, ascending by `createdAt`.
301
405
  *
@@ -14,12 +14,16 @@ import type {
14
14
  CheckpointListingScope,
15
15
  CheckpointRunScope,
16
16
  CheckpointStore,
17
+ ClaimFence,
18
+ ClaimRunOptions,
19
+ ClaimSummary,
17
20
  DurableRunEntry,
18
21
  DurableRunOrder,
19
22
  DurableRunPage,
20
23
  ListDurableRunsOptions,
21
24
  ParkState,
22
25
  ParkSummary,
26
+ RunClaim,
23
27
  } from '../../types/run/checkpoint-store.js'
24
28
 
25
29
  /** Page size when the caller names none. */
@@ -186,11 +190,20 @@ export function paginateDurableRuns(
186
190
  options?: ListDurableRunsOptions,
187
191
  ): DurableRunPage {
188
192
  const wanted = options?.park
189
- const filtered =
193
+ const byPark =
190
194
  wanted && wanted.length > 0
191
195
  ? entries.filter((e) => e.park !== undefined && wanted.includes(e.park.state))
192
196
  : entries
193
197
 
198
+ // An expired claim counts as unheld. That is what expiry means, and a
199
+ // queue reader that treated an expired claim as held would leave a dead
200
+ // worker's runs invisible forever — the exact failure a lease exists to
201
+ // prevent, reintroduced by the filter that reads it.
202
+ const filtered =
203
+ options?.claimed === undefined
204
+ ? byPark
205
+ : byPark.filter((e) => (e.claim !== undefined && !e.claim.expired) === options.claimed)
206
+
194
207
  // Both orders sort on a key that cannot move under a paging caller — see
195
208
  // the contract comment on `listDurableRuns`.
196
209
  const orderBy = options?.orderBy ?? 'runId'
@@ -305,3 +318,105 @@ export async function listDurableRuns(
305
318
  assertContiguousListingScope(scope, 'listDurableRuns')
306
319
  return store.listDurableRuns(scope, options)
307
320
  }
321
+
322
+ /**
323
+ * Take working possession of a run, refusing when the store cannot arbitrate.
324
+ *
325
+ * The refusal is the entire safety property. `claimRun` is optional on the
326
+ * contract, and the natural way to reach an absent optional method is to skip
327
+ * it — which here means every worker proceeds, believing it holds a run
328
+ * nobody arbitrated. Two workers then restore the same checkpoint, both run
329
+ * the tools, and both write under one run id; half the work vanishes with no
330
+ * error anywhere.
331
+ *
332
+ * So a store with no claim support does not get "claimed by default". It gets
333
+ * an error naming the deployment shape it cannot support. A single-writer
334
+ * host never calls this and is unaffected.
335
+ *
336
+ * Returns `null` — not an error — when another holder has the run. That is
337
+ * the ordinary outcome of two readers on one queue, and a caller loops to the
338
+ * next run rather than handling a fault.
339
+ */
340
+ export async function claimRun(
341
+ store: CheckpointStore,
342
+ scope: CheckpointRunScope,
343
+ options: ClaimRunOptions,
344
+ ): Promise<RunClaim | null> {
345
+ if (typeof store.claimRun !== 'function') {
346
+ throw new NamzuError({
347
+ code: 'capability_unavailable',
348
+ message:
349
+ 'claimRun: the injected checkpoint store does not implement `claimRun`, so it cannot arbitrate between two workers taking the same run. Refusing rather than proceeding unclaimed — proceeding would let two workers restore one checkpoint, both execute its tools, and both write under one run id, which loses half the work and reports nothing. Supply a store that implements it, or run a single writer per run.',
350
+ details: { runId: scope.runId },
351
+ })
352
+ }
353
+ if (!Number.isFinite(options.ttlMs) || options.ttlMs <= 0) {
354
+ throw new NamzuError({
355
+ code: 'invalid_config',
356
+ message: `claimRun: ttlMs must be a positive number of milliseconds, got ${String(options.ttlMs)}. A lease that expires immediately is a lease every worker can take at once, which is the condition this call exists to prevent.`,
357
+ details: { runId: scope.runId, ttlMs: options.ttlMs },
358
+ })
359
+ }
360
+ return store.claimRun(scope, options)
361
+ }
362
+
363
+ /**
364
+ * Give a claim up early, refusing when the store cannot arbitrate.
365
+ *
366
+ * Refuses for the same reason as {@link claimRun}: a host that believes it is
367
+ * releasing a claim on a store that has none is a host that believes the
368
+ * whole mechanism is running.
369
+ */
370
+ export async function releaseRun(
371
+ store: CheckpointStore,
372
+ scope: CheckpointRunScope,
373
+ fence: ClaimFence,
374
+ ): Promise<void> {
375
+ if (typeof store.releaseRun !== 'function') {
376
+ throw new NamzuError({
377
+ code: 'capability_unavailable',
378
+ message:
379
+ 'releaseRun: the injected checkpoint store does not implement `releaseRun`. A release that silently does nothing would leave the run held until its lease expires while the caller believes it is back on the queue.',
380
+ details: { runId: scope.runId },
381
+ })
382
+ }
383
+ return store.releaseRun(scope, fence)
384
+ }
385
+
386
+ /**
387
+ * The refusal a store raises when a write presents a superseded fence.
388
+ *
389
+ * Shared so both shipped stores say the same thing, and so a host writing its
390
+ * own backend raises something a caller can branch on rather than a message
391
+ * string. This is the moment a stalled worker learns it lost the run — the
392
+ * only moment it CAN learn, since from the inside a pause and a partition
393
+ * both look like time not passing.
394
+ */
395
+ export function fencedOut(
396
+ scope: CheckpointRunScope,
397
+ presented: number,
398
+ current: number,
399
+ ): NamzuError {
400
+ return new NamzuError({
401
+ code: 'storage_error',
402
+ message: `writeCheckpoint: refusing a write for run ${scope.runId} fenced at ${presented} — the run is now claimed at ${current}. Another worker took this run over, so this process no longer holds it and its work is not the record. Stop the run rather than retrying: the claim is gone, not busy.`,
403
+ details: { runId: scope.runId, presentedFence: presented, currentFence: current },
404
+ retryable: false,
405
+ })
406
+ }
407
+
408
+ /**
409
+ * Whether a recorded claim still holds at `now`, and the summary a listing
410
+ * reports for it.
411
+ */
412
+ export function toClaimSummary(claim: RunClaim, now: number): ClaimSummary {
413
+ return {
414
+ holder: claim.holder,
415
+ fence: claim.fence,
416
+ expiresAt: claim.expiresAt,
417
+ // Judged here, once, against the store's clock. Left to the caller it
418
+ // would be judged against a different one, and a single page could
419
+ // then disagree with itself about which rows are available.
420
+ expired: now >= claim.expiresAt,
421
+ }
422
+ }
@@ -1,6 +1,6 @@
1
1
  import type { Run } from '../../types/run/entity.js'
2
- import type { RunEvent } from '../../types/run/events.js'
3
- import type { CompletedToolRecord, RunStore } from '../../types/run/store.js'
2
+ import type { PersistedRunEvent, RunEvent } from '../../types/run/events.js'
3
+ import type { CompletedToolRecord, ReadRunEventsOptions, RunStore } from '../../types/run/store.js'
4
4
 
5
5
  /**
6
6
  * Process-local {@link RunStore}: a run's evidence with no filesystem.
@@ -22,9 +22,21 @@ export class InMemoryRunStore implements RunStore {
22
22
  private meta: Run | null = null
23
23
  private messages: Run['messages'] = []
24
24
  private report: string | null = null
25
- private readonly events: RunEvent[] = []
25
+ private events: PersistedRunEvent[] = []
26
26
 
27
27
  async initRun(runId: string, parentRunId?: string): Promise<string | null> {
28
+ // Rebinding to a DIFFERENT run starts that run's evidence empty. The disk
29
+ // store gets this for free — a different id is a different directory —
30
+ // and this one has to say it, because one instance reused for a replay
31
+ // fork otherwise reports the origin run's events, its messages and its
32
+ // report as the new run's own. Evidence attributed to the wrong run is
33
+ // worse than none: it is wrong and it looks right.
34
+ if (this.runId !== null && this.runId !== runId) {
35
+ this.meta = null
36
+ this.messages = []
37
+ this.report = null
38
+ this.events = []
39
+ }
28
40
  this.runId = runId
29
41
  this.parentRunId = parentRunId
30
42
  // No location, and that is the honest answer rather than a defect.
@@ -68,7 +80,25 @@ export class InMemoryRunStore implements RunStore {
68
80
  // line — a parity test compares the two read-backs, and a timestamp
69
81
  // present in one medium and absent in the other would make identical
70
82
  // runs look different depending on where they were recorded.
71
- this.events.push({ ...event, timestamp: Date.now() } as unknown as RunEvent)
83
+ //
84
+ // An unsequenced event takes its position in the log, which is the same
85
+ // rule the disk reader applies to a line written before events were
86
+ // numbered. Nothing in the kernel appends unsequenced today; the rule is
87
+ // here so the two backends cannot answer differently if something does.
88
+ this.events.push({
89
+ ...event,
90
+ seq: event.seq ?? this.events.length + 1,
91
+ timestamp: Date.now(),
92
+ } as unknown as PersistedRunEvent)
93
+ }
94
+
95
+ async readEvents(options?: ReadRunEventsOptions): Promise<readonly PersistedRunEvent[]> {
96
+ this.requireInit()
97
+ const sinceSeq = options?.sinceSeq ?? 0
98
+ // Copied, not sliced by reference, for the same reason `writeRunMeta`
99
+ // clones: a caller holding the array must not be able to reach into the
100
+ // log through it.
101
+ return this.events.filter((event) => event.seq > sinceSeq).map((event) => ({ ...event }))
72
102
  }
73
103
 
74
104
  async writeReport(content: string): Promise<string | null> {
@@ -114,7 +144,7 @@ export class InMemoryRunStore implements RunStore {
114
144
  meta: Run | null
115
145
  messages: Run['messages']
116
146
  report: string | null
117
- events: readonly RunEvent[]
147
+ events: readonly PersistedRunEvent[]
118
148
  } {
119
149
  return { meta: this.meta, messages: this.messages, report: this.report, events: this.events }
120
150
  }
@@ -1,4 +1,21 @@
1
- export type DoctorStatus = 'pass' | 'fail' | 'inconclusive' | 'warn'
1
+ /**
2
+ * What a check concluded.
3
+ *
4
+ * `skipped` and `inconclusive` are the two ways a check produces no verdict,
5
+ * and they are separate because a caller acts on them differently.
6
+ *
7
+ * - `skipped` — the check LOOKED and there was nothing here to check: an
8
+ * optional package is not installed, a registry the check reads has no
9
+ * auto-discovery to read, nothing is configured yet. A permanent or
10
+ * by-design absence, and an ordinary state of a healthy machine.
11
+ * - `inconclusive` — the check DID NOT ANSWER: it timed out, it was aborted,
12
+ * the thing it reads threw. Nothing is known either way, and that is itself
13
+ * a gap in the report worth acting on.
14
+ *
15
+ * The word used to cover both, so `namzu doctor` could not tell "healthy" from
16
+ * "did not manage to look", and neither could anything reading its exit code.
17
+ */
18
+ export type DoctorStatus = 'pass' | 'fail' | 'inconclusive' | 'warn' | 'skipped'
2
19
 
3
20
  export type DoctorCategory =
4
21
  | 'sandbox'
@@ -47,7 +64,16 @@ export interface DoctorReport {
47
64
  readonly fail: number
48
65
  readonly inconclusive: number
49
66
  readonly warn: number
67
+ readonly skipped: number
68
+ /** Every other count sums to this. A reader may rely on that. */
50
69
  readonly total: number
51
70
  }
52
- readonly exit: 0 | 1 | 2 | 70
71
+ /**
72
+ * `69` is sysexits `EX_UNAVAILABLE`, whose own definition ends "a catchall
73
+ * when something you wanted to do doesn't work, but you don't know why".
74
+ * That is `inconclusive`. It is not `2`, which this report already spends on
75
+ * "nothing was registered", and not `70`, which says the CLI is broken and
76
+ * is worth a bug report.
77
+ */
78
+ readonly exit: 0 | 1 | 2 | 69 | 70
53
79
  }
@@ -175,6 +175,17 @@ export interface DurableRunEntry extends CheckpointRunScope {
175
175
  readonly latestCheckpointAt: number
176
176
  /** Absent when the run has never parked. */
177
177
  readonly park?: ParkSummary
178
+
179
+ /**
180
+ * Absent when no process has ever claimed the run.
181
+ *
182
+ * A SIBLING of {@link DurableRunEntry.park}, not a member of
183
+ * {@link ParkState} — see the note on that union. A park is a question put
184
+ * to a human; a claim is a lease held by a process. A run can have both,
185
+ * neither, or either, and the state a queue worker needs most is parked
186
+ * AND unclaimed, which one union cannot say.
187
+ */
188
+ readonly claim?: ClaimSummary
178
189
  }
179
190
 
180
191
  /**
@@ -208,6 +219,65 @@ export type DurableRunOrder =
208
219
  */
209
220
  | 'createdAt'
210
221
 
222
+ /**
223
+ * A monotonically increasing number identifying one holding of a run's claim.
224
+ *
225
+ * The load-bearing word is *fencing*. A mutex answers "may I proceed", and a
226
+ * holder that stalls past its lease — a long GC pause, a suspended container,
227
+ * a partitioned network — answers it "yes" and then writes, long after
228
+ * somebody else legitimately took over. A fence answers a different question
229
+ * at the moment of the WRITE: "is the holding I belong to still the current
230
+ * one". Every claim of a run mints a number strictly greater than the last,
231
+ * so a store can reject a write from a superseded holder without knowing
232
+ * anything about processes, clocks or liveness.
233
+ *
234
+ * Not a random token, deliberately: randomness proves identity and cannot
235
+ * establish *order*, and order is the entire mechanism.
236
+ */
237
+ export type ClaimFence = number
238
+
239
+ /** A holding of a run's claim, as issued to the process that took it. */
240
+ export interface RunClaim {
241
+ /** Opaque caller-supplied identity — a worker id, a pod name. Evidence, not authority. */
242
+ readonly holder: string
243
+ /** See {@link ClaimFence}. Present it on every durable write. */
244
+ readonly fence: ClaimFence
245
+ /**
246
+ * Absolute epoch ms after which the claim may be taken by somebody else.
247
+ *
248
+ * Absolute rather than a duration for the same reason a park's deadline
249
+ * is: it has to survive the process that set it. A duration plus an
250
+ * in-process timer cannot — the holder is the thing that dies.
251
+ */
252
+ readonly expiresAt: number
253
+ }
254
+
255
+ /** A run's claim as a listing reports it. */
256
+ export interface ClaimSummary {
257
+ readonly holder: string
258
+ readonly fence: ClaimFence
259
+ readonly expiresAt: number
260
+ /**
261
+ * Whether the claim had expired at the instant the listing was taken.
262
+ *
263
+ * A separate field rather than something the caller derives from
264
+ * `expiresAt`, because the caller would derive it against a DIFFERENT
265
+ * clock than the store used, and one page would then disagree with
266
+ * itself about which rows are available.
267
+ */
268
+ readonly expired: boolean
269
+ }
270
+
271
+ /** What a caller asks for when taking a run. */
272
+ export interface ClaimRunOptions {
273
+ /** Who is taking it. Recorded so an operator can see what holds a stuck run. */
274
+ readonly holder: string
275
+ /** How long the holding is good for, in ms. */
276
+ readonly ttlMs: number
277
+ /** Clock, for tests and so one operation judges every expiry against one instant. */
278
+ readonly now?: number
279
+ }
280
+
211
281
  /** Filters and paging for {@link CheckpointStore.listDurableRuns}. */
212
282
  export interface ListDurableRunsOptions {
213
283
  /**
@@ -223,6 +293,16 @@ export interface ListDurableRunsOptions {
223
293
  * to include it.
224
294
  */
225
295
  readonly park?: readonly ParkState[]
296
+ /**
297
+ * Keep only runs that are, or are not, currently held by a worker.
298
+ *
299
+ * `false` is the queue-reader's filter: give me the work nobody has. A
300
+ * claim that has expired counts as NOT held, because that is what expiry
301
+ * means and a reader that skipped expired claims would leave a dead
302
+ * worker's runs invisible forever — the exact failure the lease exists to
303
+ * prevent.
304
+ */
305
+ readonly claimed?: boolean
226
306
  /** Page size. Defaults to 100, clamped to at least 1. */
227
307
  readonly limit?: number
228
308
  /** Resume token from the previous page's {@link DurableRunPage.cursor}. */
@@ -279,8 +359,27 @@ export interface DurableRunPage {
279
359
  * proceed.
280
360
  */
281
361
  export interface CheckpointStore {
282
- /** Persist one checkpoint. Overwrites an existing checkpoint with the same id. */
283
- writeCheckpoint(scope: CheckpointRunScope, checkpoint: IterationCheckpoint): Promise<void>
362
+ /**
363
+ * Persist one checkpoint. Overwrites an existing checkpoint with the same id.
364
+ *
365
+ * @param fence the {@link ClaimFence} of the holding this write belongs
366
+ * to, when the run is claimed. A store that supports claims REFUSES a
367
+ * write whose fence is below the run's current one — that refusal is
368
+ * what makes a claim a lease rather than a suggestion, because a holder
369
+ * stalled past its expiry believes it still holds and is wrong only at
370
+ * the moment it writes.
371
+ *
372
+ * Omit it and the write is unfenced, which is exactly today's behaviour
373
+ * and correct for a single-writer deployment. A store MUST NOT start
374
+ * refusing unfenced writes because some other write carried a fence:
375
+ * that would make adding a claim to one worker break every worker that
376
+ * has not adopted it yet.
377
+ */
378
+ writeCheckpoint(
379
+ scope: CheckpointRunScope,
380
+ checkpoint: IterationCheckpoint,
381
+ fence?: ClaimFence,
382
+ ): Promise<void>
284
383
 
285
384
  /** Load a single checkpoint by id. Returns `null` when it does not exist. */
286
385
  readCheckpoint(
@@ -342,4 +441,92 @@ export interface CheckpointStore {
342
441
  scope: CheckpointListingScope,
343
442
  options?: ListDurableRunsOptions,
344
443
  ): Promise<DurableRunPage>
444
+
445
+ /**
446
+ * Take exclusive working possession of a run, or report that somebody
447
+ * else has it. OPTIONAL — see the optional-capability rule on this
448
+ * interface.
449
+ *
450
+ * Returns the holding on success and `null` when the run is currently
451
+ * held by somebody else. `null` is not an error: "another worker got
452
+ * there first" is the ordinary outcome of a queue with more than one
453
+ * reader, and a thrown exception would make the normal case look like a
454
+ * fault.
455
+ *
456
+ * ### What it is for
457
+ *
458
+ * Putting parked runs on a queue and letting more than one worker drain
459
+ * it. Without this, two workers restore the same checkpoint, both execute
460
+ * the run's tools, and both write checkpoints under one run id — each
461
+ * write minting a fresh checkpoint id, so two divergent chains land in
462
+ * one list and the pending lookup returns whichever wrote last. Half the
463
+ * work vanishes and nothing reports an error.
464
+ *
465
+ * ### The lease, and why it expires
466
+ *
467
+ * A claim is a LEASE, not a lock. A lock held by a process that dies is
468
+ * held forever, and the runs behind it are unreachable by anything except
469
+ * a human with a shell. The expiry is what makes a dead holder's work
470
+ * recoverable without one.
471
+ *
472
+ * The expiry is also why a fence exists. A holder does not know it has
473
+ * expired — a long pause, a suspended container and a partition all look
474
+ * from the inside like time not passing — so it wakes and writes as
475
+ * though it still holds. Liveness cannot be checked from here. What CAN
476
+ * be checked, at the write, is whether the holding that write belongs to
477
+ * is still the current one, and that is a comparison of two numbers.
478
+ *
479
+ * ### Reclaiming
480
+ *
481
+ * Calling this on a run whose claim has expired SUCCEEDS and mints a
482
+ * fence strictly greater than the expired holding's. The previous holder
483
+ * is not notified — it cannot be, that is the premise — it simply stops
484
+ * being able to write.
485
+ *
486
+ * Calling it again as the CURRENT holder also succeeds and extends the
487
+ * lease, minting a new fence. Renewal and reclamation are the same
488
+ * operation from the store's side, which is why there is no separate
489
+ * `renew`: two code paths that must agree about who holds a run is one
490
+ * more than can be kept correct.
491
+ *
492
+ * ### What a backend implementing this MUST guarantee
493
+ *
494
+ * These are the properties the fence comparison depends on. None of them
495
+ * is checkable from here, and every one of them was violated by the first
496
+ * built-in implementation, so they are written down rather than assumed:
497
+ *
498
+ * 1. **A fence exceeds every fence ever issued for the run** — including
499
+ * across a release, and across deletion of whatever recorded it. A
500
+ * counter that rewinds re-issues a number a stalled worker still
501
+ * believes it holds, and that worker's writes become legal again.
502
+ * 2. **Fences are unique.** The write check is `fence < current`, so two
503
+ * holders at one number are both admitted. Equality is permissive
504
+ * here, which makes a duplicate worse than a gap.
505
+ * 3. **The fence check is atomic with the write.** Reading the current
506
+ * fence and then writing is check-then-act, and the gap is a race. A
507
+ * database gets this free (`UPDATE … WHERE fence >= ?`); a filesystem
508
+ * does not, and the built-in disk store narrows rather than closes it.
509
+ * 4. **`holder` is unique per process.** It is evidence rather than
510
+ * authority, but it is the only thing distinguishing a RENEWAL from a
511
+ * theft — two workers sharing a holder string take a live, unexpired
512
+ * claim from each other instantly. Use something per-process, not a
513
+ * per-deployment name.
514
+ */
515
+ claimRun?(scope: CheckpointRunScope, options: ClaimRunOptions): Promise<RunClaim | null>
516
+
517
+ /**
518
+ * Give a claim up early. Idempotent: releasing a claim that already
519
+ * expired, was superseded, or never existed succeeds as a no-op.
520
+ *
521
+ * Presenting a stale fence releases NOTHING — a worker that stalled past
522
+ * its lease must not be able to hand away a run somebody else is now
523
+ * holding, and that is the same fencing comparison the write path makes.
524
+ *
525
+ * Optional to call, not optional to matter: a worker that finishes and
526
+ * releases returns the run to the queue immediately, where one that just
527
+ * exits leaves it stuck until the lease expires. That is a latency
528
+ * difference, never a correctness one, which is the property that lets a
529
+ * crashed worker be indistinguishable from a slow one.
530
+ */
531
+ releaseRun?(scope: CheckpointRunScope, fence: ClaimFence): Promise<void>
345
532
  }