@namzu/sdk 20.3.0 → 21.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/CHANGELOG.md +321 -0
  2. package/dist/bridge/sse/mapper.d.ts +19 -0
  3. package/dist/bridge/sse/mapper.d.ts.map +1 -1
  4. package/dist/bridge/sse/mapper.js +9 -1
  5. package/dist/bridge/sse/mapper.js.map +1 -1
  6. package/dist/contracts/schemas.d.ts +10 -10
  7. package/dist/manager/run/persistence.d.ts +26 -0
  8. package/dist/manager/run/persistence.d.ts.map +1 -1
  9. package/dist/manager/run/persistence.js +40 -0
  10. package/dist/manager/run/persistence.js.map +1 -1
  11. package/dist/public-runtime.d.ts +5 -0
  12. package/dist/public-runtime.d.ts.map +1 -1
  13. package/dist/public-runtime.js +21 -0
  14. package/dist/public-runtime.js.map +1 -1
  15. package/dist/run/drain.d.ts +194 -0
  16. package/dist/run/drain.d.ts.map +1 -0
  17. package/dist/run/drain.js +228 -0
  18. package/dist/run/drain.js.map +1 -0
  19. package/dist/run/index.d.ts +2 -0
  20. package/dist/run/index.d.ts.map +1 -1
  21. package/dist/run/index.js +1 -0
  22. package/dist/run/index.js.map +1 -1
  23. package/dist/runtime/query/checkpoint.d.ts +27 -1
  24. package/dist/runtime/query/checkpoint.d.ts.map +1 -1
  25. package/dist/runtime/query/checkpoint.js +34 -4
  26. package/dist/runtime/query/checkpoint.js.map +1 -1
  27. package/dist/runtime/query/events.d.ts +12 -0
  28. package/dist/runtime/query/events.d.ts.map +1 -1
  29. package/dist/runtime/query/events.js +61 -3
  30. package/dist/runtime/query/events.js.map +1 -1
  31. package/dist/runtime/query/index.d.ts +57 -1
  32. package/dist/runtime/query/index.d.ts.map +1 -1
  33. package/dist/runtime/query/index.js +36 -0
  34. package/dist/runtime/query/index.js.map +1 -1
  35. package/dist/runtime/query/resume-run.d.ts +34 -1
  36. package/dist/runtime/query/resume-run.d.ts.map +1 -1
  37. package/dist/runtime/query/resume-run.js +16 -3
  38. package/dist/runtime/query/resume-run.js.map +1 -1
  39. package/dist/store/index.d.ts +2 -2
  40. package/dist/store/index.d.ts.map +1 -1
  41. package/dist/store/index.js +2 -2
  42. package/dist/store/index.js.map +1 -1
  43. package/dist/store/run/checkpoint-disk.d.ts +18 -2
  44. package/dist/store/run/checkpoint-disk.d.ts.map +1 -1
  45. package/dist/store/run/checkpoint-disk.js +50 -5
  46. package/dist/store/run/checkpoint-disk.js.map +1 -1
  47. package/dist/store/run/checkpoint-memory.d.ts +22 -2
  48. package/dist/store/run/checkpoint-memory.d.ts.map +1 -1
  49. package/dist/store/run/checkpoint-memory.js +99 -4
  50. package/dist/store/run/checkpoint-memory.js.map +1 -1
  51. package/dist/store/run/claim-disk.d.ts +130 -0
  52. package/dist/store/run/claim-disk.d.ts.map +1 -0
  53. package/dist/store/run/claim-disk.js +550 -0
  54. package/dist/store/run/claim-disk.js.map +1 -0
  55. package/dist/store/run/conformance.d.ts +159 -0
  56. package/dist/store/run/conformance.d.ts.map +1 -0
  57. package/dist/store/run/conformance.js +451 -0
  58. package/dist/store/run/conformance.js.map +1 -0
  59. package/dist/store/run/disk.d.ts +30 -2
  60. package/dist/store/run/disk.d.ts.map +1 -1
  61. package/dist/store/run/disk.js +101 -0
  62. package/dist/store/run/disk.js.map +1 -1
  63. package/dist/store/run/listing.d.ts +44 -1
  64. package/dist/store/run/listing.d.ts.map +1 -1
  65. package/dist/store/run/listing.js +92 -1
  66. package/dist/store/run/listing.js.map +1 -1
  67. package/dist/store/run/memory.d.ts +5 -4
  68. package/dist/store/run/memory.d.ts.map +1 -1
  69. package/dist/store/run/memory.js +30 -1
  70. package/dist/store/run/memory.js.map +1 -1
  71. package/dist/types/doctor/check.d.ts +28 -2
  72. package/dist/types/doctor/check.d.ts.map +1 -1
  73. package/dist/types/run/checkpoint-store.d.ts +178 -2
  74. package/dist/types/run/checkpoint-store.d.ts.map +1 -1
  75. package/dist/types/run/event-cursor.d.ts +90 -0
  76. package/dist/types/run/event-cursor.d.ts.map +1 -0
  77. package/dist/types/run/event-cursor.js +47 -0
  78. package/dist/types/run/event-cursor.js.map +1 -0
  79. package/dist/types/run/events.d.ts +71 -1
  80. package/dist/types/run/events.d.ts.map +1 -1
  81. package/dist/types/run/events.js.map +1 -1
  82. package/dist/types/run/index.d.ts +1 -0
  83. package/dist/types/run/index.d.ts.map +1 -1
  84. package/dist/types/run/index.js +1 -0
  85. package/dist/types/run/index.js.map +1 -1
  86. package/dist/types/run/store.d.ts +41 -1
  87. package/dist/types/run/store.d.ts.map +1 -1
  88. package/package.json +6 -1
  89. package/src/bridge/sse/mapper.ts +28 -1
  90. package/src/manager/run/persistence.ts +43 -0
  91. package/src/public-runtime.ts +27 -0
  92. package/src/run/drain.ts +393 -0
  93. package/src/run/index.ts +3 -0
  94. package/src/runtime/query/checkpoint.ts +42 -5
  95. package/src/runtime/query/events.ts +68 -4
  96. package/src/runtime/query/index.ts +113 -1
  97. package/src/runtime/query/resume-run.ts +77 -12
  98. package/src/store/index.ts +5 -1
  99. package/src/store/run/checkpoint-disk.ts +70 -5
  100. package/src/store/run/checkpoint-memory.ts +118 -3
  101. package/src/store/run/claim-disk.ts +593 -0
  102. package/src/store/run/conformance.ts +705 -0
  103. package/src/store/run/disk.ts +106 -2
  104. package/src/store/run/listing.ts +116 -1
  105. package/src/store/run/memory.ts +35 -5
  106. package/src/types/doctor/check.ts +28 -2
  107. package/src/types/run/checkpoint-store.ts +189 -2
  108. package/src/types/run/event-cursor.ts +118 -0
  109. package/src/types/run/events.ts +80 -3
  110. package/src/types/run/index.ts +1 -0
  111. package/src/types/run/store.ts +43 -1
@@ -4,6 +4,7 @@ import { buildProbeContext } from '../../probe/context.js'
4
4
  import { type ProbeRegistry, probe as defaultProbeRegistry } from '../../probe/registry.js'
5
5
  import type { ActivityEvent, ActivityStore } from '../../store/activity/memory.js'
6
6
  import type { RunId } from '../../types/ids/index.js'
7
+ import type { ClaimFence } from '../../types/run/checkpoint-store.js'
7
8
  import { isEphemeralEvent } from '../../types/run/events.js'
8
9
  import type { RunEvent } from '../../types/run/index.js'
9
10
  import type { TaskEvent, TaskStore } from '../../types/task/index.js'
@@ -37,6 +38,22 @@ export class EventTranslator {
37
38
  this.probes = probeRegistry
38
39
  }
39
40
 
41
+ /**
42
+ * The claim this run is being written under, when it holds one.
43
+ *
44
+ * Stamped on every durable event as its `generation`, so a consumer whose
45
+ * cursor predates a takeover is told its sequence space changed instead of
46
+ * being handed a splice from a different writer's log.
47
+ */
48
+ private generation: ClaimFence | undefined
49
+
50
+ /** Serializes sequence assignment against the append. See {@link emitEvent}. */
51
+ private appendChain: Promise<void> = Promise.resolve()
52
+
53
+ setGeneration(fence: ClaimFence | undefined): void {
54
+ this.generation = fence
55
+ }
56
+
40
57
  readonly emitEvent: EmitEvent = async (event: RunEvent): Promise<void> => {
41
58
  this.probes.dispatch(event, buildProbeContext({ runId: event.runId }))
42
59
 
@@ -64,15 +81,62 @@ export class EventTranslator {
64
81
  // briefly than to drop a state transition.
65
82
  }
66
83
 
67
- this.pendingEvents.push(event)
68
-
69
84
  // D1 middle path: ephemeral events never enter `transcript.jsonl`.
70
85
  // They live only on the in-memory bus for live UI rendering.
71
86
  // Replay (`runtime/query/replay/prepare.ts`) reads checkpoints
72
87
  // not transcripts, so this preserves replay fidelity while
73
88
  // eliminating the durable bloat review flagged.
74
- if (!isEphemeralEvent(event)) {
75
- await this.runMgr.getRunStore().appendEvent(event)
89
+ if (isEphemeralEvent(event)) {
90
+ // No number, and that is the honest statement: nothing will
91
+ // persist this, so a consumer must never advance a cursor to it.
92
+ this.pendingEvents.push(event)
93
+ return
94
+ }
95
+
96
+ // One appender at a time, and this is not a precaution — it is the fix
97
+ // for a measured defect. Taking the number, awaiting the write and then
98
+ // committing is a read-modify-write, and emits genuinely interleave:
99
+ // the task store, the plan manager and a batch of parallel tools all
100
+ // emit into this one funnel. Measured on a two-tool run, three events
101
+ // took the number 15 and two took 12. A duplicated sequence is worse
102
+ // than a missing one — a consumer asking for everything above 15 is
103
+ // handed part of the run it already had, spliced in as if it were new.
104
+ const previous = this.appendChain
105
+ let release!: () => void
106
+ this.appendChain = new Promise<void>((resolve) => {
107
+ release = resolve
108
+ })
109
+
110
+ try {
111
+ await previous
112
+
113
+ // The number is a claim that the event is IN the log, so it is taken
114
+ // against the append and not before it. The candidate goes to the
115
+ // store first; only a write that landed advances the counter and
116
+ // reaches the live stream carrying it.
117
+ //
118
+ // The failure path still delivers the event — unstamped. A store
119
+ // that cannot record a `run_failed` must not also swallow it, and an
120
+ // unstamped event says exactly what is true of it: it happened, and
121
+ // it is not recoverable.
122
+ const seq = this.runMgr.nextEventSeq()
123
+ const stamped = {
124
+ ...event,
125
+ seq,
126
+ ...(this.generation !== undefined ? { generation: this.generation } : {}),
127
+ } as RunEvent
128
+
129
+ try {
130
+ await this.runMgr.getRunStore().appendEvent(stamped)
131
+ } catch (err) {
132
+ this.pendingEvents.push(event)
133
+ throw err
134
+ }
135
+
136
+ this.runMgr.commitEventSeq(seq)
137
+ this.pendingEvents.push(stamped)
138
+ } finally {
139
+ release()
76
140
  }
77
141
  };
78
142
 
@@ -15,6 +15,7 @@ import { restoreWorkingState, snapshotWorkingState } from '../../compaction/wire
15
15
  import type { CompactionConfig } from '../../config/runtime.js'
16
16
  import { TOOL_OUTPUT_DIR_NAME } from '../../constants/tools/index.js'
17
17
  import { EmergencySaveManager } from '../../manager/run/emergency.js'
18
+ import type { RunPersistence } from '../../manager/run/persistence.js'
18
19
  import { resolveProviderCapabilities } from '../../provider/capabilities.js'
19
20
  import {
20
21
  type ProviderChainMember,
@@ -58,7 +59,9 @@ import type { AgentPersona } from '../../types/persona/index.js'
58
59
  import type { LLMProvider } from '../../types/provider/index.js'
59
60
  import type { TaskRouterConfig } from '../../types/router/index.js'
60
61
  import type { ReviewAnswer } from '../../types/run/answer-review.js'
61
- import type { CheckpointStore } from '../../types/run/checkpoint-store.js'
62
+ import type { CheckpointStore, ClaimFence } from '../../types/run/checkpoint-store.js'
63
+ import type { RunEventCursor, RunEventReplay } from '../../types/run/event-cursor.js'
64
+ import { resolveRunEventReplay } from '../../types/run/event-cursor.js'
62
65
  import type {
63
66
  AgentRunConfig,
64
67
  PrepareStepChain,
@@ -413,6 +416,27 @@ export interface QueryParams {
413
416
  */
414
417
  checkpointStore?: CheckpointStore
415
418
 
419
+ /**
420
+ * The fence of the claim this worker holds on the run, from `claimRun`.
421
+ *
422
+ * Presented on every checkpoint the run writes, so a worker that stalled
423
+ * past its lease is refused rather than writing into a run somebody else
424
+ * has taken over. Omit it for single-writer deployments, which is what
425
+ * every run did before claims existed.
426
+ *
427
+ * This hop did not exist for a release. The claim, the fence and the
428
+ * store-side refusal were all built and tested, and no path between a run
429
+ * and its store carried the number — so every checkpoint a RUN wrote went
430
+ * out unfenced while the tests, which called the store directly, all
431
+ * passed. A capability complete except for the wire between its halves
432
+ * reads exactly like a working one.
433
+ *
434
+ * It fences checkpoints and nothing else. {@link QueryParams.runStore}
435
+ * takes no fence, so two workers that both took one run still overwrite
436
+ * each other's run record, transcript and report — see the changeset.
437
+ */
438
+ claimFence?: ClaimFence
439
+
416
440
  /**
417
441
  * Where this run records its own evidence — the run record, its messages,
418
442
  * its transcript and its report. Defaults to the disk layout under the
@@ -424,6 +448,43 @@ export interface QueryParams {
424
448
  */
425
449
  runStore?: RunStore
426
450
 
451
+ /**
452
+ * Where a reconnecting consumer left off, so this run's stream can start by
453
+ * handing back what it missed.
454
+ *
455
+ * The case this serves is the one that exists without a network hop: the
456
+ * process holding the run died, and the consumer watching it is coming back
457
+ * to a run that has to be resumed. Pair it with `resumeFromCheckpoint` — or
458
+ * reach it through {@link import('./resume-run.js').resumeRun}, which is the
459
+ * surface that does both — and the missed durable events are yielded, in
460
+ * order, before the resumed run emits anything of its own.
461
+ *
462
+ * On a run with no log to catch up on the cursor is answered honestly rather
463
+ * than ignored: a `sinceSeq` above what exists is `cursor_ahead`, not
464
+ * silence.
465
+ *
466
+ * What comes back is message-granular. Streaming deltas are never persisted
467
+ * — see {@link import('../../types/run/store.js').RunStore.appendEvent} —
468
+ * so a late subscriber recovers the assistant text, the tool results and the
469
+ * lifecycle, not the keystroke cadence that produced them.
470
+ */
471
+ eventCursor?: RunEventCursor
472
+
473
+ /**
474
+ * What became of {@link QueryParams.eventCursor}.
475
+ *
476
+ * A callback rather than an event on the stream, because the answer is about
477
+ * the SUBSCRIPTION and not about the run — and rather than a throw, because
478
+ * a stale cursor is a client's problem and must not be able to stop a run
479
+ * from continuing. A host that receives `unavailable` re-derives from the
480
+ * transcript; one that receives nothing at all would splice a hole into its
481
+ * state and never know.
482
+ *
483
+ * Called once, before the run's first event, and only when a cursor was
484
+ * supplied.
485
+ */
486
+ onEventReplay?: (replay: RunEventReplay) => void
487
+
427
488
  runId?: RunId
428
489
 
429
490
  parentRunId?: RunId
@@ -1013,6 +1074,14 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
1013
1074
  // And every park it records carries an absolute deadline, so an
1014
1075
  // unanswered approval cannot outlive the worker that asked for it.
1015
1076
  checkpointMgr.setParkTtl(params.runConfig.hitlParkTtlMs)
1077
+ // The claim this worker holds, if it took one. Without this hop the
1078
+ // fence exists, the refusal exists, and no checkpoint a RUN writes ever
1079
+ // carries a number — so a stalled worker is refused nowhere.
1080
+ checkpointMgr.setClaimFence(params.claimFence)
1081
+ // And every EVENT it records carries the same fence as its generation,
1082
+ // so a consumer whose cursor was minted under an older holding is told
1083
+ // the sequence space changed rather than handed a splice from it.
1084
+ eventTranslator.setGeneration(params.claimFence)
1016
1085
 
1017
1086
  // A question raised from inside a tool becomes a real checkpoint
1018
1087
  // here. It used to park under a synthetic id nothing ever wrote, so
@@ -1090,6 +1159,20 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
1090
1159
  try {
1091
1160
  await ctx.runMgr.init()
1092
1161
 
1162
+ // A consumer coming back gets what it missed BEFORE the run says
1163
+ // anything new, which is the only order that lets it fold one
1164
+ // stream into one state. It has to follow `init()` — that is what
1165
+ // binds the store and reads the log's head — and precede every
1166
+ // emit below.
1167
+ if (params.eventCursor) {
1168
+ yield* catchUpFromCursor(
1169
+ ctx.runMgr,
1170
+ params.eventCursor,
1171
+ params.onEventReplay,
1172
+ params.claimFence,
1173
+ )
1174
+ }
1175
+
1093
1176
  // Handed over here, and the position is load-bearing in BOTH
1094
1177
  // directions. It has to follow `wirePlanManager`, or a host that
1095
1178
  // builds its plan in this callback — which is what the callback is
@@ -1569,6 +1652,35 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
1569
1652
  })()
1570
1653
  }
1571
1654
 
1655
+ /**
1656
+ * Hand a returning consumer what it missed, or tell it why it cannot have it.
1657
+ *
1658
+ * Yields NOTHING on a refusal. A partial catch-up is the failure this exists to
1659
+ * prevent: a consumer that receives some of the gap folds it into its state and
1660
+ * cannot tell the state is wrong, where one that receives an explicit
1661
+ * `unavailable` re-derives from the transcript and is right. The run continues
1662
+ * either way — a stale cursor belongs to the client, and must not be able to
1663
+ * stop the work.
1664
+ */
1665
+ async function* catchUpFromCursor(
1666
+ runMgr: RunPersistence,
1667
+ cursor: RunEventCursor,
1668
+ onEventReplay: ((replay: RunEventReplay) => void) | undefined,
1669
+ generation: ClaimFence | undefined,
1670
+ ): AsyncGenerator<RunEvent, void> {
1671
+ const missed = await runMgr.getRunStore().readEvents({ sinceSeq: cursor.sinceSeq })
1672
+ const replay = resolveRunEventReplay(
1673
+ cursor,
1674
+ { lastSeq: runMgr.lastEventSeq, ...(generation !== undefined ? { generation } : {}) },
1675
+ missed,
1676
+ )
1677
+
1678
+ onEventReplay?.(replay)
1679
+
1680
+ if (replay.status !== 'replayed') return
1681
+ for (const event of replay.events) yield event
1682
+ }
1683
+
1572
1684
  export async function drainQuery(
1573
1685
  params: Omit<QueryParams, 'resumeHandler'> & { resumeHandler?: ResumeHandler },
1574
1686
  listener?: RunEventListener,
@@ -1,6 +1,8 @@
1
1
  import type { PendingDecision } from '../../types/hitl/index.js'
2
- import type { CheckpointStore } from '../../types/run/checkpoint-store.js'
2
+ import type { CheckpointStore, ClaimFence } from '../../types/run/checkpoint-store.js'
3
3
  import type { Run } from '../../types/run/entity.js'
4
+ import type { RunEventReplay } from '../../types/run/event-cursor.js'
5
+ import type { RunEventListener } from '../../types/run/events.js'
4
6
  import type { RunState } from '../../types/run/state.js'
5
7
  import { type QueryParams, drainQuery } from './index.js'
6
8
  import { type RunStateScope, loadRunState } from './run-state.js'
@@ -23,7 +25,21 @@ export type ResumeOutcome =
23
25
  readonly pending: PendingDecision
24
26
  readonly state: RunState
25
27
  }
26
- | { readonly resumed: true; readonly run: Run; readonly state: RunState }
28
+ | {
29
+ readonly resumed: true
30
+ readonly run: Run
31
+ readonly state: RunState
32
+ /**
33
+ * What became of {@link ResumeRunParams.eventCursor}. Absent when no
34
+ * cursor was supplied.
35
+ *
36
+ * Reported rather than thrown: a stale cursor is the client's
37
+ * problem and the run still had to be resumed. A caller that sees
38
+ * `unavailable` re-derives its view from the run's transcript
39
+ * instead of folding a hole into it.
40
+ */
41
+ readonly replay?: RunEventReplay
42
+ }
27
43
 
28
44
  /**
29
45
  * The half of a run that cannot be serialized, plus where to look.
@@ -38,11 +54,34 @@ export interface ResumeRunParams
38
54
  readonly scope: RunStateScope
39
55
  /** Required to find the checkpoint; also threaded into the resumed run. */
40
56
  readonly checkpointStore: CheckpointStore
57
+
58
+ /**
59
+ * The fence of the claim this worker took on the run before resuming.
60
+ *
61
+ * A resume is the one moment two workers are most likely to collide — it
62
+ * is what a queue reader does with a parked run — so this is the call that
63
+ * most needs to carry one.
64
+ */
65
+ readonly claimFence?: ClaimFence
41
66
  /**
42
67
  * Resume a specific checkpoint instead of the one the store would pick.
43
68
  * Absent means the parked checkpoint if there is one, else the newest.
44
69
  */
45
70
  readonly checkpointId?: RunState['checkpointId']
71
+
72
+ /**
73
+ * Where to send the resumed run's events.
74
+ *
75
+ * This call had none, and the consequence was total: it drains the run to
76
+ * completion and every event the run emits — every tool call, every park,
77
+ * every token update — was discarded, because `drainQuery` forwards to a
78
+ * listener and none was ever passed. So the one API for continuing a run
79
+ * another process started could not show anybody what the run was doing.
80
+ *
81
+ * It is also what makes {@link ResumeRunParams.eventCursor} mean anything:
82
+ * a catch-up delivered into a stream nobody receives is not a catch-up.
83
+ */
84
+ readonly listener?: RunEventListener
46
85
  }
47
86
 
48
87
  /**
@@ -67,7 +106,16 @@ export interface ResumeRunParams
67
106
  * resumed past without the answer it is waiting for.
68
107
  */
69
108
  export async function resumeRun(params: ResumeRunParams): Promise<ResumeOutcome> {
70
- const { scope, checkpointStore, checkpointId, pendingDecision, ...rest } = params
109
+ const {
110
+ scope,
111
+ checkpointStore,
112
+ checkpointId,
113
+ pendingDecision,
114
+ claimFence,
115
+ listener,
116
+ onEventReplay,
117
+ ...rest
118
+ } = params
71
119
 
72
120
  const state = await loadRunState(checkpointStore, scope, checkpointId)
73
121
  if (!state?.checkpointId) return { resumed: false, reason: 'no-checkpoint' }
@@ -80,14 +128,31 @@ export async function resumeRun(params: ResumeRunParams): Promise<ResumeOutcome>
80
128
  return { resumed: false, reason: 'awaiting-decision', pending: outstanding, state }
81
129
  }
82
130
 
83
- const run = await drainQuery({
84
- ...rest,
85
- messages: [],
86
- runId: state.runId,
87
- resumeFromCheckpoint: state.checkpointId,
88
- checkpointStore,
89
- ...(pendingDecision ? { pendingDecision } : {}),
90
- } as QueryParams)
131
+ let replay: RunEventReplay | undefined
132
+
133
+ const run = await drainQuery(
134
+ {
135
+ ...rest,
136
+ messages: [],
137
+ runId: state.runId,
138
+ // Forwarded, and it is not cosmetic: the run store nests a sub-run's
139
+ // evidence under `<parent>/children/<run>`, so resuming a sub-run
140
+ // without this binds `<base>/<run>` instead — a second, empty
141
+ // transcript under a run id that already has one, a sequence that
142
+ // restarts at 1, and a catch-up that reports a live run as having
143
+ // produced nothing.
144
+ ...(state.parentRunId !== undefined ? { parentRunId: state.parentRunId } : {}),
145
+ resumeFromCheckpoint: state.checkpointId,
146
+ checkpointStore,
147
+ ...(claimFence !== undefined ? { claimFence } : {}),
148
+ ...(pendingDecision ? { pendingDecision } : {}),
149
+ onEventReplay: (verdict: RunEventReplay) => {
150
+ replay = verdict
151
+ onEventReplay?.(verdict)
152
+ },
153
+ } as QueryParams,
154
+ listener,
155
+ )
91
156
 
92
- return { resumed: true, run, state }
157
+ return { resumed: true, run, state, ...(replay !== undefined ? { replay } : {}) }
93
158
  }
@@ -1,7 +1,7 @@
1
1
  export { InMemoryStore } from './InMemoryStore.js'
2
2
  export type { Identifiable, Timestamped } from './InMemoryStore.js'
3
3
 
4
- export { RunDiskStore } from './run/disk.js'
4
+ export { RunDiskStore, readRunEventsIn } from './run/disk.js'
5
5
  export { InMemoryRunStore } from './run/memory.js'
6
6
  export { DiskCheckpointStore } from './run/checkpoint-disk.js'
7
7
  export type { DiskCheckpointStoreAttribution } from './run/checkpoint-disk.js'
@@ -18,8 +18,12 @@ export { InMemoryCheckpointStore } from './run/checkpoint-memory.js'
18
18
  // host has no use for is surface to keep correct forever for nobody.
19
19
  export {
20
20
  assertContiguousListingScope,
21
+ claimRun,
22
+ fencedOut,
21
23
  listDurableRuns,
22
24
  paginateDurableRuns,
25
+ releaseRun,
26
+ toClaimSummary,
23
27
  toDurableRunEntry,
24
28
  } from './run/listing.js'
25
29
 
@@ -7,14 +7,24 @@ import type {
7
7
  CheckpointListingScope,
8
8
  CheckpointRunScope,
9
9
  CheckpointStore,
10
+ ClaimFence,
11
+ ClaimRunOptions,
10
12
  DurableRunEntry,
11
13
  DurableRunPage,
12
14
  ListDurableRunsOptions,
15
+ RunClaim,
13
16
  } from '../../types/run/checkpoint-store.js'
14
17
  import type { RunStoreConfig } from '../../types/run/index.js'
15
18
  import type { ProjectId } from '../../types/session/ids.js'
19
+ import { acquireClaim, currentFence, readClaim, releaseClaim } from './claim-disk.js'
16
20
  import { RunDiskStore, readCheckpointsIn } from './disk.js'
17
- import { assertContiguousListingScope, paginateDurableRuns, toDurableRunEntry } from './listing.js'
21
+ import {
22
+ assertContiguousListingScope,
23
+ fencedOut,
24
+ paginateDurableRuns,
25
+ toClaimSummary,
26
+ toDurableRunEntry,
27
+ } from './listing.js'
18
28
 
19
29
  /**
20
30
  * The attribution a disk store's own layout does not record.
@@ -84,11 +94,47 @@ export class DiskCheckpointStore implements CheckpointStore {
84
94
  return promise
85
95
  }
86
96
 
87
- async writeCheckpoint(scope: CheckpointRunScope, checkpoint: IterationCheckpoint): Promise<void> {
97
+ async writeCheckpoint(
98
+ scope: CheckpointRunScope,
99
+ checkpoint: IterationCheckpoint,
100
+ fence?: ClaimFence,
101
+ ): Promise<void> {
88
102
  const store = await this.bind(scope)
103
+ if (fence !== undefined) {
104
+ // Read at the moment of the write, not at the start of the run.
105
+ // A holder that stalled past its lease believes it still holds,
106
+ // and this is the only point at which it can be told otherwise.
107
+ // Names only, no parsing. The fence is the file name, so a corrupt
108
+ // body cannot make this check skip itself — which the previous shape
109
+ // did, at the one site whose entire job is refusing.
110
+ const current = await currentFence(this.runDir(scope))
111
+ if (fence < current) throw fencedOut(scope, fence, current)
112
+ }
89
113
  await store.writeCheckpoint(checkpoint)
90
114
  }
91
115
 
116
+ async claimRun(scope: CheckpointRunScope, options: ClaimRunOptions): Promise<RunClaim | null> {
117
+ return acquireClaim(this.runDir(scope), options)
118
+ }
119
+
120
+ async releaseRun(scope: CheckpointRunScope, fence: ClaimFence): Promise<void> {
121
+ await releaseClaim(this.runDir(scope), fence)
122
+ }
123
+
124
+ /**
125
+ * The run's directory, resolved the same way `RunDiskStore.initRun` does.
126
+ *
127
+ * Duplicated rather than shared because the claim path must be derivable
128
+ * WITHOUT binding a store — binding creates the directory, and a claim
129
+ * read is a read. Kept beside the layout comment on `listDurableRuns` so
130
+ * the two stay together if the layout ever moves.
131
+ */
132
+ private runDir(scope: CheckpointRunScope): string {
133
+ return scope.parentRunId
134
+ ? join(this.config.baseDir, scope.parentRunId, 'children', scope.runId)
135
+ : join(this.config.baseDir, scope.runId)
136
+ }
137
+
92
138
  async readCheckpoint(
93
139
  scope: CheckpointRunScope,
94
140
  checkpointId: CheckpointId,
@@ -163,21 +209,40 @@ export class DiskCheckpointStore implements CheckpointStore {
163
209
  const runDir = join(this.config.baseDir, runId)
164
210
 
165
211
  const own = toDurableRunEntry({ ...attribution, runId }, await readCheckpointsIn(runDir), now)
166
- if (own) entries.push(own)
212
+ if (own) entries.push(await this.withClaim(own, runDir, now))
167
213
 
168
214
  for (const childId of await this.readRunDirs(join(runDir, 'children'))) {
215
+ const childDir = join(runDir, 'children', childId)
169
216
  const child = toDurableRunEntry(
170
217
  { ...attribution, runId: childId, parentRunId: runId },
171
- await readCheckpointsIn(join(runDir, 'children', childId)),
218
+ await readCheckpointsIn(childDir),
172
219
  now,
173
220
  )
174
- if (child) entries.push(child)
221
+ if (child) entries.push(await this.withClaim(child, childDir, now))
175
222
  }
176
223
  }
177
224
 
178
225
  return paginateDurableRuns(entries, options)
179
226
  }
180
227
 
228
+ /**
229
+ * Attach the run's claim to its listing row, judged against the page's
230
+ * own clock so one page cannot disagree with itself about availability.
231
+ */
232
+ private async withClaim(
233
+ entry: DurableRunEntry,
234
+ runDir: string,
235
+ now: number,
236
+ ): Promise<DurableRunEntry> {
237
+ const claim = await readClaim(runDir)
238
+ // A holding whose body could not be read still appears on the row,
239
+ // carrying an expiry of 0 so it reads as available. Dropping the field
240
+ // entirely — which is what happened before — put the run under
241
+ // `claimed: false` by looking unclaimed rather than by being
242
+ // reclaimable, so a queue reader was told a wedged run was free work.
243
+ return claim ? { ...entry, claim: toClaimSummary(claim, now) } : entry
244
+ }
245
+
181
246
  /** Directory names under `dir`, or none when `dir` does not exist. */
182
247
  private async readRunDirs(dir: string): Promise<RunId[]> {
183
248
  try {
@@ -3,11 +3,20 @@ import type {
3
3
  CheckpointListingScope,
4
4
  CheckpointRunScope,
5
5
  CheckpointStore,
6
+ ClaimFence,
7
+ ClaimRunOptions,
6
8
  DurableRunEntry,
7
9
  DurableRunPage,
8
10
  ListDurableRunsOptions,
11
+ RunClaim,
9
12
  } from '../../types/run/checkpoint-store.js'
10
- import { assertContiguousListingScope, paginateDurableRuns, toDurableRunEntry } from './listing.js'
13
+ import {
14
+ assertContiguousListingScope,
15
+ fencedOut,
16
+ paginateDurableRuns,
17
+ toClaimSummary,
18
+ toDurableRunEntry,
19
+ } from './listing.js'
11
20
 
12
21
  /**
13
22
  * Process-local {@link CheckpointStore}, keyed by the full five-layer scope.
@@ -35,7 +44,111 @@ export class InMemoryCheckpointStore implements CheckpointStore {
35
44
  return [scope.tenantId, scope.projectId, scope.sessionId, scope.runId].join('/')
36
45
  }
37
46
 
38
- async writeCheckpoint(scope: CheckpointRunScope, checkpoint: IterationCheckpoint): Promise<void> {
47
+ /** `tenant/project/session/run` → the run's current holding, if any. */
48
+ private readonly claims = new Map<string, RunClaim>()
49
+
50
+ /**
51
+ * The highest fence ever issued per run, kept separately from the claim.
52
+ *
53
+ * The claim is removed on release; this is not. That separation is the
54
+ * whole point: the first version deleted the claim and then computed the
55
+ * next fence from it, so releasing rewound the counter to 1 and a worker
56
+ * stalled at fence 1 could write beside a new holder also at fence 1 —
57
+ * and the documented `finally { releaseRun() }` did it on every pass.
58
+ *
59
+ * The disk store gets this property from file names that persist. In
60
+ * memory the equivalent is a high-water mark nothing clears, and the two
61
+ * must agree, because this class is what a host reads when writing a
62
+ * backend of its own.
63
+ */
64
+ private readonly highWater = new Map<string, ClaimFence>()
65
+
66
+ async claimRun(scope: CheckpointRunScope, options: ClaimRunOptions): Promise<RunClaim | null> {
67
+ const key = this.key(scope)
68
+ const now = options.now ?? Date.now()
69
+ const held = this.claims.get(key)
70
+
71
+ // Held by somebody else and still live. Not an error: two readers on
72
+ // one queue is the ordinary case.
73
+ if (held && now < held.expiresAt && held.holder !== options.holder) return null
74
+
75
+ // A reclaim of an expired holding and a renewal by the current holder
76
+ // are the same write. The fence advances either way, so a previous
77
+ // holder that wakes up is fenced out in both cases — a renewal that
78
+ // kept the fence would leave a stalled twin able to write.
79
+ // Counted from the high-water mark, never from the live claim. A
80
+ // released run has no claim, and computing from that absence is what
81
+ // rewound the counter to 1 on every release.
82
+ const fence = (this.highWater.get(key) ?? 0) + 1
83
+ const claim: RunClaim = { holder: options.holder, fence, expiresAt: now + options.ttlMs }
84
+ this.highWater.set(key, fence)
85
+ this.claims.set(key, claim)
86
+ return claim
87
+ }
88
+
89
+ async releaseRun(scope: CheckpointRunScope, fence: ClaimFence): Promise<void> {
90
+ const key = this.key(scope)
91
+ const held = this.claims.get(key)
92
+ // A stale fence releases nothing. A worker that stalled past its lease
93
+ // must not be able to hand away a run somebody else now holds.
94
+ //
95
+ // The high-water mark deliberately survives this. Dropping the claim
96
+ // returns the run to the queue; forgetting the number it reached would
97
+ // re-issue a fence a stalled worker still believes it holds.
98
+ if (!held || held.fence !== fence) return
99
+ this.claims.delete(key)
100
+ // And the counter steps past the released holding, because the disk
101
+ // store's release does: it appends a TOMBSTONE at `fence + 1`, which
102
+ // raises the maximum name and therefore the fence every later write is
103
+ // checked against. Without this step a holder that released could still
104
+ // write with the fence it just gave up — refused on disk, accepted here.
105
+ //
106
+ // The consequence is visible in the numbering: a released run's next
107
+ // claim is `fence + 2` in both stores, because the tombstone consumed
108
+ // one. That is parity, not an off-by-one.
109
+ this.highWater.set(key, fence + 1)
110
+ }
111
+
112
+ async writeCheckpoint(
113
+ scope: CheckpointRunScope,
114
+ checkpoint: IterationCheckpoint,
115
+ fence?: ClaimFence,
116
+ ): Promise<void> {
117
+ const key = this.key(scope)
118
+ // An unfenced write is allowed even on a claimed run: a host adopting
119
+ // claims on one worker must not break the workers that have not
120
+ // adopted them. A fenced write is checked, and that check is what
121
+ // makes the lease real.
122
+ if (fence !== undefined) {
123
+ // Against the HIGH-WATER MARK, not the live claim, and the difference
124
+ // is a silent loss rather than a duplicate.
125
+ //
126
+ // `releaseRun` deletes the claim. Reading `claims` here meant that
127
+ // between a release and the next take there was no holding to
128
+ // compare against, so `held` was `undefined` and every fence was
129
+ // accepted however stale. w1 stalls at fence 1; w2 reclaims at 2,
130
+ // finishes the work, releases cleanly; w1 wakes and writes with
131
+ // fence 1 — accepted — and its checkpoint carries a fresh
132
+ // `createdAt`, so it sorts newest and the next resume restores w1's
133
+ // stale history. w2's completed work is gone, with no error
134
+ // anywhere.
135
+ //
136
+ // The disk store refuses that write, because `currentFence` reads
137
+ // file names and the release tombstone raised the maximum. The
138
+ // minting side here already counted from the high-water mark; only
139
+ // the enforcement side was left reading the claim, so the two
140
+ // shipped stores disagreed at the one point that decides whether a
141
+ // lease is real.
142
+ const current = this.highWater.get(key) ?? 0
143
+ if (fence < current) throw fencedOut(scope, fence, current)
144
+ }
145
+ return this.writeUnchecked(scope, checkpoint)
146
+ }
147
+
148
+ private async writeUnchecked(
149
+ scope: CheckpointRunScope,
150
+ checkpoint: IterationCheckpoint,
151
+ ): Promise<void> {
39
152
  const key = this.key(scope)
40
153
  let run = this.runs.get(key)
41
154
  if (!run) {
@@ -93,7 +206,9 @@ export class InMemoryCheckpointStore implements CheckpointStore {
93
206
  if (scope.sessionId !== undefined && runScope.sessionId !== scope.sessionId) continue
94
207
 
95
208
  const entry = toDurableRunEntry(runScope, [...checkpoints.values()], now)
96
- if (entry) entries.push(entry)
209
+ if (!entry) continue
210
+ const claim = this.claims.get(key)
211
+ entries.push(claim ? { ...entry, claim: toClaimSummary(claim, now) } : entry)
97
212
  }
98
213
 
99
214
  return paginateDurableRuns(entries, options)