@namzu/sdk 20.3.0 → 21.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +321 -0
- package/dist/bridge/sse/mapper.d.ts +19 -0
- package/dist/bridge/sse/mapper.d.ts.map +1 -1
- package/dist/bridge/sse/mapper.js +9 -1
- package/dist/bridge/sse/mapper.js.map +1 -1
- package/dist/contracts/schemas.d.ts +10 -10
- package/dist/manager/run/persistence.d.ts +26 -0
- package/dist/manager/run/persistence.d.ts.map +1 -1
- package/dist/manager/run/persistence.js +40 -0
- package/dist/manager/run/persistence.js.map +1 -1
- package/dist/public-runtime.d.ts +5 -0
- package/dist/public-runtime.d.ts.map +1 -1
- package/dist/public-runtime.js +21 -0
- package/dist/public-runtime.js.map +1 -1
- package/dist/run/drain.d.ts +194 -0
- package/dist/run/drain.d.ts.map +1 -0
- package/dist/run/drain.js +228 -0
- package/dist/run/drain.js.map +1 -0
- package/dist/run/index.d.ts +2 -0
- package/dist/run/index.d.ts.map +1 -1
- package/dist/run/index.js +1 -0
- package/dist/run/index.js.map +1 -1
- package/dist/runtime/query/checkpoint.d.ts +27 -1
- package/dist/runtime/query/checkpoint.d.ts.map +1 -1
- package/dist/runtime/query/checkpoint.js +34 -4
- package/dist/runtime/query/checkpoint.js.map +1 -1
- package/dist/runtime/query/events.d.ts +12 -0
- package/dist/runtime/query/events.d.ts.map +1 -1
- package/dist/runtime/query/events.js +61 -3
- package/dist/runtime/query/events.js.map +1 -1
- package/dist/runtime/query/index.d.ts +57 -1
- package/dist/runtime/query/index.d.ts.map +1 -1
- package/dist/runtime/query/index.js +36 -0
- package/dist/runtime/query/index.js.map +1 -1
- package/dist/runtime/query/resume-run.d.ts +34 -1
- package/dist/runtime/query/resume-run.d.ts.map +1 -1
- package/dist/runtime/query/resume-run.js +16 -3
- package/dist/runtime/query/resume-run.js.map +1 -1
- package/dist/store/index.d.ts +2 -2
- package/dist/store/index.d.ts.map +1 -1
- package/dist/store/index.js +2 -2
- package/dist/store/index.js.map +1 -1
- package/dist/store/run/checkpoint-disk.d.ts +18 -2
- package/dist/store/run/checkpoint-disk.d.ts.map +1 -1
- package/dist/store/run/checkpoint-disk.js +50 -5
- package/dist/store/run/checkpoint-disk.js.map +1 -1
- package/dist/store/run/checkpoint-memory.d.ts +22 -2
- package/dist/store/run/checkpoint-memory.d.ts.map +1 -1
- package/dist/store/run/checkpoint-memory.js +99 -4
- package/dist/store/run/checkpoint-memory.js.map +1 -1
- package/dist/store/run/claim-disk.d.ts +130 -0
- package/dist/store/run/claim-disk.d.ts.map +1 -0
- package/dist/store/run/claim-disk.js +550 -0
- package/dist/store/run/claim-disk.js.map +1 -0
- package/dist/store/run/conformance.d.ts +159 -0
- package/dist/store/run/conformance.d.ts.map +1 -0
- package/dist/store/run/conformance.js +451 -0
- package/dist/store/run/conformance.js.map +1 -0
- package/dist/store/run/disk.d.ts +30 -2
- package/dist/store/run/disk.d.ts.map +1 -1
- package/dist/store/run/disk.js +101 -0
- package/dist/store/run/disk.js.map +1 -1
- package/dist/store/run/listing.d.ts +44 -1
- package/dist/store/run/listing.d.ts.map +1 -1
- package/dist/store/run/listing.js +92 -1
- package/dist/store/run/listing.js.map +1 -1
- package/dist/store/run/memory.d.ts +5 -4
- package/dist/store/run/memory.d.ts.map +1 -1
- package/dist/store/run/memory.js +30 -1
- package/dist/store/run/memory.js.map +1 -1
- package/dist/types/doctor/check.d.ts +28 -2
- package/dist/types/doctor/check.d.ts.map +1 -1
- package/dist/types/run/checkpoint-store.d.ts +178 -2
- package/dist/types/run/checkpoint-store.d.ts.map +1 -1
- package/dist/types/run/event-cursor.d.ts +90 -0
- package/dist/types/run/event-cursor.d.ts.map +1 -0
- package/dist/types/run/event-cursor.js +47 -0
- package/dist/types/run/event-cursor.js.map +1 -0
- package/dist/types/run/events.d.ts +71 -1
- package/dist/types/run/events.d.ts.map +1 -1
- package/dist/types/run/events.js.map +1 -1
- package/dist/types/run/index.d.ts +1 -0
- package/dist/types/run/index.d.ts.map +1 -1
- package/dist/types/run/index.js +1 -0
- package/dist/types/run/index.js.map +1 -1
- package/dist/types/run/store.d.ts +41 -1
- package/dist/types/run/store.d.ts.map +1 -1
- package/package.json +6 -1
- package/src/bridge/sse/mapper.ts +28 -1
- package/src/manager/run/persistence.ts +43 -0
- package/src/public-runtime.ts +27 -0
- package/src/run/drain.ts +393 -0
- package/src/run/index.ts +3 -0
- package/src/runtime/query/checkpoint.ts +42 -5
- package/src/runtime/query/events.ts +68 -4
- package/src/runtime/query/index.ts +113 -1
- package/src/runtime/query/resume-run.ts +77 -12
- package/src/store/index.ts +5 -1
- package/src/store/run/checkpoint-disk.ts +70 -5
- package/src/store/run/checkpoint-memory.ts +118 -3
- package/src/store/run/claim-disk.ts +593 -0
- package/src/store/run/conformance.ts +705 -0
- package/src/store/run/disk.ts +106 -2
- package/src/store/run/listing.ts +116 -1
- package/src/store/run/memory.ts +35 -5
- package/src/types/doctor/check.ts +28 -2
- package/src/types/run/checkpoint-store.ts +189 -2
- package/src/types/run/event-cursor.ts +118 -0
- package/src/types/run/events.ts +80 -3
- package/src/types/run/index.ts +1 -0
- package/src/types/run/store.ts +43 -1
|
@@ -4,6 +4,7 @@ import { buildProbeContext } from '../../probe/context.js'
|
|
|
4
4
|
import { type ProbeRegistry, probe as defaultProbeRegistry } from '../../probe/registry.js'
|
|
5
5
|
import type { ActivityEvent, ActivityStore } from '../../store/activity/memory.js'
|
|
6
6
|
import type { RunId } from '../../types/ids/index.js'
|
|
7
|
+
import type { ClaimFence } from '../../types/run/checkpoint-store.js'
|
|
7
8
|
import { isEphemeralEvent } from '../../types/run/events.js'
|
|
8
9
|
import type { RunEvent } from '../../types/run/index.js'
|
|
9
10
|
import type { TaskEvent, TaskStore } from '../../types/task/index.js'
|
|
@@ -37,6 +38,22 @@ export class EventTranslator {
|
|
|
37
38
|
this.probes = probeRegistry
|
|
38
39
|
}
|
|
39
40
|
|
|
41
|
+
/**
|
|
42
|
+
* The claim this run is being written under, when it holds one.
|
|
43
|
+
*
|
|
44
|
+
* Stamped on every durable event as its `generation`, so a consumer whose
|
|
45
|
+
* cursor predates a takeover is told its sequence space changed instead of
|
|
46
|
+
* being handed a splice from a different writer's log.
|
|
47
|
+
*/
|
|
48
|
+
private generation: ClaimFence | undefined
|
|
49
|
+
|
|
50
|
+
/** Serializes sequence assignment against the append. See {@link emitEvent}. */
|
|
51
|
+
private appendChain: Promise<void> = Promise.resolve()
|
|
52
|
+
|
|
53
|
+
setGeneration(fence: ClaimFence | undefined): void {
|
|
54
|
+
this.generation = fence
|
|
55
|
+
}
|
|
56
|
+
|
|
40
57
|
readonly emitEvent: EmitEvent = async (event: RunEvent): Promise<void> => {
|
|
41
58
|
this.probes.dispatch(event, buildProbeContext({ runId: event.runId }))
|
|
42
59
|
|
|
@@ -64,15 +81,62 @@ export class EventTranslator {
|
|
|
64
81
|
// briefly than to drop a state transition.
|
|
65
82
|
}
|
|
66
83
|
|
|
67
|
-
this.pendingEvents.push(event)
|
|
68
|
-
|
|
69
84
|
// D1 middle path: ephemeral events never enter `transcript.jsonl`.
|
|
70
85
|
// They live only on the in-memory bus for live UI rendering.
|
|
71
86
|
// Replay (`runtime/query/replay/prepare.ts`) reads checkpoints
|
|
72
87
|
// not transcripts, so this preserves replay fidelity while
|
|
73
88
|
// eliminating the durable bloat review flagged.
|
|
74
|
-
if (
|
|
75
|
-
|
|
89
|
+
if (isEphemeralEvent(event)) {
|
|
90
|
+
// No number, and that is the honest statement: nothing will
|
|
91
|
+
// persist this, so a consumer must never advance a cursor to it.
|
|
92
|
+
this.pendingEvents.push(event)
|
|
93
|
+
return
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// One appender at a time, and this is not a precaution — it is the fix
|
|
97
|
+
// for a measured defect. Taking the number, awaiting the write and then
|
|
98
|
+
// committing is a read-modify-write, and emits genuinely interleave:
|
|
99
|
+
// the task store, the plan manager and a batch of parallel tools all
|
|
100
|
+
// emit into this one funnel. Measured on a two-tool run, three events
|
|
101
|
+
// took the number 15 and two took 12. A duplicated sequence is worse
|
|
102
|
+
// than a missing one — a consumer asking for everything above 15 is
|
|
103
|
+
// handed part of the run it already had, spliced in as if it were new.
|
|
104
|
+
const previous = this.appendChain
|
|
105
|
+
let release!: () => void
|
|
106
|
+
this.appendChain = new Promise<void>((resolve) => {
|
|
107
|
+
release = resolve
|
|
108
|
+
})
|
|
109
|
+
|
|
110
|
+
try {
|
|
111
|
+
await previous
|
|
112
|
+
|
|
113
|
+
// The number is a claim that the event is IN the log, so it is taken
|
|
114
|
+
// against the append and not before it. The candidate goes to the
|
|
115
|
+
// store first; only a write that landed advances the counter and
|
|
116
|
+
// reaches the live stream carrying it.
|
|
117
|
+
//
|
|
118
|
+
// The failure path still delivers the event — unstamped. A store
|
|
119
|
+
// that cannot record a `run_failed` must not also swallow it, and an
|
|
120
|
+
// unstamped event says exactly what is true of it: it happened, and
|
|
121
|
+
// it is not recoverable.
|
|
122
|
+
const seq = this.runMgr.nextEventSeq()
|
|
123
|
+
const stamped = {
|
|
124
|
+
...event,
|
|
125
|
+
seq,
|
|
126
|
+
...(this.generation !== undefined ? { generation: this.generation } : {}),
|
|
127
|
+
} as RunEvent
|
|
128
|
+
|
|
129
|
+
try {
|
|
130
|
+
await this.runMgr.getRunStore().appendEvent(stamped)
|
|
131
|
+
} catch (err) {
|
|
132
|
+
this.pendingEvents.push(event)
|
|
133
|
+
throw err
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
this.runMgr.commitEventSeq(seq)
|
|
137
|
+
this.pendingEvents.push(stamped)
|
|
138
|
+
} finally {
|
|
139
|
+
release()
|
|
76
140
|
}
|
|
77
141
|
};
|
|
78
142
|
|
|
@@ -15,6 +15,7 @@ import { restoreWorkingState, snapshotWorkingState } from '../../compaction/wire
|
|
|
15
15
|
import type { CompactionConfig } from '../../config/runtime.js'
|
|
16
16
|
import { TOOL_OUTPUT_DIR_NAME } from '../../constants/tools/index.js'
|
|
17
17
|
import { EmergencySaveManager } from '../../manager/run/emergency.js'
|
|
18
|
+
import type { RunPersistence } from '../../manager/run/persistence.js'
|
|
18
19
|
import { resolveProviderCapabilities } from '../../provider/capabilities.js'
|
|
19
20
|
import {
|
|
20
21
|
type ProviderChainMember,
|
|
@@ -58,7 +59,9 @@ import type { AgentPersona } from '../../types/persona/index.js'
|
|
|
58
59
|
import type { LLMProvider } from '../../types/provider/index.js'
|
|
59
60
|
import type { TaskRouterConfig } from '../../types/router/index.js'
|
|
60
61
|
import type { ReviewAnswer } from '../../types/run/answer-review.js'
|
|
61
|
-
import type { CheckpointStore } from '../../types/run/checkpoint-store.js'
|
|
62
|
+
import type { CheckpointStore, ClaimFence } from '../../types/run/checkpoint-store.js'
|
|
63
|
+
import type { RunEventCursor, RunEventReplay } from '../../types/run/event-cursor.js'
|
|
64
|
+
import { resolveRunEventReplay } from '../../types/run/event-cursor.js'
|
|
62
65
|
import type {
|
|
63
66
|
AgentRunConfig,
|
|
64
67
|
PrepareStepChain,
|
|
@@ -413,6 +416,27 @@ export interface QueryParams {
|
|
|
413
416
|
*/
|
|
414
417
|
checkpointStore?: CheckpointStore
|
|
415
418
|
|
|
419
|
+
/**
|
|
420
|
+
* The fence of the claim this worker holds on the run, from `claimRun`.
|
|
421
|
+
*
|
|
422
|
+
* Presented on every checkpoint the run writes, so a worker that stalled
|
|
423
|
+
* past its lease is refused rather than writing into a run somebody else
|
|
424
|
+
* has taken over. Omit it for single-writer deployments, which is what
|
|
425
|
+
* every run did before claims existed.
|
|
426
|
+
*
|
|
427
|
+
* This hop did not exist for a release. The claim, the fence and the
|
|
428
|
+
* store-side refusal were all built and tested, and no path between a run
|
|
429
|
+
* and its store carried the number — so every checkpoint a RUN wrote went
|
|
430
|
+
* out unfenced while the tests, which called the store directly, all
|
|
431
|
+
* passed. A capability complete except for the wire between its halves
|
|
432
|
+
* reads exactly like a working one.
|
|
433
|
+
*
|
|
434
|
+
* It fences checkpoints and nothing else. {@link QueryParams.runStore}
|
|
435
|
+
* takes no fence, so two workers that both took one run still overwrite
|
|
436
|
+
* each other's run record, transcript and report — see the changeset.
|
|
437
|
+
*/
|
|
438
|
+
claimFence?: ClaimFence
|
|
439
|
+
|
|
416
440
|
/**
|
|
417
441
|
* Where this run records its own evidence — the run record, its messages,
|
|
418
442
|
* its transcript and its report. Defaults to the disk layout under the
|
|
@@ -424,6 +448,43 @@ export interface QueryParams {
|
|
|
424
448
|
*/
|
|
425
449
|
runStore?: RunStore
|
|
426
450
|
|
|
451
|
+
/**
|
|
452
|
+
* Where a reconnecting consumer left off, so this run's stream can start by
|
|
453
|
+
* handing back what it missed.
|
|
454
|
+
*
|
|
455
|
+
* The case this serves is the one that exists without a network hop: the
|
|
456
|
+
* process holding the run died, and the consumer watching it is coming back
|
|
457
|
+
* to a run that has to be resumed. Pair it with `resumeFromCheckpoint` — or
|
|
458
|
+
* reach it through {@link import('./resume-run.js').resumeRun}, which is the
|
|
459
|
+
* surface that does both — and the missed durable events are yielded, in
|
|
460
|
+
* order, before the resumed run emits anything of its own.
|
|
461
|
+
*
|
|
462
|
+
* On a run with no log to catch up on the cursor is answered honestly rather
|
|
463
|
+
* than ignored: a `sinceSeq` above what exists is `cursor_ahead`, not
|
|
464
|
+
* silence.
|
|
465
|
+
*
|
|
466
|
+
* What comes back is message-granular. Streaming deltas are never persisted
|
|
467
|
+
* — see {@link import('../../types/run/store.js').RunStore.appendEvent} —
|
|
468
|
+
* so a late subscriber recovers the assistant text, the tool results and the
|
|
469
|
+
* lifecycle, not the keystroke cadence that produced them.
|
|
470
|
+
*/
|
|
471
|
+
eventCursor?: RunEventCursor
|
|
472
|
+
|
|
473
|
+
/**
|
|
474
|
+
* What became of {@link QueryParams.eventCursor}.
|
|
475
|
+
*
|
|
476
|
+
* A callback rather than an event on the stream, because the answer is about
|
|
477
|
+
* the SUBSCRIPTION and not about the run — and rather than a throw, because
|
|
478
|
+
* a stale cursor is a client's problem and must not be able to stop a run
|
|
479
|
+
* from continuing. A host that receives `unavailable` re-derives from the
|
|
480
|
+
* transcript; one that receives nothing at all would splice a hole into its
|
|
481
|
+
* state and never know.
|
|
482
|
+
*
|
|
483
|
+
* Called once, before the run's first event, and only when a cursor was
|
|
484
|
+
* supplied.
|
|
485
|
+
*/
|
|
486
|
+
onEventReplay?: (replay: RunEventReplay) => void
|
|
487
|
+
|
|
427
488
|
runId?: RunId
|
|
428
489
|
|
|
429
490
|
parentRunId?: RunId
|
|
@@ -1013,6 +1074,14 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
|
|
|
1013
1074
|
// And every park it records carries an absolute deadline, so an
|
|
1014
1075
|
// unanswered approval cannot outlive the worker that asked for it.
|
|
1015
1076
|
checkpointMgr.setParkTtl(params.runConfig.hitlParkTtlMs)
|
|
1077
|
+
// The claim this worker holds, if it took one. Without this hop the
|
|
1078
|
+
// fence exists, the refusal exists, and no checkpoint a RUN writes ever
|
|
1079
|
+
// carries a number — so a stalled worker is refused nowhere.
|
|
1080
|
+
checkpointMgr.setClaimFence(params.claimFence)
|
|
1081
|
+
// And every EVENT it records carries the same fence as its generation,
|
|
1082
|
+
// so a consumer whose cursor was minted under an older holding is told
|
|
1083
|
+
// the sequence space changed rather than handed a splice from it.
|
|
1084
|
+
eventTranslator.setGeneration(params.claimFence)
|
|
1016
1085
|
|
|
1017
1086
|
// A question raised from inside a tool becomes a real checkpoint
|
|
1018
1087
|
// here. It used to park under a synthetic id nothing ever wrote, so
|
|
@@ -1090,6 +1159,20 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
|
|
|
1090
1159
|
try {
|
|
1091
1160
|
await ctx.runMgr.init()
|
|
1092
1161
|
|
|
1162
|
+
// A consumer coming back gets what it missed BEFORE the run says
|
|
1163
|
+
// anything new, which is the only order that lets it fold one
|
|
1164
|
+
// stream into one state. It has to follow `init()` — that is what
|
|
1165
|
+
// binds the store and reads the log's head — and precede every
|
|
1166
|
+
// emit below.
|
|
1167
|
+
if (params.eventCursor) {
|
|
1168
|
+
yield* catchUpFromCursor(
|
|
1169
|
+
ctx.runMgr,
|
|
1170
|
+
params.eventCursor,
|
|
1171
|
+
params.onEventReplay,
|
|
1172
|
+
params.claimFence,
|
|
1173
|
+
)
|
|
1174
|
+
}
|
|
1175
|
+
|
|
1093
1176
|
// Handed over here, and the position is load-bearing in BOTH
|
|
1094
1177
|
// directions. It has to follow `wirePlanManager`, or a host that
|
|
1095
1178
|
// builds its plan in this callback — which is what the callback is
|
|
@@ -1569,6 +1652,35 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
|
|
|
1569
1652
|
})()
|
|
1570
1653
|
}
|
|
1571
1654
|
|
|
1655
|
+
/**
|
|
1656
|
+
* Hand a returning consumer what it missed, or tell it why it cannot have it.
|
|
1657
|
+
*
|
|
1658
|
+
* Yields NOTHING on a refusal. A partial catch-up is the failure this exists to
|
|
1659
|
+
* prevent: a consumer that receives some of the gap folds it into its state and
|
|
1660
|
+
* cannot tell the state is wrong, where one that receives an explicit
|
|
1661
|
+
* `unavailable` re-derives from the transcript and is right. The run continues
|
|
1662
|
+
* either way — a stale cursor belongs to the client, and must not be able to
|
|
1663
|
+
* stop the work.
|
|
1664
|
+
*/
|
|
1665
|
+
async function* catchUpFromCursor(
|
|
1666
|
+
runMgr: RunPersistence,
|
|
1667
|
+
cursor: RunEventCursor,
|
|
1668
|
+
onEventReplay: ((replay: RunEventReplay) => void) | undefined,
|
|
1669
|
+
generation: ClaimFence | undefined,
|
|
1670
|
+
): AsyncGenerator<RunEvent, void> {
|
|
1671
|
+
const missed = await runMgr.getRunStore().readEvents({ sinceSeq: cursor.sinceSeq })
|
|
1672
|
+
const replay = resolveRunEventReplay(
|
|
1673
|
+
cursor,
|
|
1674
|
+
{ lastSeq: runMgr.lastEventSeq, ...(generation !== undefined ? { generation } : {}) },
|
|
1675
|
+
missed,
|
|
1676
|
+
)
|
|
1677
|
+
|
|
1678
|
+
onEventReplay?.(replay)
|
|
1679
|
+
|
|
1680
|
+
if (replay.status !== 'replayed') return
|
|
1681
|
+
for (const event of replay.events) yield event
|
|
1682
|
+
}
|
|
1683
|
+
|
|
1572
1684
|
export async function drainQuery(
|
|
1573
1685
|
params: Omit<QueryParams, 'resumeHandler'> & { resumeHandler?: ResumeHandler },
|
|
1574
1686
|
listener?: RunEventListener,
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import type { PendingDecision } from '../../types/hitl/index.js'
|
|
2
|
-
import type { CheckpointStore } from '../../types/run/checkpoint-store.js'
|
|
2
|
+
import type { CheckpointStore, ClaimFence } from '../../types/run/checkpoint-store.js'
|
|
3
3
|
import type { Run } from '../../types/run/entity.js'
|
|
4
|
+
import type { RunEventReplay } from '../../types/run/event-cursor.js'
|
|
5
|
+
import type { RunEventListener } from '../../types/run/events.js'
|
|
4
6
|
import type { RunState } from '../../types/run/state.js'
|
|
5
7
|
import { type QueryParams, drainQuery } from './index.js'
|
|
6
8
|
import { type RunStateScope, loadRunState } from './run-state.js'
|
|
@@ -23,7 +25,21 @@ export type ResumeOutcome =
|
|
|
23
25
|
readonly pending: PendingDecision
|
|
24
26
|
readonly state: RunState
|
|
25
27
|
}
|
|
26
|
-
| {
|
|
28
|
+
| {
|
|
29
|
+
readonly resumed: true
|
|
30
|
+
readonly run: Run
|
|
31
|
+
readonly state: RunState
|
|
32
|
+
/**
|
|
33
|
+
* What became of {@link ResumeRunParams.eventCursor}. Absent when no
|
|
34
|
+
* cursor was supplied.
|
|
35
|
+
*
|
|
36
|
+
* Reported rather than thrown: a stale cursor is the client's
|
|
37
|
+
* problem and the run still had to be resumed. A caller that sees
|
|
38
|
+
* `unavailable` re-derives its view from the run's transcript
|
|
39
|
+
* instead of folding a hole into it.
|
|
40
|
+
*/
|
|
41
|
+
readonly replay?: RunEventReplay
|
|
42
|
+
}
|
|
27
43
|
|
|
28
44
|
/**
|
|
29
45
|
* The half of a run that cannot be serialized, plus where to look.
|
|
@@ -38,11 +54,34 @@ export interface ResumeRunParams
|
|
|
38
54
|
readonly scope: RunStateScope
|
|
39
55
|
/** Required to find the checkpoint; also threaded into the resumed run. */
|
|
40
56
|
readonly checkpointStore: CheckpointStore
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* The fence of the claim this worker took on the run before resuming.
|
|
60
|
+
*
|
|
61
|
+
* A resume is the one moment two workers are most likely to collide — it
|
|
62
|
+
* is what a queue reader does with a parked run — so this is the call that
|
|
63
|
+
* most needs to carry one.
|
|
64
|
+
*/
|
|
65
|
+
readonly claimFence?: ClaimFence
|
|
41
66
|
/**
|
|
42
67
|
* Resume a specific checkpoint instead of the one the store would pick.
|
|
43
68
|
* Absent means the parked checkpoint if there is one, else the newest.
|
|
44
69
|
*/
|
|
45
70
|
readonly checkpointId?: RunState['checkpointId']
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Where to send the resumed run's events.
|
|
74
|
+
*
|
|
75
|
+
* This call had none, and the consequence was total: it drains the run to
|
|
76
|
+
* completion and every event the run emits — every tool call, every park,
|
|
77
|
+
* every token update — was discarded, because `drainQuery` forwards to a
|
|
78
|
+
* listener and none was ever passed. So the one API for continuing a run
|
|
79
|
+
* another process started could not show anybody what the run was doing.
|
|
80
|
+
*
|
|
81
|
+
* It is also what makes {@link ResumeRunParams.eventCursor} mean anything:
|
|
82
|
+
* a catch-up delivered into a stream nobody receives is not a catch-up.
|
|
83
|
+
*/
|
|
84
|
+
readonly listener?: RunEventListener
|
|
46
85
|
}
|
|
47
86
|
|
|
48
87
|
/**
|
|
@@ -67,7 +106,16 @@ export interface ResumeRunParams
|
|
|
67
106
|
* resumed past without the answer it is waiting for.
|
|
68
107
|
*/
|
|
69
108
|
export async function resumeRun(params: ResumeRunParams): Promise<ResumeOutcome> {
|
|
70
|
-
const {
|
|
109
|
+
const {
|
|
110
|
+
scope,
|
|
111
|
+
checkpointStore,
|
|
112
|
+
checkpointId,
|
|
113
|
+
pendingDecision,
|
|
114
|
+
claimFence,
|
|
115
|
+
listener,
|
|
116
|
+
onEventReplay,
|
|
117
|
+
...rest
|
|
118
|
+
} = params
|
|
71
119
|
|
|
72
120
|
const state = await loadRunState(checkpointStore, scope, checkpointId)
|
|
73
121
|
if (!state?.checkpointId) return { resumed: false, reason: 'no-checkpoint' }
|
|
@@ -80,14 +128,31 @@ export async function resumeRun(params: ResumeRunParams): Promise<ResumeOutcome>
|
|
|
80
128
|
return { resumed: false, reason: 'awaiting-decision', pending: outstanding, state }
|
|
81
129
|
}
|
|
82
130
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
131
|
+
let replay: RunEventReplay | undefined
|
|
132
|
+
|
|
133
|
+
const run = await drainQuery(
|
|
134
|
+
{
|
|
135
|
+
...rest,
|
|
136
|
+
messages: [],
|
|
137
|
+
runId: state.runId,
|
|
138
|
+
// Forwarded, and it is not cosmetic: the run store nests a sub-run's
|
|
139
|
+
// evidence under `<parent>/children/<run>`, so resuming a sub-run
|
|
140
|
+
// without this binds `<base>/<run>` instead — a second, empty
|
|
141
|
+
// transcript under a run id that already has one, a sequence that
|
|
142
|
+
// restarts at 1, and a catch-up that reports a live run as having
|
|
143
|
+
// produced nothing.
|
|
144
|
+
...(state.parentRunId !== undefined ? { parentRunId: state.parentRunId } : {}),
|
|
145
|
+
resumeFromCheckpoint: state.checkpointId,
|
|
146
|
+
checkpointStore,
|
|
147
|
+
...(claimFence !== undefined ? { claimFence } : {}),
|
|
148
|
+
...(pendingDecision ? { pendingDecision } : {}),
|
|
149
|
+
onEventReplay: (verdict: RunEventReplay) => {
|
|
150
|
+
replay = verdict
|
|
151
|
+
onEventReplay?.(verdict)
|
|
152
|
+
},
|
|
153
|
+
} as QueryParams,
|
|
154
|
+
listener,
|
|
155
|
+
)
|
|
91
156
|
|
|
92
|
-
return { resumed: true, run, state }
|
|
157
|
+
return { resumed: true, run, state, ...(replay !== undefined ? { replay } : {}) }
|
|
93
158
|
}
|
package/src/store/index.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
export { InMemoryStore } from './InMemoryStore.js'
|
|
2
2
|
export type { Identifiable, Timestamped } from './InMemoryStore.js'
|
|
3
3
|
|
|
4
|
-
export { RunDiskStore } from './run/disk.js'
|
|
4
|
+
export { RunDiskStore, readRunEventsIn } from './run/disk.js'
|
|
5
5
|
export { InMemoryRunStore } from './run/memory.js'
|
|
6
6
|
export { DiskCheckpointStore } from './run/checkpoint-disk.js'
|
|
7
7
|
export type { DiskCheckpointStoreAttribution } from './run/checkpoint-disk.js'
|
|
@@ -18,8 +18,12 @@ export { InMemoryCheckpointStore } from './run/checkpoint-memory.js'
|
|
|
18
18
|
// host has no use for is surface to keep correct forever for nobody.
|
|
19
19
|
export {
|
|
20
20
|
assertContiguousListingScope,
|
|
21
|
+
claimRun,
|
|
22
|
+
fencedOut,
|
|
21
23
|
listDurableRuns,
|
|
22
24
|
paginateDurableRuns,
|
|
25
|
+
releaseRun,
|
|
26
|
+
toClaimSummary,
|
|
23
27
|
toDurableRunEntry,
|
|
24
28
|
} from './run/listing.js'
|
|
25
29
|
|
|
@@ -7,14 +7,24 @@ import type {
|
|
|
7
7
|
CheckpointListingScope,
|
|
8
8
|
CheckpointRunScope,
|
|
9
9
|
CheckpointStore,
|
|
10
|
+
ClaimFence,
|
|
11
|
+
ClaimRunOptions,
|
|
10
12
|
DurableRunEntry,
|
|
11
13
|
DurableRunPage,
|
|
12
14
|
ListDurableRunsOptions,
|
|
15
|
+
RunClaim,
|
|
13
16
|
} from '../../types/run/checkpoint-store.js'
|
|
14
17
|
import type { RunStoreConfig } from '../../types/run/index.js'
|
|
15
18
|
import type { ProjectId } from '../../types/session/ids.js'
|
|
19
|
+
import { acquireClaim, currentFence, readClaim, releaseClaim } from './claim-disk.js'
|
|
16
20
|
import { RunDiskStore, readCheckpointsIn } from './disk.js'
|
|
17
|
-
import {
|
|
21
|
+
import {
|
|
22
|
+
assertContiguousListingScope,
|
|
23
|
+
fencedOut,
|
|
24
|
+
paginateDurableRuns,
|
|
25
|
+
toClaimSummary,
|
|
26
|
+
toDurableRunEntry,
|
|
27
|
+
} from './listing.js'
|
|
18
28
|
|
|
19
29
|
/**
|
|
20
30
|
* The attribution a disk store's own layout does not record.
|
|
@@ -84,11 +94,47 @@ export class DiskCheckpointStore implements CheckpointStore {
|
|
|
84
94
|
return promise
|
|
85
95
|
}
|
|
86
96
|
|
|
87
|
-
async writeCheckpoint(
|
|
97
|
+
async writeCheckpoint(
|
|
98
|
+
scope: CheckpointRunScope,
|
|
99
|
+
checkpoint: IterationCheckpoint,
|
|
100
|
+
fence?: ClaimFence,
|
|
101
|
+
): Promise<void> {
|
|
88
102
|
const store = await this.bind(scope)
|
|
103
|
+
if (fence !== undefined) {
|
|
104
|
+
// Read at the moment of the write, not at the start of the run.
|
|
105
|
+
// A holder that stalled past its lease believes it still holds,
|
|
106
|
+
// and this is the only point at which it can be told otherwise.
|
|
107
|
+
// Names only, no parsing. The fence is the file name, so a corrupt
|
|
108
|
+
// body cannot make this check skip itself — which the previous shape
|
|
109
|
+
// did, at the one site whose entire job is refusing.
|
|
110
|
+
const current = await currentFence(this.runDir(scope))
|
|
111
|
+
if (fence < current) throw fencedOut(scope, fence, current)
|
|
112
|
+
}
|
|
89
113
|
await store.writeCheckpoint(checkpoint)
|
|
90
114
|
}
|
|
91
115
|
|
|
116
|
+
async claimRun(scope: CheckpointRunScope, options: ClaimRunOptions): Promise<RunClaim | null> {
|
|
117
|
+
return acquireClaim(this.runDir(scope), options)
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
async releaseRun(scope: CheckpointRunScope, fence: ClaimFence): Promise<void> {
|
|
121
|
+
await releaseClaim(this.runDir(scope), fence)
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* The run's directory, resolved the same way `RunDiskStore.initRun` does.
|
|
126
|
+
*
|
|
127
|
+
* Duplicated rather than shared because the claim path must be derivable
|
|
128
|
+
* WITHOUT binding a store — binding creates the directory, and a claim
|
|
129
|
+
* read is a read. Kept beside the layout comment on `listDurableRuns` so
|
|
130
|
+
* the two stay together if the layout ever moves.
|
|
131
|
+
*/
|
|
132
|
+
private runDir(scope: CheckpointRunScope): string {
|
|
133
|
+
return scope.parentRunId
|
|
134
|
+
? join(this.config.baseDir, scope.parentRunId, 'children', scope.runId)
|
|
135
|
+
: join(this.config.baseDir, scope.runId)
|
|
136
|
+
}
|
|
137
|
+
|
|
92
138
|
async readCheckpoint(
|
|
93
139
|
scope: CheckpointRunScope,
|
|
94
140
|
checkpointId: CheckpointId,
|
|
@@ -163,21 +209,40 @@ export class DiskCheckpointStore implements CheckpointStore {
|
|
|
163
209
|
const runDir = join(this.config.baseDir, runId)
|
|
164
210
|
|
|
165
211
|
const own = toDurableRunEntry({ ...attribution, runId }, await readCheckpointsIn(runDir), now)
|
|
166
|
-
if (own) entries.push(own)
|
|
212
|
+
if (own) entries.push(await this.withClaim(own, runDir, now))
|
|
167
213
|
|
|
168
214
|
for (const childId of await this.readRunDirs(join(runDir, 'children'))) {
|
|
215
|
+
const childDir = join(runDir, 'children', childId)
|
|
169
216
|
const child = toDurableRunEntry(
|
|
170
217
|
{ ...attribution, runId: childId, parentRunId: runId },
|
|
171
|
-
await readCheckpointsIn(
|
|
218
|
+
await readCheckpointsIn(childDir),
|
|
172
219
|
now,
|
|
173
220
|
)
|
|
174
|
-
if (child) entries.push(child)
|
|
221
|
+
if (child) entries.push(await this.withClaim(child, childDir, now))
|
|
175
222
|
}
|
|
176
223
|
}
|
|
177
224
|
|
|
178
225
|
return paginateDurableRuns(entries, options)
|
|
179
226
|
}
|
|
180
227
|
|
|
228
|
+
/**
|
|
229
|
+
* Attach the run's claim to its listing row, judged against the page's
|
|
230
|
+
* own clock so one page cannot disagree with itself about availability.
|
|
231
|
+
*/
|
|
232
|
+
private async withClaim(
|
|
233
|
+
entry: DurableRunEntry,
|
|
234
|
+
runDir: string,
|
|
235
|
+
now: number,
|
|
236
|
+
): Promise<DurableRunEntry> {
|
|
237
|
+
const claim = await readClaim(runDir)
|
|
238
|
+
// A holding whose body could not be read still appears on the row,
|
|
239
|
+
// carrying an expiry of 0 so it reads as available. Dropping the field
|
|
240
|
+
// entirely — which is what happened before — put the run under
|
|
241
|
+
// `claimed: false` by looking unclaimed rather than by being
|
|
242
|
+
// reclaimable, so a queue reader was told a wedged run was free work.
|
|
243
|
+
return claim ? { ...entry, claim: toClaimSummary(claim, now) } : entry
|
|
244
|
+
}
|
|
245
|
+
|
|
181
246
|
/** Directory names under `dir`, or none when `dir` does not exist. */
|
|
182
247
|
private async readRunDirs(dir: string): Promise<RunId[]> {
|
|
183
248
|
try {
|
|
@@ -3,11 +3,20 @@ import type {
|
|
|
3
3
|
CheckpointListingScope,
|
|
4
4
|
CheckpointRunScope,
|
|
5
5
|
CheckpointStore,
|
|
6
|
+
ClaimFence,
|
|
7
|
+
ClaimRunOptions,
|
|
6
8
|
DurableRunEntry,
|
|
7
9
|
DurableRunPage,
|
|
8
10
|
ListDurableRunsOptions,
|
|
11
|
+
RunClaim,
|
|
9
12
|
} from '../../types/run/checkpoint-store.js'
|
|
10
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
assertContiguousListingScope,
|
|
15
|
+
fencedOut,
|
|
16
|
+
paginateDurableRuns,
|
|
17
|
+
toClaimSummary,
|
|
18
|
+
toDurableRunEntry,
|
|
19
|
+
} from './listing.js'
|
|
11
20
|
|
|
12
21
|
/**
|
|
13
22
|
* Process-local {@link CheckpointStore}, keyed by the full five-layer scope.
|
|
@@ -35,7 +44,111 @@ export class InMemoryCheckpointStore implements CheckpointStore {
|
|
|
35
44
|
return [scope.tenantId, scope.projectId, scope.sessionId, scope.runId].join('/')
|
|
36
45
|
}
|
|
37
46
|
|
|
38
|
-
|
|
47
|
+
/** `tenant/project/session/run` → the run's current holding, if any. */
|
|
48
|
+
private readonly claims = new Map<string, RunClaim>()
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* The highest fence ever issued per run, kept separately from the claim.
|
|
52
|
+
*
|
|
53
|
+
* The claim is removed on release; this is not. That separation is the
|
|
54
|
+
* whole point: the first version deleted the claim and then computed the
|
|
55
|
+
* next fence from it, so releasing rewound the counter to 1 and a worker
|
|
56
|
+
* stalled at fence 1 could write beside a new holder also at fence 1 —
|
|
57
|
+
* and the documented `finally { releaseRun() }` did it on every pass.
|
|
58
|
+
*
|
|
59
|
+
* The disk store gets this property from file names that persist. In
|
|
60
|
+
* memory the equivalent is a high-water mark nothing clears, and the two
|
|
61
|
+
* must agree, because this class is what a host reads when writing a
|
|
62
|
+
* backend of its own.
|
|
63
|
+
*/
|
|
64
|
+
private readonly highWater = new Map<string, ClaimFence>()
|
|
65
|
+
|
|
66
|
+
async claimRun(scope: CheckpointRunScope, options: ClaimRunOptions): Promise<RunClaim | null> {
|
|
67
|
+
const key = this.key(scope)
|
|
68
|
+
const now = options.now ?? Date.now()
|
|
69
|
+
const held = this.claims.get(key)
|
|
70
|
+
|
|
71
|
+
// Held by somebody else and still live. Not an error: two readers on
|
|
72
|
+
// one queue is the ordinary case.
|
|
73
|
+
if (held && now < held.expiresAt && held.holder !== options.holder) return null
|
|
74
|
+
|
|
75
|
+
// A reclaim of an expired holding and a renewal by the current holder
|
|
76
|
+
// are the same write. The fence advances either way, so a previous
|
|
77
|
+
// holder that wakes up is fenced out in both cases — a renewal that
|
|
78
|
+
// kept the fence would leave a stalled twin able to write.
|
|
79
|
+
// Counted from the high-water mark, never from the live claim. A
|
|
80
|
+
// released run has no claim, and computing from that absence is what
|
|
81
|
+
// rewound the counter to 1 on every release.
|
|
82
|
+
const fence = (this.highWater.get(key) ?? 0) + 1
|
|
83
|
+
const claim: RunClaim = { holder: options.holder, fence, expiresAt: now + options.ttlMs }
|
|
84
|
+
this.highWater.set(key, fence)
|
|
85
|
+
this.claims.set(key, claim)
|
|
86
|
+
return claim
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
async releaseRun(scope: CheckpointRunScope, fence: ClaimFence): Promise<void> {
|
|
90
|
+
const key = this.key(scope)
|
|
91
|
+
const held = this.claims.get(key)
|
|
92
|
+
// A stale fence releases nothing. A worker that stalled past its lease
|
|
93
|
+
// must not be able to hand away a run somebody else now holds.
|
|
94
|
+
//
|
|
95
|
+
// The high-water mark deliberately survives this. Dropping the claim
|
|
96
|
+
// returns the run to the queue; forgetting the number it reached would
|
|
97
|
+
// re-issue a fence a stalled worker still believes it holds.
|
|
98
|
+
if (!held || held.fence !== fence) return
|
|
99
|
+
this.claims.delete(key)
|
|
100
|
+
// And the counter steps past the released holding, because the disk
|
|
101
|
+
// store's release does: it appends a TOMBSTONE at `fence + 1`, which
|
|
102
|
+
// raises the maximum name and therefore the fence every later write is
|
|
103
|
+
// checked against. Without this step a holder that released could still
|
|
104
|
+
// write with the fence it just gave up — refused on disk, accepted here.
|
|
105
|
+
//
|
|
106
|
+
// The consequence is visible in the numbering: a released run's next
|
|
107
|
+
// claim is `fence + 2` in both stores, because the tombstone consumed
|
|
108
|
+
// one. That is parity, not an off-by-one.
|
|
109
|
+
this.highWater.set(key, fence + 1)
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
async writeCheckpoint(
|
|
113
|
+
scope: CheckpointRunScope,
|
|
114
|
+
checkpoint: IterationCheckpoint,
|
|
115
|
+
fence?: ClaimFence,
|
|
116
|
+
): Promise<void> {
|
|
117
|
+
const key = this.key(scope)
|
|
118
|
+
// An unfenced write is allowed even on a claimed run: a host adopting
|
|
119
|
+
// claims on one worker must not break the workers that have not
|
|
120
|
+
// adopted them. A fenced write is checked, and that check is what
|
|
121
|
+
// makes the lease real.
|
|
122
|
+
if (fence !== undefined) {
|
|
123
|
+
// Against the HIGH-WATER MARK, not the live claim, and the difference
|
|
124
|
+
// is a silent loss rather than a duplicate.
|
|
125
|
+
//
|
|
126
|
+
// `releaseRun` deletes the claim. Reading `claims` here meant that
|
|
127
|
+
// between a release and the next take there was no holding to
|
|
128
|
+
// compare against, so `held` was `undefined` and every fence was
|
|
129
|
+
// accepted however stale. w1 stalls at fence 1; w2 reclaims at 2,
|
|
130
|
+
// finishes the work, releases cleanly; w1 wakes and writes with
|
|
131
|
+
// fence 1 — accepted — and its checkpoint carries a fresh
|
|
132
|
+
// `createdAt`, so it sorts newest and the next resume restores w1's
|
|
133
|
+
// stale history. w2's completed work is gone, with no error
|
|
134
|
+
// anywhere.
|
|
135
|
+
//
|
|
136
|
+
// The disk store refuses that write, because `currentFence` reads
|
|
137
|
+
// file names and the release tombstone raised the maximum. The
|
|
138
|
+
// minting side here already counted from the high-water mark; only
|
|
139
|
+
// the enforcement side was left reading the claim, so the two
|
|
140
|
+
// shipped stores disagreed at the one point that decides whether a
|
|
141
|
+
// lease is real.
|
|
142
|
+
const current = this.highWater.get(key) ?? 0
|
|
143
|
+
if (fence < current) throw fencedOut(scope, fence, current)
|
|
144
|
+
}
|
|
145
|
+
return this.writeUnchecked(scope, checkpoint)
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
private async writeUnchecked(
|
|
149
|
+
scope: CheckpointRunScope,
|
|
150
|
+
checkpoint: IterationCheckpoint,
|
|
151
|
+
): Promise<void> {
|
|
39
152
|
const key = this.key(scope)
|
|
40
153
|
let run = this.runs.get(key)
|
|
41
154
|
if (!run) {
|
|
@@ -93,7 +206,9 @@ export class InMemoryCheckpointStore implements CheckpointStore {
|
|
|
93
206
|
if (scope.sessionId !== undefined && runScope.sessionId !== scope.sessionId) continue
|
|
94
207
|
|
|
95
208
|
const entry = toDurableRunEntry(runScope, [...checkpoints.values()], now)
|
|
96
|
-
if (entry)
|
|
209
|
+
if (!entry) continue
|
|
210
|
+
const claim = this.claims.get(key)
|
|
211
|
+
entries.push(claim ? { ...entry, claim: toClaimSummary(claim, now) } : entry)
|
|
97
212
|
}
|
|
98
213
|
|
|
99
214
|
return paginateDurableRuns(entries, options)
|