@namzu/sdk 20.2.0 → 20.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +185 -0
- package/dist/manager/run/persistence.d.ts +2 -2
- package/dist/manager/run/persistence.d.ts.map +1 -1
- package/dist/manager/run/persistence.js +14 -5
- package/dist/manager/run/persistence.js.map +1 -1
- package/dist/public-runtime.d.ts +2 -1
- package/dist/public-runtime.d.ts.map +1 -1
- package/dist/public-runtime.js +6 -1
- package/dist/public-runtime.js.map +1 -1
- package/dist/runtime/query/checkpoint.d.ts +27 -1
- package/dist/runtime/query/checkpoint.d.ts.map +1 -1
- package/dist/runtime/query/checkpoint.js +34 -4
- package/dist/runtime/query/checkpoint.js.map +1 -1
- package/dist/runtime/query/context.d.ts +2 -0
- package/dist/runtime/query/context.d.ts.map +1 -1
- package/dist/runtime/query/context.js +1 -0
- package/dist/runtime/query/context.js.map +1 -1
- package/dist/runtime/query/index.d.ts +32 -1
- package/dist/runtime/query/index.d.ts.map +1 -1
- package/dist/runtime/query/index.js +5 -0
- package/dist/runtime/query/index.js.map +1 -1
- package/dist/runtime/query/resume-run.d.ts +9 -1
- package/dist/runtime/query/resume-run.d.ts.map +1 -1
- package/dist/runtime/query/resume-run.js +2 -1
- package/dist/runtime/query/resume-run.js.map +1 -1
- package/dist/store/index.d.ts +2 -1
- package/dist/store/index.d.ts.map +1 -1
- package/dist/store/index.js +2 -1
- package/dist/store/index.js.map +1 -1
- package/dist/store/run/checkpoint-disk.d.ts +18 -2
- package/dist/store/run/checkpoint-disk.d.ts.map +1 -1
- package/dist/store/run/checkpoint-disk.js +50 -5
- package/dist/store/run/checkpoint-disk.js.map +1 -1
- package/dist/store/run/checkpoint-memory.d.ts +22 -2
- package/dist/store/run/checkpoint-memory.d.ts.map +1 -1
- package/dist/store/run/checkpoint-memory.js +99 -4
- package/dist/store/run/checkpoint-memory.js.map +1 -1
- package/dist/store/run/claim-disk.d.ts +130 -0
- package/dist/store/run/claim-disk.d.ts.map +1 -0
- package/dist/store/run/claim-disk.js +550 -0
- package/dist/store/run/claim-disk.js.map +1 -0
- package/dist/store/run/disk.d.ts +10 -8
- package/dist/store/run/disk.d.ts.map +1 -1
- package/dist/store/run/disk.js.map +1 -1
- package/dist/store/run/listing.d.ts +44 -1
- package/dist/store/run/listing.d.ts.map +1 -1
- package/dist/store/run/listing.js +92 -1
- package/dist/store/run/listing.js.map +1 -1
- package/dist/store/run/memory.d.ts +46 -0
- package/dist/store/run/memory.d.ts.map +1 -0
- package/dist/store/run/memory.js +104 -0
- package/dist/store/run/memory.js.map +1 -0
- package/dist/types/run/checkpoint-store.d.ts +178 -2
- package/dist/types/run/checkpoint-store.d.ts.map +1 -1
- package/dist/types/run/config.d.ts +12 -0
- package/dist/types/run/config.d.ts.map +1 -1
- package/dist/types/run/index.d.ts +1 -0
- package/dist/types/run/index.d.ts.map +1 -1
- package/dist/types/run/index.js +1 -0
- package/dist/types/run/index.js.map +1 -1
- package/dist/types/run/store.d.ts +103 -0
- package/dist/types/run/store.d.ts.map +1 -0
- package/dist/types/run/store.js +30 -0
- package/dist/types/run/store.js.map +1 -0
- package/package.json +1 -1
- package/src/manager/run/persistence.ts +17 -7
- package/src/public-runtime.ts +6 -0
- package/src/runtime/query/checkpoint.ts +42 -5
- package/src/runtime/query/context.ts +3 -0
- package/src/runtime/query/index.ts +39 -1
- package/src/runtime/query/resume-run.ts +12 -2
- package/src/store/index.ts +5 -0
- package/src/store/run/checkpoint-disk.ts +70 -5
- package/src/store/run/checkpoint-memory.ts +118 -3
- package/src/store/run/claim-disk.ts +593 -0
- package/src/store/run/disk.ts +10 -8
- package/src/store/run/listing.ts +116 -1
- package/src/store/run/memory.ts +121 -0
- package/src/types/run/checkpoint-store.ts +189 -2
- package/src/types/run/config.ts +13 -0
- package/src/types/run/index.ts +1 -0
- package/src/types/run/store.ts +112 -0
package/src/store/run/listing.ts
CHANGED
|
@@ -14,12 +14,16 @@ import type {
|
|
|
14
14
|
CheckpointListingScope,
|
|
15
15
|
CheckpointRunScope,
|
|
16
16
|
CheckpointStore,
|
|
17
|
+
ClaimFence,
|
|
18
|
+
ClaimRunOptions,
|
|
19
|
+
ClaimSummary,
|
|
17
20
|
DurableRunEntry,
|
|
18
21
|
DurableRunOrder,
|
|
19
22
|
DurableRunPage,
|
|
20
23
|
ListDurableRunsOptions,
|
|
21
24
|
ParkState,
|
|
22
25
|
ParkSummary,
|
|
26
|
+
RunClaim,
|
|
23
27
|
} from '../../types/run/checkpoint-store.js'
|
|
24
28
|
|
|
25
29
|
/** Page size when the caller names none. */
|
|
@@ -186,11 +190,20 @@ export function paginateDurableRuns(
|
|
|
186
190
|
options?: ListDurableRunsOptions,
|
|
187
191
|
): DurableRunPage {
|
|
188
192
|
const wanted = options?.park
|
|
189
|
-
const
|
|
193
|
+
const byPark =
|
|
190
194
|
wanted && wanted.length > 0
|
|
191
195
|
? entries.filter((e) => e.park !== undefined && wanted.includes(e.park.state))
|
|
192
196
|
: entries
|
|
193
197
|
|
|
198
|
+
// An expired claim counts as unheld. That is what expiry means, and a
|
|
199
|
+
// queue reader that treated an expired claim as held would leave a dead
|
|
200
|
+
// worker's runs invisible forever — the exact failure a lease exists to
|
|
201
|
+
// prevent, reintroduced by the filter that reads it.
|
|
202
|
+
const filtered =
|
|
203
|
+
options?.claimed === undefined
|
|
204
|
+
? byPark
|
|
205
|
+
: byPark.filter((e) => (e.claim !== undefined && !e.claim.expired) === options.claimed)
|
|
206
|
+
|
|
194
207
|
// Both orders sort on a key that cannot move under a paging caller — see
|
|
195
208
|
// the contract comment on `listDurableRuns`.
|
|
196
209
|
const orderBy = options?.orderBy ?? 'runId'
|
|
@@ -305,3 +318,105 @@ export async function listDurableRuns(
|
|
|
305
318
|
assertContiguousListingScope(scope, 'listDurableRuns')
|
|
306
319
|
return store.listDurableRuns(scope, options)
|
|
307
320
|
}
|
|
321
|
+
|
|
322
|
+
/**
|
|
323
|
+
* Take working possession of a run, refusing when the store cannot arbitrate.
|
|
324
|
+
*
|
|
325
|
+
* The refusal is the entire safety property. `claimRun` is optional on the
|
|
326
|
+
* contract, and the natural way to reach an absent optional method is to skip
|
|
327
|
+
* it — which here means every worker proceeds, believing it holds a run
|
|
328
|
+
* nobody arbitrated. Two workers then restore the same checkpoint, both run
|
|
329
|
+
* the tools, and both write under one run id; half the work vanishes with no
|
|
330
|
+
* error anywhere.
|
|
331
|
+
*
|
|
332
|
+
* So a store with no claim support does not get "claimed by default". It gets
|
|
333
|
+
* an error naming the deployment shape it cannot support. A single-writer
|
|
334
|
+
* host never calls this and is unaffected.
|
|
335
|
+
*
|
|
336
|
+
* Returns `null` — not an error — when another holder has the run. That is
|
|
337
|
+
* the ordinary outcome of two readers on one queue, and a caller loops to the
|
|
338
|
+
* next run rather than handling a fault.
|
|
339
|
+
*/
|
|
340
|
+
export async function claimRun(
|
|
341
|
+
store: CheckpointStore,
|
|
342
|
+
scope: CheckpointRunScope,
|
|
343
|
+
options: ClaimRunOptions,
|
|
344
|
+
): Promise<RunClaim | null> {
|
|
345
|
+
if (typeof store.claimRun !== 'function') {
|
|
346
|
+
throw new NamzuError({
|
|
347
|
+
code: 'capability_unavailable',
|
|
348
|
+
message:
|
|
349
|
+
'claimRun: the injected checkpoint store does not implement `claimRun`, so it cannot arbitrate between two workers taking the same run. Refusing rather than proceeding unclaimed — proceeding would let two workers restore one checkpoint, both execute its tools, and both write under one run id, which loses half the work and reports nothing. Supply a store that implements it, or run a single writer per run.',
|
|
350
|
+
details: { runId: scope.runId },
|
|
351
|
+
})
|
|
352
|
+
}
|
|
353
|
+
if (!Number.isFinite(options.ttlMs) || options.ttlMs <= 0) {
|
|
354
|
+
throw new NamzuError({
|
|
355
|
+
code: 'invalid_config',
|
|
356
|
+
message: `claimRun: ttlMs must be a positive number of milliseconds, got ${String(options.ttlMs)}. A lease that expires immediately is a lease every worker can take at once, which is the condition this call exists to prevent.`,
|
|
357
|
+
details: { runId: scope.runId, ttlMs: options.ttlMs },
|
|
358
|
+
})
|
|
359
|
+
}
|
|
360
|
+
return store.claimRun(scope, options)
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/**
|
|
364
|
+
* Give a claim up early, refusing when the store cannot arbitrate.
|
|
365
|
+
*
|
|
366
|
+
* Refuses for the same reason as {@link claimRun}: a host that believes it is
|
|
367
|
+
* releasing a claim on a store that has none is a host that believes the
|
|
368
|
+
* whole mechanism is running.
|
|
369
|
+
*/
|
|
370
|
+
export async function releaseRun(
|
|
371
|
+
store: CheckpointStore,
|
|
372
|
+
scope: CheckpointRunScope,
|
|
373
|
+
fence: ClaimFence,
|
|
374
|
+
): Promise<void> {
|
|
375
|
+
if (typeof store.releaseRun !== 'function') {
|
|
376
|
+
throw new NamzuError({
|
|
377
|
+
code: 'capability_unavailable',
|
|
378
|
+
message:
|
|
379
|
+
'releaseRun: the injected checkpoint store does not implement `releaseRun`. A release that silently does nothing would leave the run held until its lease expires while the caller believes it is back on the queue.',
|
|
380
|
+
details: { runId: scope.runId },
|
|
381
|
+
})
|
|
382
|
+
}
|
|
383
|
+
return store.releaseRun(scope, fence)
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
/**
|
|
387
|
+
* The refusal a store raises when a write presents a superseded fence.
|
|
388
|
+
*
|
|
389
|
+
* Shared so both shipped stores say the same thing, and so a host writing its
|
|
390
|
+
* own backend raises something a caller can branch on rather than a message
|
|
391
|
+
* string. This is the moment a stalled worker learns it lost the run — the
|
|
392
|
+
* only moment it CAN learn, since from the inside a pause and a partition
|
|
393
|
+
* both look like time not passing.
|
|
394
|
+
*/
|
|
395
|
+
export function fencedOut(
|
|
396
|
+
scope: CheckpointRunScope,
|
|
397
|
+
presented: number,
|
|
398
|
+
current: number,
|
|
399
|
+
): NamzuError {
|
|
400
|
+
return new NamzuError({
|
|
401
|
+
code: 'storage_error',
|
|
402
|
+
message: `writeCheckpoint: refusing a write for run ${scope.runId} fenced at ${presented} — the run is now claimed at ${current}. Another worker took this run over, so this process no longer holds it and its work is not the record. Stop the run rather than retrying: the claim is gone, not busy.`,
|
|
403
|
+
details: { runId: scope.runId, presentedFence: presented, currentFence: current },
|
|
404
|
+
retryable: false,
|
|
405
|
+
})
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/**
|
|
409
|
+
* Whether a recorded claim still holds at `now`, and the summary a listing
|
|
410
|
+
* reports for it.
|
|
411
|
+
*/
|
|
412
|
+
export function toClaimSummary(claim: RunClaim, now: number): ClaimSummary {
|
|
413
|
+
return {
|
|
414
|
+
holder: claim.holder,
|
|
415
|
+
fence: claim.fence,
|
|
416
|
+
expiresAt: claim.expiresAt,
|
|
417
|
+
// Judged here, once, against the store's clock. Left to the caller it
|
|
418
|
+
// would be judged against a different one, and a single page could
|
|
419
|
+
// then disagree with itself about which rows are available.
|
|
420
|
+
expired: now >= claim.expiresAt,
|
|
421
|
+
}
|
|
422
|
+
}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import type { Run } from '../../types/run/entity.js'
|
|
2
|
+
import type { RunEvent } from '../../types/run/events.js'
|
|
3
|
+
import type { CompletedToolRecord, RunStore } from '../../types/run/store.js'
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Process-local {@link RunStore}: a run's evidence with no filesystem.
|
|
7
|
+
*
|
|
8
|
+
* The reason it ships rather than living in a test file is that it is the
|
|
9
|
+
* only way to demonstrate the seam actually is one. A contract with a single
|
|
10
|
+
* implementation is a refactor; the second implementation is what proves a
|
|
11
|
+
* host could supply a third. It is also the parity partner for the disk
|
|
12
|
+
* store — a memory store that answers differently from disk is worse than
|
|
13
|
+
* none, because a host tests against one and ships the other.
|
|
14
|
+
*
|
|
15
|
+
* Deliberately not durable. It is for tests, for a single-process host that
|
|
16
|
+
* genuinely wants a run's evidence to die with the process, and for
|
|
17
|
+
* environments with no writable filesystem at all.
|
|
18
|
+
*/
|
|
19
|
+
export class InMemoryRunStore implements RunStore {
|
|
20
|
+
private runId: string | null = null
|
|
21
|
+
private parentRunId: string | undefined
|
|
22
|
+
private meta: Run | null = null
|
|
23
|
+
private messages: Run['messages'] = []
|
|
24
|
+
private report: string | null = null
|
|
25
|
+
private readonly events: RunEvent[] = []
|
|
26
|
+
|
|
27
|
+
async initRun(runId: string, parentRunId?: string): Promise<string | null> {
|
|
28
|
+
this.runId = runId
|
|
29
|
+
this.parentRunId = parentRunId
|
|
30
|
+
// No location, and that is the honest answer rather than a defect.
|
|
31
|
+
// Callers render `null` as "this run is not on a filesystem"; a
|
|
32
|
+
// synthesized path would put a directory that does not exist in front
|
|
33
|
+
// of an operator.
|
|
34
|
+
return null
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
private requireInit(): string {
|
|
38
|
+
if (this.runId === null) {
|
|
39
|
+
throw new Error('InMemoryRunStore not initialized — call initRun() first')
|
|
40
|
+
}
|
|
41
|
+
return this.runId
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** The run this store is bound to, and its parent when it has one. */
|
|
45
|
+
get boundTo(): { runId: string; parentRunId?: string } | null {
|
|
46
|
+
return this.runId === null
|
|
47
|
+
? null
|
|
48
|
+
: { runId: this.runId, ...(this.parentRunId ? { parentRunId: this.parentRunId } : {}) }
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
async writeRunMeta(run: Run): Promise<void> {
|
|
52
|
+
this.requireInit()
|
|
53
|
+
// Copied, not referenced. The caller keeps mutating this object for
|
|
54
|
+
// the rest of the run, so storing it by reference would make every
|
|
55
|
+
// historical read return the run's present state — a transcript that
|
|
56
|
+
// silently rewrites itself is worse than no transcript.
|
|
57
|
+
this.meta = structuredClone(run)
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
async writeMessages(run: Run): Promise<void> {
|
|
61
|
+
this.requireInit()
|
|
62
|
+
this.messages = structuredClone(run.messages)
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
async appendEvent(event: RunEvent): Promise<void> {
|
|
66
|
+
this.requireInit()
|
|
67
|
+
// Stamped on write, exactly as the disk store stamps its transcript
|
|
68
|
+
// line — a parity test compares the two read-backs, and a timestamp
|
|
69
|
+
// present in one medium and absent in the other would make identical
|
|
70
|
+
// runs look different depending on where they were recorded.
|
|
71
|
+
this.events.push({ ...event, timestamp: Date.now() } as unknown as RunEvent)
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
async writeReport(content: string): Promise<string | null> {
|
|
75
|
+
this.requireInit()
|
|
76
|
+
this.report = content
|
|
77
|
+
return null
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
async readCompletedTools(): Promise<Map<string, CompletedToolRecord>> {
|
|
81
|
+
this.requireInit()
|
|
82
|
+
const completed = new Map<string, CompletedToolRecord>()
|
|
83
|
+
for (const event of this.events) {
|
|
84
|
+
const e = event as unknown as Record<string, unknown>
|
|
85
|
+
if (e.type !== 'tool_completed') continue
|
|
86
|
+
const toolUseId = e.toolUseId
|
|
87
|
+
const toolName = e.toolName
|
|
88
|
+
if (typeof toolUseId !== 'string' || typeof toolName !== 'string') continue
|
|
89
|
+
// Last write wins: a retried tool emits one event per attempt and
|
|
90
|
+
// the final one is what actually answered the call. Same rule the
|
|
91
|
+
// disk store applies, and it has to be the same rule — a resumed
|
|
92
|
+
// run must not depend on which backend it was recorded with.
|
|
93
|
+
completed.set(toolUseId, {
|
|
94
|
+
toolUseId,
|
|
95
|
+
toolName,
|
|
96
|
+
result: typeof e.result === 'string' ? e.result : '',
|
|
97
|
+
isError: e.isError === true,
|
|
98
|
+
})
|
|
99
|
+
}
|
|
100
|
+
return completed
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
getRunDir(): string | null {
|
|
104
|
+
return null
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// `addToIndex` is deliberately not implemented. It maintains a browsable
|
|
108
|
+
// catalogue for a human reading a directory, and there is no directory
|
|
109
|
+
// here. The optional method exists on the contract precisely so a backend
|
|
110
|
+
// can decline it rather than implement a no-op that looks like a listing.
|
|
111
|
+
|
|
112
|
+
/** Everything recorded for the bound run, for tests and parity checks. */
|
|
113
|
+
snapshot(): {
|
|
114
|
+
meta: Run | null
|
|
115
|
+
messages: Run['messages']
|
|
116
|
+
report: string | null
|
|
117
|
+
events: readonly RunEvent[]
|
|
118
|
+
} {
|
|
119
|
+
return { meta: this.meta, messages: this.messages, report: this.report, events: this.events }
|
|
120
|
+
}
|
|
121
|
+
}
|
|
@@ -175,6 +175,17 @@ export interface DurableRunEntry extends CheckpointRunScope {
|
|
|
175
175
|
readonly latestCheckpointAt: number
|
|
176
176
|
/** Absent when the run has never parked. */
|
|
177
177
|
readonly park?: ParkSummary
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* Absent when no process has ever claimed the run.
|
|
181
|
+
*
|
|
182
|
+
* A SIBLING of {@link DurableRunEntry.park}, not a member of
|
|
183
|
+
* {@link ParkState} — see the note on that union. A park is a question put
|
|
184
|
+
* to a human; a claim is a lease held by a process. A run can have both,
|
|
185
|
+
* neither, or either, and the state a queue worker needs most is parked
|
|
186
|
+
* AND unclaimed, which one union cannot say.
|
|
187
|
+
*/
|
|
188
|
+
readonly claim?: ClaimSummary
|
|
178
189
|
}
|
|
179
190
|
|
|
180
191
|
/**
|
|
@@ -208,6 +219,65 @@ export type DurableRunOrder =
|
|
|
208
219
|
*/
|
|
209
220
|
| 'createdAt'
|
|
210
221
|
|
|
222
|
+
/**
|
|
223
|
+
* A monotonically increasing number identifying one holding of a run's claim.
|
|
224
|
+
*
|
|
225
|
+
* The load-bearing word is *fencing*. A mutex answers "may I proceed", and a
|
|
226
|
+
* holder that stalls past its lease — a long GC pause, a suspended container,
|
|
227
|
+
* a partitioned network — answers it "yes" and then writes, long after
|
|
228
|
+
* somebody else legitimately took over. A fence answers a different question
|
|
229
|
+
* at the moment of the WRITE: "is the holding I belong to still the current
|
|
230
|
+
* one". Every claim of a run mints a number strictly greater than the last,
|
|
231
|
+
* so a store can reject a write from a superseded holder without knowing
|
|
232
|
+
* anything about processes, clocks or liveness.
|
|
233
|
+
*
|
|
234
|
+
* Not a random token, deliberately: randomness proves identity and cannot
|
|
235
|
+
* establish *order*, and order is the entire mechanism.
|
|
236
|
+
*/
|
|
237
|
+
export type ClaimFence = number
|
|
238
|
+
|
|
239
|
+
/** A holding of a run's claim, as issued to the process that took it. */
|
|
240
|
+
export interface RunClaim {
|
|
241
|
+
/** Opaque caller-supplied identity — a worker id, a pod name. Evidence, not authority. */
|
|
242
|
+
readonly holder: string
|
|
243
|
+
/** See {@link ClaimFence}. Present it on every durable write. */
|
|
244
|
+
readonly fence: ClaimFence
|
|
245
|
+
/**
|
|
246
|
+
* Absolute epoch ms after which the claim may be taken by somebody else.
|
|
247
|
+
*
|
|
248
|
+
* Absolute rather than a duration for the same reason a park's deadline
|
|
249
|
+
* is: it has to survive the process that set it. A duration plus an
|
|
250
|
+
* in-process timer cannot — the holder is the thing that dies.
|
|
251
|
+
*/
|
|
252
|
+
readonly expiresAt: number
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/** A run's claim as a listing reports it. */
|
|
256
|
+
export interface ClaimSummary {
|
|
257
|
+
readonly holder: string
|
|
258
|
+
readonly fence: ClaimFence
|
|
259
|
+
readonly expiresAt: number
|
|
260
|
+
/**
|
|
261
|
+
* Whether the claim had expired at the instant the listing was taken.
|
|
262
|
+
*
|
|
263
|
+
* A separate field rather than something the caller derives from
|
|
264
|
+
* `expiresAt`, because the caller would derive it against a DIFFERENT
|
|
265
|
+
* clock than the store used, and one page would then disagree with
|
|
266
|
+
* itself about which rows are available.
|
|
267
|
+
*/
|
|
268
|
+
readonly expired: boolean
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/** What a caller asks for when taking a run. */
|
|
272
|
+
export interface ClaimRunOptions {
|
|
273
|
+
/** Who is taking it. Recorded so an operator can see what holds a stuck run. */
|
|
274
|
+
readonly holder: string
|
|
275
|
+
/** How long the holding is good for, in ms. */
|
|
276
|
+
readonly ttlMs: number
|
|
277
|
+
/** Clock, for tests and so one operation judges every expiry against one instant. */
|
|
278
|
+
readonly now?: number
|
|
279
|
+
}
|
|
280
|
+
|
|
211
281
|
/** Filters and paging for {@link CheckpointStore.listDurableRuns}. */
|
|
212
282
|
export interface ListDurableRunsOptions {
|
|
213
283
|
/**
|
|
@@ -223,6 +293,16 @@ export interface ListDurableRunsOptions {
|
|
|
223
293
|
* to include it.
|
|
224
294
|
*/
|
|
225
295
|
readonly park?: readonly ParkState[]
|
|
296
|
+
/**
|
|
297
|
+
* Keep only runs that are, or are not, currently held by a worker.
|
|
298
|
+
*
|
|
299
|
+
* `false` is the queue-reader's filter: give me the work nobody has. A
|
|
300
|
+
* claim that has expired counts as NOT held, because that is what expiry
|
|
301
|
+
* means and a reader that skipped expired claims would leave a dead
|
|
302
|
+
* worker's runs invisible forever — the exact failure the lease exists to
|
|
303
|
+
* prevent.
|
|
304
|
+
*/
|
|
305
|
+
readonly claimed?: boolean
|
|
226
306
|
/** Page size. Defaults to 100, clamped to at least 1. */
|
|
227
307
|
readonly limit?: number
|
|
228
308
|
/** Resume token from the previous page's {@link DurableRunPage.cursor}. */
|
|
@@ -279,8 +359,27 @@ export interface DurableRunPage {
|
|
|
279
359
|
* proceed.
|
|
280
360
|
*/
|
|
281
361
|
export interface CheckpointStore {
|
|
282
|
-
/**
|
|
283
|
-
|
|
362
|
+
/**
|
|
363
|
+
* Persist one checkpoint. Overwrites an existing checkpoint with the same id.
|
|
364
|
+
*
|
|
365
|
+
* @param fence the {@link ClaimFence} of the holding this write belongs
|
|
366
|
+
* to, when the run is claimed. A store that supports claims REFUSES a
|
|
367
|
+
* write whose fence is below the run's current one — that refusal is
|
|
368
|
+
* what makes a claim a lease rather than a suggestion, because a holder
|
|
369
|
+
* stalled past its expiry believes it still holds and is wrong only at
|
|
370
|
+
* the moment it writes.
|
|
371
|
+
*
|
|
372
|
+
* Omit it and the write is unfenced, which is exactly today's behaviour
|
|
373
|
+
* and correct for a single-writer deployment. A store MUST NOT start
|
|
374
|
+
* refusing unfenced writes because some other write carried a fence:
|
|
375
|
+
* that would make adding a claim to one worker break every worker that
|
|
376
|
+
* has not adopted it yet.
|
|
377
|
+
*/
|
|
378
|
+
writeCheckpoint(
|
|
379
|
+
scope: CheckpointRunScope,
|
|
380
|
+
checkpoint: IterationCheckpoint,
|
|
381
|
+
fence?: ClaimFence,
|
|
382
|
+
): Promise<void>
|
|
284
383
|
|
|
285
384
|
/** Load a single checkpoint by id. Returns `null` when it does not exist. */
|
|
286
385
|
readCheckpoint(
|
|
@@ -342,4 +441,92 @@ export interface CheckpointStore {
|
|
|
342
441
|
scope: CheckpointListingScope,
|
|
343
442
|
options?: ListDurableRunsOptions,
|
|
344
443
|
): Promise<DurableRunPage>
|
|
444
|
+
|
|
445
|
+
/**
|
|
446
|
+
* Take exclusive working possession of a run, or report that somebody
|
|
447
|
+
* else has it. OPTIONAL — see the optional-capability rule on this
|
|
448
|
+
* interface.
|
|
449
|
+
*
|
|
450
|
+
* Returns the holding on success and `null` when the run is currently
|
|
451
|
+
* held by somebody else. `null` is not an error: "another worker got
|
|
452
|
+
* there first" is the ordinary outcome of a queue with more than one
|
|
453
|
+
* reader, and a thrown exception would make the normal case look like a
|
|
454
|
+
* fault.
|
|
455
|
+
*
|
|
456
|
+
* ### What it is for
|
|
457
|
+
*
|
|
458
|
+
* Putting parked runs on a queue and letting more than one worker drain
|
|
459
|
+
* it. Without this, two workers restore the same checkpoint, both execute
|
|
460
|
+
* the run's tools, and both write checkpoints under one run id — each
|
|
461
|
+
* write minting a fresh checkpoint id, so two divergent chains land in
|
|
462
|
+
* one list and the pending lookup returns whichever wrote last. Half the
|
|
463
|
+
* work vanishes and nothing reports an error.
|
|
464
|
+
*
|
|
465
|
+
* ### The lease, and why it expires
|
|
466
|
+
*
|
|
467
|
+
* A claim is a LEASE, not a lock. A lock held by a process that dies is
|
|
468
|
+
* held forever, and the runs behind it are unreachable by anything except
|
|
469
|
+
* a human with a shell. The expiry is what makes a dead holder's work
|
|
470
|
+
* recoverable without one.
|
|
471
|
+
*
|
|
472
|
+
* The expiry is also why a fence exists. A holder does not know it has
|
|
473
|
+
* expired — a long pause, a suspended container and a partition all look
|
|
474
|
+
* from the inside like time not passing — so it wakes and writes as
|
|
475
|
+
* though it still holds. Liveness cannot be checked from here. What CAN
|
|
476
|
+
* be checked, at the write, is whether the holding that write belongs to
|
|
477
|
+
* is still the current one, and that is a comparison of two numbers.
|
|
478
|
+
*
|
|
479
|
+
* ### Reclaiming
|
|
480
|
+
*
|
|
481
|
+
* Calling this on a run whose claim has expired SUCCEEDS and mints a
|
|
482
|
+
* fence strictly greater than the expired holding's. The previous holder
|
|
483
|
+
* is not notified — it cannot be, that is the premise — it simply stops
|
|
484
|
+
* being able to write.
|
|
485
|
+
*
|
|
486
|
+
* Calling it again as the CURRENT holder also succeeds and extends the
|
|
487
|
+
* lease, minting a new fence. Renewal and reclamation are the same
|
|
488
|
+
* operation from the store's side, which is why there is no separate
|
|
489
|
+
* `renew`: two code paths that must agree about who holds a run is one
|
|
490
|
+
* more than can be kept correct.
|
|
491
|
+
*
|
|
492
|
+
* ### What a backend implementing this MUST guarantee
|
|
493
|
+
*
|
|
494
|
+
* These are the properties the fence comparison depends on. None of them
|
|
495
|
+
* is checkable from here, and every one of them was violated by the first
|
|
496
|
+
* built-in implementation, so they are written down rather than assumed:
|
|
497
|
+
*
|
|
498
|
+
* 1. **A fence exceeds every fence ever issued for the run** — including
|
|
499
|
+
* across a release, and across deletion of whatever recorded it. A
|
|
500
|
+
* counter that rewinds re-issues a number a stalled worker still
|
|
501
|
+
* believes it holds, and that worker's writes become legal again.
|
|
502
|
+
* 2. **Fences are unique.** The write check is `fence < current`, so two
|
|
503
|
+
* holders at one number are both admitted. Equality is permissive
|
|
504
|
+
* here, which makes a duplicate worse than a gap.
|
|
505
|
+
* 3. **The fence check is atomic with the write.** Reading the current
|
|
506
|
+
* fence and then writing is check-then-act, and the gap is a race. A
|
|
507
|
+
* database gets this free (`UPDATE … WHERE fence >= ?`); a filesystem
|
|
508
|
+
* does not, and the built-in disk store narrows rather than closes it.
|
|
509
|
+
* 4. **`holder` is unique per process.** It is evidence rather than
|
|
510
|
+
* authority, but it is the only thing distinguishing a RENEWAL from a
|
|
511
|
+
* theft — two workers sharing a holder string take a live, unexpired
|
|
512
|
+
* claim from each other instantly. Use something per-process, not a
|
|
513
|
+
* per-deployment name.
|
|
514
|
+
*/
|
|
515
|
+
claimRun?(scope: CheckpointRunScope, options: ClaimRunOptions): Promise<RunClaim | null>
|
|
516
|
+
|
|
517
|
+
/**
|
|
518
|
+
* Give a claim up early. Idempotent: releasing a claim that already
|
|
519
|
+
* expired, was superseded, or never existed succeeds as a no-op.
|
|
520
|
+
*
|
|
521
|
+
* Presenting a stale fence releases NOTHING — a worker that stalled past
|
|
522
|
+
* its lease must not be able to hand away a run somebody else is now
|
|
523
|
+
* holding, and that is the same fencing comparison the write path makes.
|
|
524
|
+
*
|
|
525
|
+
* Optional to call, not optional to matter: a worker that finishes and
|
|
526
|
+
* releases returns the run to the queue immediately, where one that just
|
|
527
|
+
* exits leaves it stuck until the lease expires. That is a latency
|
|
528
|
+
* difference, never a correctness one, which is the property that lets a
|
|
529
|
+
* crashed worker be indistinguishable from a slow one.
|
|
530
|
+
*/
|
|
531
|
+
releaseRun?(scope: CheckpointRunScope, fence: ClaimFence): Promise<void>
|
|
345
532
|
}
|
package/src/types/run/config.ts
CHANGED
|
@@ -136,6 +136,19 @@ export interface RunPersistenceConfig {
|
|
|
136
136
|
* hosts inject a scope-keyed backend (e.g. Postgres) here.
|
|
137
137
|
*/
|
|
138
138
|
checkpointStore?: CheckpointStore
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Optional run-evidence persistence override. Defaults to the disk layout
|
|
142
|
+
* under `outputDir` (a
|
|
143
|
+
* {@link import('../../store/run/disk.js').RunDiskStore}); hosts inject
|
|
144
|
+
* their own backend here.
|
|
145
|
+
*
|
|
146
|
+
* The sibling of `checkpointStore`, and it should have been one from the
|
|
147
|
+
* start: checkpoints got an injectable seam while the run record, its
|
|
148
|
+
* messages, its transcript and its report did not, so the evidence was
|
|
149
|
+
* the one part of a run that could not leave the local filesystem.
|
|
150
|
+
*/
|
|
151
|
+
runStore?: import('./store.js').RunStore
|
|
139
152
|
}
|
|
140
153
|
|
|
141
154
|
export interface RunStoreConfig {
|
package/src/types/run/index.ts
CHANGED
|
@@ -3,6 +3,7 @@ export * from './prepare-step.js'
|
|
|
3
3
|
export * from './stop-reason.js'
|
|
4
4
|
export * from './config.js'
|
|
5
5
|
export * from './checkpoint-store.js'
|
|
6
|
+
export * from './store.js'
|
|
6
7
|
export * from './entity.js'
|
|
7
8
|
export * from './replay.js'
|
|
8
9
|
// Domain `RunStatus` (session-hierarchy.md §4.6 state machine). Safe to
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RunStore — persistence contract for a run's own evidence.
|
|
3
|
+
*
|
|
4
|
+
* The checkpoint store got an injectable seam and this did not, which left
|
|
5
|
+
* the run record, its messages, its transcript and its report reachable only
|
|
6
|
+
* through a concrete filesystem class. For a kernel whose stated purpose is
|
|
7
|
+
* auditable evidence, the evidence was the one thing that could not be
|
|
8
|
+
* pointed at durable storage: on ephemeral infrastructure the transcript dies
|
|
9
|
+
* with the container, and behind a load balancer two replicas write two
|
|
10
|
+
* disjoint run trees for one tenant.
|
|
11
|
+
*
|
|
12
|
+
* The location was already injectable through a path builder — but that
|
|
13
|
+
* returns filesystem path strings, so it relocates the directory without
|
|
14
|
+
* changing the medium.
|
|
15
|
+
*
|
|
16
|
+
* ## Bound to one run, unlike {@link CheckpointStore}
|
|
17
|
+
*
|
|
18
|
+
* Every accessor here addresses the run the store was bound to by
|
|
19
|
+
* {@link RunStore.initRun}, where a `CheckpointStore` takes an explicit scope
|
|
20
|
+
* per call. That asymmetry is inherited rather than chosen: this contract is
|
|
21
|
+
* extracted from a class the runtime already constructs per run and holds for
|
|
22
|
+
* the run's lifetime, and re-keying it would change every call site in the
|
|
23
|
+
* same change that introduces the seam — two risks where one will do.
|
|
24
|
+
*
|
|
25
|
+
* A host implementing a shared backend therefore keys its rows by the
|
|
26
|
+
* attribution it was constructed with plus the bound run id. If this is later
|
|
27
|
+
* re-keyed per call, it happens once, deliberately, as its own change.
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
import type { Run } from './entity.js'
|
|
31
|
+
import type { RunEvent } from './events.js'
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* One finished tool call, recovered from the run's own transcript.
|
|
35
|
+
*
|
|
36
|
+
* Re-declared here rather than imported from the disk store so the contract
|
|
37
|
+
* does not depend on an implementation of itself.
|
|
38
|
+
*/
|
|
39
|
+
export interface CompletedToolRecord {
|
|
40
|
+
readonly toolUseId: string
|
|
41
|
+
readonly toolName: string
|
|
42
|
+
readonly result: string
|
|
43
|
+
readonly isError: boolean
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export interface RunStore {
|
|
47
|
+
/**
|
|
48
|
+
* Bind this store to a run, before any other call.
|
|
49
|
+
*
|
|
50
|
+
* Returns a location when the backend has one — the built-in disk store
|
|
51
|
+
* returns the run's directory — and `null` when it does not. A caller
|
|
52
|
+
* that renders the value must treat `null` as "this run is not on a
|
|
53
|
+
* filesystem" rather than as an error: an in-memory or object-storage
|
|
54
|
+
* backend has nothing to print, and inventing a path for it would put a
|
|
55
|
+
* directory that does not exist in front of an operator.
|
|
56
|
+
*/
|
|
57
|
+
initRun(runId: string, parentRunId?: string): Promise<string | null>
|
|
58
|
+
|
|
59
|
+
/** Persist the run record: status, metadata, usage, timings. */
|
|
60
|
+
writeRunMeta(run: Run): Promise<void>
|
|
61
|
+
|
|
62
|
+
/** Persist the run's full message history. */
|
|
63
|
+
writeMessages(run: Run): Promise<void>
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Append one event to the run's durable event log.
|
|
67
|
+
*
|
|
68
|
+
* High-frequency streaming deltas are excluded before they reach here —
|
|
69
|
+
* that exclusion is a deliberate trade and belongs to the emitter, not to
|
|
70
|
+
* the backend, so a store must not re-filter.
|
|
71
|
+
*/
|
|
72
|
+
appendEvent(event: RunEvent): Promise<void>
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Persist the run's final report. Returns a location, or `null` when the
|
|
76
|
+
* backend has none. See {@link RunStore.initRun}.
|
|
77
|
+
*/
|
|
78
|
+
writeReport(content: string): Promise<string | null>
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Every tool call this run has already finished, keyed by `toolUseId`.
|
|
82
|
+
*
|
|
83
|
+
* A batch's results reach the message history only once the WHOLE batch
|
|
84
|
+
* settles, so a hard kill part-way through loses every result that had
|
|
85
|
+
* already come back, and the resumed run re-executes those calls. For a
|
|
86
|
+
* file write that is waste; for a payment or an email it is a second one.
|
|
87
|
+
*
|
|
88
|
+
* A backend that does not retain individual events answers with an empty
|
|
89
|
+
* map, which costs re-execution and is honest. It must not answer with a
|
|
90
|
+
* PARTIAL map: a caller reads a present entry as "this call is already
|
|
91
|
+
* answered", so a half-remembered batch is worse than a forgotten one.
|
|
92
|
+
*/
|
|
93
|
+
readCompletedTools(): Promise<Map<string, CompletedToolRecord>>
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Where this run's evidence lives, or `null` when it is not on a
|
|
97
|
+
* filesystem. Valid only after {@link RunStore.initRun}.
|
|
98
|
+
*/
|
|
99
|
+
getRunDir(): string | null
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Record the run in a browsable catalogue of runs. OPTIONAL.
|
|
103
|
+
*
|
|
104
|
+
* Optional because it is the one method here that is not evidence: it
|
|
105
|
+
* maintains a convenience listing for a human reading the directory, and
|
|
106
|
+
* a backend whose runs are already queryable has nothing to add. The
|
|
107
|
+
* programmatic answer to "which runs are there" is
|
|
108
|
+
* `CheckpointStore.listDurableRuns`, which carries attribution and
|
|
109
|
+
* includes sub-runs; this does neither.
|
|
110
|
+
*/
|
|
111
|
+
addToIndex?(run: Run): Promise<void>
|
|
112
|
+
}
|