@namzu/sdk 20.3.0 → 21.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +321 -0
- package/dist/bridge/sse/mapper.d.ts +19 -0
- package/dist/bridge/sse/mapper.d.ts.map +1 -1
- package/dist/bridge/sse/mapper.js +9 -1
- package/dist/bridge/sse/mapper.js.map +1 -1
- package/dist/contracts/schemas.d.ts +10 -10
- package/dist/manager/run/persistence.d.ts +26 -0
- package/dist/manager/run/persistence.d.ts.map +1 -1
- package/dist/manager/run/persistence.js +40 -0
- package/dist/manager/run/persistence.js.map +1 -1
- package/dist/public-runtime.d.ts +5 -0
- package/dist/public-runtime.d.ts.map +1 -1
- package/dist/public-runtime.js +21 -0
- package/dist/public-runtime.js.map +1 -1
- package/dist/run/drain.d.ts +194 -0
- package/dist/run/drain.d.ts.map +1 -0
- package/dist/run/drain.js +228 -0
- package/dist/run/drain.js.map +1 -0
- package/dist/run/index.d.ts +2 -0
- package/dist/run/index.d.ts.map +1 -1
- package/dist/run/index.js +1 -0
- package/dist/run/index.js.map +1 -1
- package/dist/runtime/query/checkpoint.d.ts +27 -1
- package/dist/runtime/query/checkpoint.d.ts.map +1 -1
- package/dist/runtime/query/checkpoint.js +34 -4
- package/dist/runtime/query/checkpoint.js.map +1 -1
- package/dist/runtime/query/events.d.ts +12 -0
- package/dist/runtime/query/events.d.ts.map +1 -1
- package/dist/runtime/query/events.js +61 -3
- package/dist/runtime/query/events.js.map +1 -1
- package/dist/runtime/query/index.d.ts +57 -1
- package/dist/runtime/query/index.d.ts.map +1 -1
- package/dist/runtime/query/index.js +36 -0
- package/dist/runtime/query/index.js.map +1 -1
- package/dist/runtime/query/resume-run.d.ts +34 -1
- package/dist/runtime/query/resume-run.d.ts.map +1 -1
- package/dist/runtime/query/resume-run.js +16 -3
- package/dist/runtime/query/resume-run.js.map +1 -1
- package/dist/store/index.d.ts +2 -2
- package/dist/store/index.d.ts.map +1 -1
- package/dist/store/index.js +2 -2
- package/dist/store/index.js.map +1 -1
- package/dist/store/run/checkpoint-disk.d.ts +18 -2
- package/dist/store/run/checkpoint-disk.d.ts.map +1 -1
- package/dist/store/run/checkpoint-disk.js +50 -5
- package/dist/store/run/checkpoint-disk.js.map +1 -1
- package/dist/store/run/checkpoint-memory.d.ts +22 -2
- package/dist/store/run/checkpoint-memory.d.ts.map +1 -1
- package/dist/store/run/checkpoint-memory.js +99 -4
- package/dist/store/run/checkpoint-memory.js.map +1 -1
- package/dist/store/run/claim-disk.d.ts +130 -0
- package/dist/store/run/claim-disk.d.ts.map +1 -0
- package/dist/store/run/claim-disk.js +550 -0
- package/dist/store/run/claim-disk.js.map +1 -0
- package/dist/store/run/conformance.d.ts +159 -0
- package/dist/store/run/conformance.d.ts.map +1 -0
- package/dist/store/run/conformance.js +451 -0
- package/dist/store/run/conformance.js.map +1 -0
- package/dist/store/run/disk.d.ts +30 -2
- package/dist/store/run/disk.d.ts.map +1 -1
- package/dist/store/run/disk.js +101 -0
- package/dist/store/run/disk.js.map +1 -1
- package/dist/store/run/listing.d.ts +44 -1
- package/dist/store/run/listing.d.ts.map +1 -1
- package/dist/store/run/listing.js +92 -1
- package/dist/store/run/listing.js.map +1 -1
- package/dist/store/run/memory.d.ts +5 -4
- package/dist/store/run/memory.d.ts.map +1 -1
- package/dist/store/run/memory.js +30 -1
- package/dist/store/run/memory.js.map +1 -1
- package/dist/types/doctor/check.d.ts +28 -2
- package/dist/types/doctor/check.d.ts.map +1 -1
- package/dist/types/run/checkpoint-store.d.ts +178 -2
- package/dist/types/run/checkpoint-store.d.ts.map +1 -1
- package/dist/types/run/event-cursor.d.ts +90 -0
- package/dist/types/run/event-cursor.d.ts.map +1 -0
- package/dist/types/run/event-cursor.js +47 -0
- package/dist/types/run/event-cursor.js.map +1 -0
- package/dist/types/run/events.d.ts +71 -1
- package/dist/types/run/events.d.ts.map +1 -1
- package/dist/types/run/events.js.map +1 -1
- package/dist/types/run/index.d.ts +1 -0
- package/dist/types/run/index.d.ts.map +1 -1
- package/dist/types/run/index.js +1 -0
- package/dist/types/run/index.js.map +1 -1
- package/dist/types/run/store.d.ts +41 -1
- package/dist/types/run/store.d.ts.map +1 -1
- package/package.json +6 -1
- package/src/bridge/sse/mapper.ts +28 -1
- package/src/manager/run/persistence.ts +43 -0
- package/src/public-runtime.ts +27 -0
- package/src/run/drain.ts +393 -0
- package/src/run/index.ts +3 -0
- package/src/runtime/query/checkpoint.ts +42 -5
- package/src/runtime/query/events.ts +68 -4
- package/src/runtime/query/index.ts +113 -1
- package/src/runtime/query/resume-run.ts +77 -12
- package/src/store/index.ts +5 -1
- package/src/store/run/checkpoint-disk.ts +70 -5
- package/src/store/run/checkpoint-memory.ts +118 -3
- package/src/store/run/claim-disk.ts +593 -0
- package/src/store/run/conformance.ts +705 -0
- package/src/store/run/disk.ts +106 -2
- package/src/store/run/listing.ts +116 -1
- package/src/store/run/memory.ts +35 -5
- package/src/types/doctor/check.ts +28 -2
- package/src/types/run/checkpoint-store.ts +189 -2
- package/src/types/run/event-cursor.ts +118 -0
- package/src/types/run/events.ts +80 -3
- package/src/types/run/index.ts +1 -0
- package/src/types/run/store.ts +43 -1
package/src/store/run/disk.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { appendFile, mkdir, readFile, readdir, unlink } from 'node:fs/promises'
|
|
2
2
|
import { join } from 'node:path'
|
|
3
3
|
import type { CheckpointId, IterationCheckpoint } from '../../types/hitl/index.js'
|
|
4
|
-
import type { Run, RunEvent, RunStoreConfig } from '../../types/run/index.js'
|
|
5
|
-
import type { CompletedToolRecord, RunStore } from '../../types/run/store.js'
|
|
4
|
+
import type { PersistedRunEvent, Run, RunEvent, RunStoreConfig } from '../../types/run/index.js'
|
|
5
|
+
import type { CompletedToolRecord, ReadRunEventsOptions, RunStore } from '../../types/run/store.js'
|
|
6
6
|
import { atomicWriteFile } from '../../utils/atomic-write.js'
|
|
7
7
|
import { type Logger, getRootLogger } from '../../utils/logger.js'
|
|
8
8
|
import { defineSchema, migrate, stamp } from '../schema.js'
|
|
@@ -51,6 +51,7 @@ export class RunDiskStore implements RunStore {
|
|
|
51
51
|
this.runDir = join(this.baseDir, runId)
|
|
52
52
|
}
|
|
53
53
|
await mkdir(this.runDir, { recursive: true })
|
|
54
|
+
await healTornTranscript(this.runDir)
|
|
54
55
|
this.log.info(`Run directory created: ${this.runDir}`)
|
|
55
56
|
return this.runDir
|
|
56
57
|
}
|
|
@@ -66,6 +67,10 @@ export class RunDiskStore implements RunStore {
|
|
|
66
67
|
await appendFile(join(dir, 'transcript.jsonl'), line, 'utf-8')
|
|
67
68
|
}
|
|
68
69
|
|
|
70
|
+
async readEvents(options?: ReadRunEventsOptions): Promise<readonly PersistedRunEvent[]> {
|
|
71
|
+
return readRunEventsIn(this.requireInit(), options)
|
|
72
|
+
}
|
|
73
|
+
|
|
69
74
|
/**
|
|
70
75
|
* Every tool call this run has already finished, keyed by `toolUseId`.
|
|
71
76
|
*
|
|
@@ -296,6 +301,105 @@ export class RunDiskStore implements RunStore {
|
|
|
296
301
|
}
|
|
297
302
|
}
|
|
298
303
|
|
|
304
|
+
/**
|
|
305
|
+
* Every durable event under one run directory, oldest first.
|
|
306
|
+
*
|
|
307
|
+
* A free function for the same reason {@link readCheckpointsIn} is one: a
|
|
308
|
+
* caller catching up on a run this process never started would otherwise have
|
|
309
|
+
* to bind a {@link RunDiskStore} to read it, and binding one CREATES the
|
|
310
|
+
* directory. A read that mints an empty run directory then answers "no events"
|
|
311
|
+
* is indistinguishable from a run that genuinely has none.
|
|
312
|
+
*
|
|
313
|
+
* ## Unsequenced lines take their position
|
|
314
|
+
*
|
|
315
|
+
* A transcript written before events were numbered carries no `seq` at all.
|
|
316
|
+
* Numbering those lines by their 1-based position is what keeps their evidence
|
|
317
|
+
* reachable: a legacy run of five lines reads back as 1..5, seeds the emitter
|
|
318
|
+
* at 5, and its next event is 6 — continuous, and stable on every later read.
|
|
319
|
+
* Skipping them instead would erase a run's whole history from a catch-up, and
|
|
320
|
+
* synthesising nothing at all would put the emitter back at 1 on top of a log
|
|
321
|
+
* that already has five entries.
|
|
322
|
+
*
|
|
323
|
+
* A damaged line is skipped rather than refused, which is the one place this
|
|
324
|
+
* differs from the checkpoint reader next door, and deliberately: a checkpoint
|
|
325
|
+
* is read to RESUME from, so a damaged one must stop the resume, while the
|
|
326
|
+
* transcript is read to REPORT from, and dropping every event after a torn line
|
|
327
|
+
* would be a larger loss than the torn line itself. The position count still
|
|
328
|
+
* advances over it, so the numbering of the events after it is unchanged.
|
|
329
|
+
*/
|
|
330
|
+
export async function readRunEventsIn(
|
|
331
|
+
runDir: string,
|
|
332
|
+
options?: ReadRunEventsOptions,
|
|
333
|
+
): Promise<readonly PersistedRunEvent[]> {
|
|
334
|
+
let raw: string
|
|
335
|
+
try {
|
|
336
|
+
raw = await readFile(join(runDir, 'transcript.jsonl'), 'utf-8')
|
|
337
|
+
} catch (err) {
|
|
338
|
+
if (isFileNotFound(err)) return []
|
|
339
|
+
throw err
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
const sinceSeq = options?.sinceSeq ?? 0
|
|
343
|
+
const events: PersistedRunEvent[] = []
|
|
344
|
+
let position = 0
|
|
345
|
+
|
|
346
|
+
for (const line of raw.split('\n')) {
|
|
347
|
+
if (line.length === 0) continue
|
|
348
|
+
position += 1
|
|
349
|
+
|
|
350
|
+
let parsed: Record<string, unknown>
|
|
351
|
+
try {
|
|
352
|
+
parsed = JSON.parse(line) as Record<string, unknown>
|
|
353
|
+
} catch {
|
|
354
|
+
continue
|
|
355
|
+
}
|
|
356
|
+
if (parsed === null || typeof parsed !== 'object' || typeof parsed.type !== 'string') continue
|
|
357
|
+
|
|
358
|
+
const seq = typeof parsed.seq === 'number' ? parsed.seq : position
|
|
359
|
+
if (seq <= sinceSeq) continue
|
|
360
|
+
|
|
361
|
+
events.push({
|
|
362
|
+
...parsed,
|
|
363
|
+
seq,
|
|
364
|
+
// Stamped by `appendEvent` since long before it was declared. A line
|
|
365
|
+
// that predates even that gets the only honest answer available:
|
|
366
|
+
// zero, which sorts before every real moment and cannot be mistaken
|
|
367
|
+
// for one.
|
|
368
|
+
timestamp: typeof parsed.timestamp === 'number' ? parsed.timestamp : 0,
|
|
369
|
+
} as unknown as PersistedRunEvent)
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
return events
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
/**
|
|
376
|
+
* Terminate a transcript whose last line was cut off mid-write.
|
|
377
|
+
*
|
|
378
|
+
* A process killed during `appendFile` leaves a fragment with no newline. The
|
|
379
|
+
* next append lands on the same line, so the fragment and a WHOLE, correct
|
|
380
|
+
* event merge into one unparsable line — and the reader skips it. The event was
|
|
381
|
+
* written, the emitter counted it as durable, and it is gone.
|
|
382
|
+
*
|
|
383
|
+
* Ending the fragment is enough. It stays unreadable and is skipped as it
|
|
384
|
+
* always was; everything appended after it survives, which is the difference
|
|
385
|
+
* between losing one event and losing one event plus the next.
|
|
386
|
+
*
|
|
387
|
+
* Called from `initRun`, which is the only moment the store knows nothing is
|
|
388
|
+
* mid-write.
|
|
389
|
+
*/
|
|
390
|
+
async function healTornTranscript(runDir: string): Promise<void> {
|
|
391
|
+
const path = join(runDir, 'transcript.jsonl')
|
|
392
|
+
let raw: string
|
|
393
|
+
try {
|
|
394
|
+
raw = await readFile(path, 'utf-8')
|
|
395
|
+
} catch (err) {
|
|
396
|
+
if (isFileNotFound(err)) return
|
|
397
|
+
throw err
|
|
398
|
+
}
|
|
399
|
+
if (raw.length === 0 || raw.endsWith('\n')) return
|
|
400
|
+
await appendFile(path, '\n', 'utf-8')
|
|
401
|
+
}
|
|
402
|
+
|
|
299
403
|
/**
|
|
300
404
|
* Every checkpoint stored under one run directory, ascending by `createdAt`.
|
|
301
405
|
*
|
package/src/store/run/listing.ts
CHANGED
|
@@ -14,12 +14,16 @@ import type {
|
|
|
14
14
|
CheckpointListingScope,
|
|
15
15
|
CheckpointRunScope,
|
|
16
16
|
CheckpointStore,
|
|
17
|
+
ClaimFence,
|
|
18
|
+
ClaimRunOptions,
|
|
19
|
+
ClaimSummary,
|
|
17
20
|
DurableRunEntry,
|
|
18
21
|
DurableRunOrder,
|
|
19
22
|
DurableRunPage,
|
|
20
23
|
ListDurableRunsOptions,
|
|
21
24
|
ParkState,
|
|
22
25
|
ParkSummary,
|
|
26
|
+
RunClaim,
|
|
23
27
|
} from '../../types/run/checkpoint-store.js'
|
|
24
28
|
|
|
25
29
|
/** Page size when the caller names none. */
|
|
@@ -186,11 +190,20 @@ export function paginateDurableRuns(
|
|
|
186
190
|
options?: ListDurableRunsOptions,
|
|
187
191
|
): DurableRunPage {
|
|
188
192
|
const wanted = options?.park
|
|
189
|
-
const
|
|
193
|
+
const byPark =
|
|
190
194
|
wanted && wanted.length > 0
|
|
191
195
|
? entries.filter((e) => e.park !== undefined && wanted.includes(e.park.state))
|
|
192
196
|
: entries
|
|
193
197
|
|
|
198
|
+
// An expired claim counts as unheld. That is what expiry means, and a
|
|
199
|
+
// queue reader that treated an expired claim as held would leave a dead
|
|
200
|
+
// worker's runs invisible forever — the exact failure a lease exists to
|
|
201
|
+
// prevent, reintroduced by the filter that reads it.
|
|
202
|
+
const filtered =
|
|
203
|
+
options?.claimed === undefined
|
|
204
|
+
? byPark
|
|
205
|
+
: byPark.filter((e) => (e.claim !== undefined && !e.claim.expired) === options.claimed)
|
|
206
|
+
|
|
194
207
|
// Both orders sort on a key that cannot move under a paging caller — see
|
|
195
208
|
// the contract comment on `listDurableRuns`.
|
|
196
209
|
const orderBy = options?.orderBy ?? 'runId'
|
|
@@ -305,3 +318,105 @@ export async function listDurableRuns(
|
|
|
305
318
|
assertContiguousListingScope(scope, 'listDurableRuns')
|
|
306
319
|
return store.listDurableRuns(scope, options)
|
|
307
320
|
}
|
|
321
|
+
|
|
322
|
+
/**
|
|
323
|
+
* Take working possession of a run, refusing when the store cannot arbitrate.
|
|
324
|
+
*
|
|
325
|
+
* The refusal is the entire safety property. `claimRun` is optional on the
|
|
326
|
+
* contract, and the natural way to reach an absent optional method is to skip
|
|
327
|
+
* it — which here means every worker proceeds, believing it holds a run
|
|
328
|
+
* nobody arbitrated. Two workers then restore the same checkpoint, both run
|
|
329
|
+
* the tools, and both write under one run id; half the work vanishes with no
|
|
330
|
+
* error anywhere.
|
|
331
|
+
*
|
|
332
|
+
* So a store with no claim support does not get "claimed by default". It gets
|
|
333
|
+
* an error naming the deployment shape it cannot support. A single-writer
|
|
334
|
+
* host never calls this and is unaffected.
|
|
335
|
+
*
|
|
336
|
+
* Returns `null` — not an error — when another holder has the run. That is
|
|
337
|
+
* the ordinary outcome of two readers on one queue, and a caller loops to the
|
|
338
|
+
* next run rather than handling a fault.
|
|
339
|
+
*/
|
|
340
|
+
export async function claimRun(
|
|
341
|
+
store: CheckpointStore,
|
|
342
|
+
scope: CheckpointRunScope,
|
|
343
|
+
options: ClaimRunOptions,
|
|
344
|
+
): Promise<RunClaim | null> {
|
|
345
|
+
if (typeof store.claimRun !== 'function') {
|
|
346
|
+
throw new NamzuError({
|
|
347
|
+
code: 'capability_unavailable',
|
|
348
|
+
message:
|
|
349
|
+
'claimRun: the injected checkpoint store does not implement `claimRun`, so it cannot arbitrate between two workers taking the same run. Refusing rather than proceeding unclaimed — proceeding would let two workers restore one checkpoint, both execute its tools, and both write under one run id, which loses half the work and reports nothing. Supply a store that implements it, or run a single writer per run.',
|
|
350
|
+
details: { runId: scope.runId },
|
|
351
|
+
})
|
|
352
|
+
}
|
|
353
|
+
if (!Number.isFinite(options.ttlMs) || options.ttlMs <= 0) {
|
|
354
|
+
throw new NamzuError({
|
|
355
|
+
code: 'invalid_config',
|
|
356
|
+
message: `claimRun: ttlMs must be a positive number of milliseconds, got ${String(options.ttlMs)}. A lease that expires immediately is a lease every worker can take at once, which is the condition this call exists to prevent.`,
|
|
357
|
+
details: { runId: scope.runId, ttlMs: options.ttlMs },
|
|
358
|
+
})
|
|
359
|
+
}
|
|
360
|
+
return store.claimRun(scope, options)
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/**
|
|
364
|
+
* Give a claim up early, refusing when the store cannot arbitrate.
|
|
365
|
+
*
|
|
366
|
+
* Refuses for the same reason as {@link claimRun}: a host that believes it is
|
|
367
|
+
* releasing a claim on a store that has none is a host that believes the
|
|
368
|
+
* whole mechanism is running.
|
|
369
|
+
*/
|
|
370
|
+
export async function releaseRun(
|
|
371
|
+
store: CheckpointStore,
|
|
372
|
+
scope: CheckpointRunScope,
|
|
373
|
+
fence: ClaimFence,
|
|
374
|
+
): Promise<void> {
|
|
375
|
+
if (typeof store.releaseRun !== 'function') {
|
|
376
|
+
throw new NamzuError({
|
|
377
|
+
code: 'capability_unavailable',
|
|
378
|
+
message:
|
|
379
|
+
'releaseRun: the injected checkpoint store does not implement `releaseRun`. A release that silently does nothing would leave the run held until its lease expires while the caller believes it is back on the queue.',
|
|
380
|
+
details: { runId: scope.runId },
|
|
381
|
+
})
|
|
382
|
+
}
|
|
383
|
+
return store.releaseRun(scope, fence)
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
/**
|
|
387
|
+
* The refusal a store raises when a write presents a superseded fence.
|
|
388
|
+
*
|
|
389
|
+
* Shared so both shipped stores say the same thing, and so a host writing its
|
|
390
|
+
* own backend raises something a caller can branch on rather than a message
|
|
391
|
+
* string. This is the moment a stalled worker learns it lost the run — the
|
|
392
|
+
* only moment it CAN learn, since from the inside a pause and a partition
|
|
393
|
+
* both look like time not passing.
|
|
394
|
+
*/
|
|
395
|
+
export function fencedOut(
|
|
396
|
+
scope: CheckpointRunScope,
|
|
397
|
+
presented: number,
|
|
398
|
+
current: number,
|
|
399
|
+
): NamzuError {
|
|
400
|
+
return new NamzuError({
|
|
401
|
+
code: 'storage_error',
|
|
402
|
+
message: `writeCheckpoint: refusing a write for run ${scope.runId} fenced at ${presented} — the run is now claimed at ${current}. Another worker took this run over, so this process no longer holds it and its work is not the record. Stop the run rather than retrying: the claim is gone, not busy.`,
|
|
403
|
+
details: { runId: scope.runId, presentedFence: presented, currentFence: current },
|
|
404
|
+
retryable: false,
|
|
405
|
+
})
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/**
|
|
409
|
+
* Whether a recorded claim still holds at `now`, and the summary a listing
|
|
410
|
+
* reports for it.
|
|
411
|
+
*/
|
|
412
|
+
export function toClaimSummary(claim: RunClaim, now: number): ClaimSummary {
|
|
413
|
+
return {
|
|
414
|
+
holder: claim.holder,
|
|
415
|
+
fence: claim.fence,
|
|
416
|
+
expiresAt: claim.expiresAt,
|
|
417
|
+
// Judged here, once, against the store's clock. Left to the caller it
|
|
418
|
+
// would be judged against a different one, and a single page could
|
|
419
|
+
// then disagree with itself about which rows are available.
|
|
420
|
+
expired: now >= claim.expiresAt,
|
|
421
|
+
}
|
|
422
|
+
}
|
package/src/store/run/memory.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { Run } from '../../types/run/entity.js'
|
|
2
|
-
import type { RunEvent } from '../../types/run/events.js'
|
|
3
|
-
import type { CompletedToolRecord, RunStore } from '../../types/run/store.js'
|
|
2
|
+
import type { PersistedRunEvent, RunEvent } from '../../types/run/events.js'
|
|
3
|
+
import type { CompletedToolRecord, ReadRunEventsOptions, RunStore } from '../../types/run/store.js'
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Process-local {@link RunStore}: a run's evidence with no filesystem.
|
|
@@ -22,9 +22,21 @@ export class InMemoryRunStore implements RunStore {
|
|
|
22
22
|
private meta: Run | null = null
|
|
23
23
|
private messages: Run['messages'] = []
|
|
24
24
|
private report: string | null = null
|
|
25
|
-
private
|
|
25
|
+
private events: PersistedRunEvent[] = []
|
|
26
26
|
|
|
27
27
|
async initRun(runId: string, parentRunId?: string): Promise<string | null> {
|
|
28
|
+
// Rebinding to a DIFFERENT run starts that run's evidence empty. The disk
|
|
29
|
+
// store gets this for free — a different id is a different directory —
|
|
30
|
+
// and this one has to say it, because one instance reused for a replay
|
|
31
|
+
// fork otherwise reports the origin run's events, its messages and its
|
|
32
|
+
// report as the new run's own. Evidence attributed to the wrong run is
|
|
33
|
+
// worse than none: it is wrong and it looks right.
|
|
34
|
+
if (this.runId !== null && this.runId !== runId) {
|
|
35
|
+
this.meta = null
|
|
36
|
+
this.messages = []
|
|
37
|
+
this.report = null
|
|
38
|
+
this.events = []
|
|
39
|
+
}
|
|
28
40
|
this.runId = runId
|
|
29
41
|
this.parentRunId = parentRunId
|
|
30
42
|
// No location, and that is the honest answer rather than a defect.
|
|
@@ -68,7 +80,25 @@ export class InMemoryRunStore implements RunStore {
|
|
|
68
80
|
// line — a parity test compares the two read-backs, and a timestamp
|
|
69
81
|
// present in one medium and absent in the other would make identical
|
|
70
82
|
// runs look different depending on where they were recorded.
|
|
71
|
-
|
|
83
|
+
//
|
|
84
|
+
// An unsequenced event takes its position in the log, which is the same
|
|
85
|
+
// rule the disk reader applies to a line written before events were
|
|
86
|
+
// numbered. Nothing in the kernel appends unsequenced today; the rule is
|
|
87
|
+
// here so the two backends cannot answer differently if something does.
|
|
88
|
+
this.events.push({
|
|
89
|
+
...event,
|
|
90
|
+
seq: event.seq ?? this.events.length + 1,
|
|
91
|
+
timestamp: Date.now(),
|
|
92
|
+
} as unknown as PersistedRunEvent)
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
async readEvents(options?: ReadRunEventsOptions): Promise<readonly PersistedRunEvent[]> {
|
|
96
|
+
this.requireInit()
|
|
97
|
+
const sinceSeq = options?.sinceSeq ?? 0
|
|
98
|
+
// Copied, not sliced by reference, for the same reason `writeRunMeta`
|
|
99
|
+
// clones: a caller holding the array must not be able to reach into the
|
|
100
|
+
// log through it.
|
|
101
|
+
return this.events.filter((event) => event.seq > sinceSeq).map((event) => ({ ...event }))
|
|
72
102
|
}
|
|
73
103
|
|
|
74
104
|
async writeReport(content: string): Promise<string | null> {
|
|
@@ -114,7 +144,7 @@ export class InMemoryRunStore implements RunStore {
|
|
|
114
144
|
meta: Run | null
|
|
115
145
|
messages: Run['messages']
|
|
116
146
|
report: string | null
|
|
117
|
-
events: readonly
|
|
147
|
+
events: readonly PersistedRunEvent[]
|
|
118
148
|
} {
|
|
119
149
|
return { meta: this.meta, messages: this.messages, report: this.report, events: this.events }
|
|
120
150
|
}
|
|
@@ -1,4 +1,21 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* What a check concluded.
|
|
3
|
+
*
|
|
4
|
+
* `skipped` and `inconclusive` are the two ways a check produces no verdict,
|
|
5
|
+
* and they are separate because a caller acts on them differently.
|
|
6
|
+
*
|
|
7
|
+
* - `skipped` — the check LOOKED and there was nothing here to check: an
|
|
8
|
+
* optional package is not installed, a registry the check reads has no
|
|
9
|
+
* auto-discovery to read, nothing is configured yet. A permanent or
|
|
10
|
+
* by-design absence, and an ordinary state of a healthy machine.
|
|
11
|
+
* - `inconclusive` — the check DID NOT ANSWER: it timed out, it was aborted,
|
|
12
|
+
* the thing it reads threw. Nothing is known either way, and that is itself
|
|
13
|
+
* a gap in the report worth acting on.
|
|
14
|
+
*
|
|
15
|
+
* The word used to cover both, so `namzu doctor` could not tell "healthy" from
|
|
16
|
+
* "did not manage to look", and neither could anything reading its exit code.
|
|
17
|
+
*/
|
|
18
|
+
export type DoctorStatus = 'pass' | 'fail' | 'inconclusive' | 'warn' | 'skipped'
|
|
2
19
|
|
|
3
20
|
export type DoctorCategory =
|
|
4
21
|
| 'sandbox'
|
|
@@ -47,7 +64,16 @@ export interface DoctorReport {
|
|
|
47
64
|
readonly fail: number
|
|
48
65
|
readonly inconclusive: number
|
|
49
66
|
readonly warn: number
|
|
67
|
+
readonly skipped: number
|
|
68
|
+
/** Every other count sums to this. A reader may rely on that. */
|
|
50
69
|
readonly total: number
|
|
51
70
|
}
|
|
52
|
-
|
|
71
|
+
/**
|
|
72
|
+
* `69` is sysexits `EX_UNAVAILABLE`, whose own definition ends "a catchall
|
|
73
|
+
* when something you wanted to do doesn't work, but you don't know why".
|
|
74
|
+
* That is `inconclusive`. It is not `2`, which this report already spends on
|
|
75
|
+
* "nothing was registered", and not `70`, which says the CLI is broken and
|
|
76
|
+
* is worth a bug report.
|
|
77
|
+
*/
|
|
78
|
+
readonly exit: 0 | 1 | 2 | 69 | 70
|
|
53
79
|
}
|
|
@@ -175,6 +175,17 @@ export interface DurableRunEntry extends CheckpointRunScope {
|
|
|
175
175
|
readonly latestCheckpointAt: number
|
|
176
176
|
/** Absent when the run has never parked. */
|
|
177
177
|
readonly park?: ParkSummary
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* Absent when no process has ever claimed the run.
|
|
181
|
+
*
|
|
182
|
+
* A SIBLING of {@link DurableRunEntry.park}, not a member of
|
|
183
|
+
* {@link ParkState} — see the note on that union. A park is a question put
|
|
184
|
+
* to a human; a claim is a lease held by a process. A run can have both,
|
|
185
|
+
* neither, or either, and the state a queue worker needs most is parked
|
|
186
|
+
* AND unclaimed, which one union cannot say.
|
|
187
|
+
*/
|
|
188
|
+
readonly claim?: ClaimSummary
|
|
178
189
|
}
|
|
179
190
|
|
|
180
191
|
/**
|
|
@@ -208,6 +219,65 @@ export type DurableRunOrder =
|
|
|
208
219
|
*/
|
|
209
220
|
| 'createdAt'
|
|
210
221
|
|
|
222
|
+
/**
|
|
223
|
+
* A monotonically increasing number identifying one holding of a run's claim.
|
|
224
|
+
*
|
|
225
|
+
* The load-bearing word is *fencing*. A mutex answers "may I proceed", and a
|
|
226
|
+
* holder that stalls past its lease — a long GC pause, a suspended container,
|
|
227
|
+
* a partitioned network — answers it "yes" and then writes, long after
|
|
228
|
+
* somebody else legitimately took over. A fence answers a different question
|
|
229
|
+
* at the moment of the WRITE: "is the holding I belong to still the current
|
|
230
|
+
* one". Every claim of a run mints a number strictly greater than the last,
|
|
231
|
+
* so a store can reject a write from a superseded holder without knowing
|
|
232
|
+
* anything about processes, clocks or liveness.
|
|
233
|
+
*
|
|
234
|
+
* Not a random token, deliberately: randomness proves identity and cannot
|
|
235
|
+
* establish *order*, and order is the entire mechanism.
|
|
236
|
+
*/
|
|
237
|
+
export type ClaimFence = number
|
|
238
|
+
|
|
239
|
+
/** A holding of a run's claim, as issued to the process that took it. */
|
|
240
|
+
export interface RunClaim {
|
|
241
|
+
/** Opaque caller-supplied identity — a worker id, a pod name. Evidence, not authority. */
|
|
242
|
+
readonly holder: string
|
|
243
|
+
/** See {@link ClaimFence}. Present it on every durable write. */
|
|
244
|
+
readonly fence: ClaimFence
|
|
245
|
+
/**
|
|
246
|
+
* Absolute epoch ms after which the claim may be taken by somebody else.
|
|
247
|
+
*
|
|
248
|
+
* Absolute rather than a duration for the same reason a park's deadline
|
|
249
|
+
* is: it has to survive the process that set it. A duration plus an
|
|
250
|
+
* in-process timer cannot — the holder is the thing that dies.
|
|
251
|
+
*/
|
|
252
|
+
readonly expiresAt: number
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/** A run's claim as a listing reports it. */
|
|
256
|
+
export interface ClaimSummary {
|
|
257
|
+
readonly holder: string
|
|
258
|
+
readonly fence: ClaimFence
|
|
259
|
+
readonly expiresAt: number
|
|
260
|
+
/**
|
|
261
|
+
* Whether the claim had expired at the instant the listing was taken.
|
|
262
|
+
*
|
|
263
|
+
* A separate field rather than something the caller derives from
|
|
264
|
+
* `expiresAt`, because the caller would derive it against a DIFFERENT
|
|
265
|
+
* clock than the store used, and one page would then disagree with
|
|
266
|
+
* itself about which rows are available.
|
|
267
|
+
*/
|
|
268
|
+
readonly expired: boolean
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/** What a caller asks for when taking a run. */
|
|
272
|
+
export interface ClaimRunOptions {
|
|
273
|
+
/** Who is taking it. Recorded so an operator can see what holds a stuck run. */
|
|
274
|
+
readonly holder: string
|
|
275
|
+
/** How long the holding is good for, in ms. */
|
|
276
|
+
readonly ttlMs: number
|
|
277
|
+
/** Clock, for tests and so one operation judges every expiry against one instant. */
|
|
278
|
+
readonly now?: number
|
|
279
|
+
}
|
|
280
|
+
|
|
211
281
|
/** Filters and paging for {@link CheckpointStore.listDurableRuns}. */
|
|
212
282
|
export interface ListDurableRunsOptions {
|
|
213
283
|
/**
|
|
@@ -223,6 +293,16 @@ export interface ListDurableRunsOptions {
|
|
|
223
293
|
* to include it.
|
|
224
294
|
*/
|
|
225
295
|
readonly park?: readonly ParkState[]
|
|
296
|
+
/**
|
|
297
|
+
* Keep only runs that are, or are not, currently held by a worker.
|
|
298
|
+
*
|
|
299
|
+
* `false` is the queue-reader's filter: give me the work nobody has. A
|
|
300
|
+
* claim that has expired counts as NOT held, because that is what expiry
|
|
301
|
+
* means and a reader that skipped expired claims would leave a dead
|
|
302
|
+
* worker's runs invisible forever — the exact failure the lease exists to
|
|
303
|
+
* prevent.
|
|
304
|
+
*/
|
|
305
|
+
readonly claimed?: boolean
|
|
226
306
|
/** Page size. Defaults to 100, clamped to at least 1. */
|
|
227
307
|
readonly limit?: number
|
|
228
308
|
/** Resume token from the previous page's {@link DurableRunPage.cursor}. */
|
|
@@ -279,8 +359,27 @@ export interface DurableRunPage {
|
|
|
279
359
|
* proceed.
|
|
280
360
|
*/
|
|
281
361
|
export interface CheckpointStore {
|
|
282
|
-
/**
|
|
283
|
-
|
|
362
|
+
/**
|
|
363
|
+
* Persist one checkpoint. Overwrites an existing checkpoint with the same id.
|
|
364
|
+
*
|
|
365
|
+
* @param fence the {@link ClaimFence} of the holding this write belongs
|
|
366
|
+
* to, when the run is claimed. A store that supports claims REFUSES a
|
|
367
|
+
* write whose fence is below the run's current one — that refusal is
|
|
368
|
+
* what makes a claim a lease rather than a suggestion, because a holder
|
|
369
|
+
* stalled past its expiry believes it still holds and is wrong only at
|
|
370
|
+
* the moment it writes.
|
|
371
|
+
*
|
|
372
|
+
* Omit it and the write is unfenced, which is exactly today's behaviour
|
|
373
|
+
* and correct for a single-writer deployment. A store MUST NOT start
|
|
374
|
+
* refusing unfenced writes because some other write carried a fence:
|
|
375
|
+
* that would make adding a claim to one worker break every worker that
|
|
376
|
+
* has not adopted it yet.
|
|
377
|
+
*/
|
|
378
|
+
writeCheckpoint(
|
|
379
|
+
scope: CheckpointRunScope,
|
|
380
|
+
checkpoint: IterationCheckpoint,
|
|
381
|
+
fence?: ClaimFence,
|
|
382
|
+
): Promise<void>
|
|
284
383
|
|
|
285
384
|
/** Load a single checkpoint by id. Returns `null` when it does not exist. */
|
|
286
385
|
readCheckpoint(
|
|
@@ -342,4 +441,92 @@ export interface CheckpointStore {
|
|
|
342
441
|
scope: CheckpointListingScope,
|
|
343
442
|
options?: ListDurableRunsOptions,
|
|
344
443
|
): Promise<DurableRunPage>
|
|
444
|
+
|
|
445
|
+
/**
|
|
446
|
+
* Take exclusive working possession of a run, or report that somebody
|
|
447
|
+
* else has it. OPTIONAL — see the optional-capability rule on this
|
|
448
|
+
* interface.
|
|
449
|
+
*
|
|
450
|
+
* Returns the holding on success and `null` when the run is currently
|
|
451
|
+
* held by somebody else. `null` is not an error: "another worker got
|
|
452
|
+
* there first" is the ordinary outcome of a queue with more than one
|
|
453
|
+
* reader, and a thrown exception would make the normal case look like a
|
|
454
|
+
* fault.
|
|
455
|
+
*
|
|
456
|
+
* ### What it is for
|
|
457
|
+
*
|
|
458
|
+
* Putting parked runs on a queue and letting more than one worker drain
|
|
459
|
+
* it. Without this, two workers restore the same checkpoint, both execute
|
|
460
|
+
* the run's tools, and both write checkpoints under one run id — each
|
|
461
|
+
* write minting a fresh checkpoint id, so two divergent chains land in
|
|
462
|
+
* one list and the pending lookup returns whichever wrote last. Half the
|
|
463
|
+
* work vanishes and nothing reports an error.
|
|
464
|
+
*
|
|
465
|
+
* ### The lease, and why it expires
|
|
466
|
+
*
|
|
467
|
+
* A claim is a LEASE, not a lock. A lock held by a process that dies is
|
|
468
|
+
* held forever, and the runs behind it are unreachable by anything except
|
|
469
|
+
* a human with a shell. The expiry is what makes a dead holder's work
|
|
470
|
+
* recoverable without one.
|
|
471
|
+
*
|
|
472
|
+
* The expiry is also why a fence exists. A holder does not know it has
|
|
473
|
+
* expired — a long pause, a suspended container and a partition all look
|
|
474
|
+
* from the inside like time not passing — so it wakes and writes as
|
|
475
|
+
* though it still holds. Liveness cannot be checked from here. What CAN
|
|
476
|
+
* be checked, at the write, is whether the holding that write belongs to
|
|
477
|
+
* is still the current one, and that is a comparison of two numbers.
|
|
478
|
+
*
|
|
479
|
+
* ### Reclaiming
|
|
480
|
+
*
|
|
481
|
+
* Calling this on a run whose claim has expired SUCCEEDS and mints a
|
|
482
|
+
* fence strictly greater than the expired holding's. The previous holder
|
|
483
|
+
* is not notified — it cannot be, that is the premise — it simply stops
|
|
484
|
+
* being able to write.
|
|
485
|
+
*
|
|
486
|
+
* Calling it again as the CURRENT holder also succeeds and extends the
|
|
487
|
+
* lease, minting a new fence. Renewal and reclamation are the same
|
|
488
|
+
* operation from the store's side, which is why there is no separate
|
|
489
|
+
* `renew`: two code paths that must agree about who holds a run is one
|
|
490
|
+
* more than can be kept correct.
|
|
491
|
+
*
|
|
492
|
+
* ### What a backend implementing this MUST guarantee
|
|
493
|
+
*
|
|
494
|
+
* These are the properties the fence comparison depends on. None of them
|
|
495
|
+
* is checkable from here, and every one of them was violated by the first
|
|
496
|
+
* built-in implementation, so they are written down rather than assumed:
|
|
497
|
+
*
|
|
498
|
+
* 1. **A fence exceeds every fence ever issued for the run** — including
|
|
499
|
+
* across a release, and across deletion of whatever recorded it. A
|
|
500
|
+
* counter that rewinds re-issues a number a stalled worker still
|
|
501
|
+
* believes it holds, and that worker's writes become legal again.
|
|
502
|
+
* 2. **Fences are unique.** The write check is `fence < current`, so two
|
|
503
|
+
* holders at one number are both admitted. Equality is permissive
|
|
504
|
+
* here, which makes a duplicate worse than a gap.
|
|
505
|
+
* 3. **The fence check is atomic with the write.** Reading the current
|
|
506
|
+
* fence and then writing is check-then-act, and the gap is a race. A
|
|
507
|
+
* database gets this free (`UPDATE … WHERE fence >= ?`); a filesystem
|
|
508
|
+
* does not, and the built-in disk store narrows rather than closes it.
|
|
509
|
+
* 4. **`holder` is unique per process.** It is evidence rather than
|
|
510
|
+
* authority, but it is the only thing distinguishing a RENEWAL from a
|
|
511
|
+
* theft — two workers sharing a holder string take a live, unexpired
|
|
512
|
+
* claim from each other instantly. Use something per-process, not a
|
|
513
|
+
* per-deployment name.
|
|
514
|
+
*/
|
|
515
|
+
claimRun?(scope: CheckpointRunScope, options: ClaimRunOptions): Promise<RunClaim | null>
|
|
516
|
+
|
|
517
|
+
/**
|
|
518
|
+
* Give a claim up early. Idempotent: releasing a claim that already
|
|
519
|
+
* expired, was superseded, or never existed succeeds as a no-op.
|
|
520
|
+
*
|
|
521
|
+
* Presenting a stale fence releases NOTHING — a worker that stalled past
|
|
522
|
+
* its lease must not be able to hand away a run somebody else is now
|
|
523
|
+
* holding, and that is the same fencing comparison the write path makes.
|
|
524
|
+
*
|
|
525
|
+
* Optional to call, not optional to matter: a worker that finishes and
|
|
526
|
+
* releases returns the run to the queue immediately, where one that just
|
|
527
|
+
* exits leaves it stuck until the lease expires. That is a latency
|
|
528
|
+
* difference, never a correctness one, which is the property that lets a
|
|
529
|
+
* crashed worker be indistinguishable from a slow one.
|
|
530
|
+
*/
|
|
531
|
+
releaseRun?(scope: CheckpointRunScope, fence: ClaimFence): Promise<void>
|
|
345
532
|
}
|