@namzu/sdk 20.3.0 → 20.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +142 -0
- package/dist/public-runtime.d.ts +1 -0
- package/dist/public-runtime.d.ts.map +1 -1
- package/dist/public-runtime.js +5 -0
- package/dist/public-runtime.js.map +1 -1
- package/dist/runtime/query/checkpoint.d.ts +27 -1
- package/dist/runtime/query/checkpoint.d.ts.map +1 -1
- package/dist/runtime/query/checkpoint.js +34 -4
- package/dist/runtime/query/checkpoint.js.map +1 -1
- package/dist/runtime/query/index.d.ts +21 -1
- package/dist/runtime/query/index.d.ts.map +1 -1
- package/dist/runtime/query/index.js +4 -0
- package/dist/runtime/query/index.js.map +1 -1
- package/dist/runtime/query/resume-run.d.ts +9 -1
- package/dist/runtime/query/resume-run.d.ts.map +1 -1
- package/dist/runtime/query/resume-run.js +2 -1
- package/dist/runtime/query/resume-run.js.map +1 -1
- package/dist/store/index.d.ts +1 -1
- package/dist/store/index.d.ts.map +1 -1
- package/dist/store/index.js +1 -1
- package/dist/store/index.js.map +1 -1
- package/dist/store/run/checkpoint-disk.d.ts +18 -2
- package/dist/store/run/checkpoint-disk.d.ts.map +1 -1
- package/dist/store/run/checkpoint-disk.js +50 -5
- package/dist/store/run/checkpoint-disk.js.map +1 -1
- package/dist/store/run/checkpoint-memory.d.ts +22 -2
- package/dist/store/run/checkpoint-memory.d.ts.map +1 -1
- package/dist/store/run/checkpoint-memory.js +99 -4
- package/dist/store/run/checkpoint-memory.js.map +1 -1
- package/dist/store/run/claim-disk.d.ts +130 -0
- package/dist/store/run/claim-disk.d.ts.map +1 -0
- package/dist/store/run/claim-disk.js +550 -0
- package/dist/store/run/claim-disk.js.map +1 -0
- package/dist/store/run/listing.d.ts +44 -1
- package/dist/store/run/listing.d.ts.map +1 -1
- package/dist/store/run/listing.js +92 -1
- package/dist/store/run/listing.js.map +1 -1
- package/dist/types/run/checkpoint-store.d.ts +178 -2
- package/dist/types/run/checkpoint-store.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/public-runtime.ts +5 -0
- package/src/runtime/query/checkpoint.ts +42 -5
- package/src/runtime/query/index.ts +26 -1
- package/src/runtime/query/resume-run.ts +12 -2
- package/src/store/index.ts +4 -0
- package/src/store/run/checkpoint-disk.ts +70 -5
- package/src/store/run/checkpoint-memory.ts +118 -3
- package/src/store/run/claim-disk.ts +593 -0
- package/src/store/run/listing.ts +116 -1
- package/src/types/run/checkpoint-store.ts +189 -2
|
@@ -147,9 +147,16 @@ export function toDurableRunEntry(scope, checkpoints, now) {
|
|
|
147
147
|
*/
|
|
148
148
|
export function paginateDurableRuns(entries, options) {
|
|
149
149
|
const wanted = options?.park;
|
|
150
|
-
const
|
|
150
|
+
const byPark = wanted && wanted.length > 0
|
|
151
151
|
? entries.filter((e) => e.park !== undefined && wanted.includes(e.park.state))
|
|
152
152
|
: entries;
|
|
153
|
+
// An expired claim counts as unheld. That is what expiry means, and a
|
|
154
|
+
// queue reader that treated an expired claim as held would leave a dead
|
|
155
|
+
// worker's runs invisible forever — the exact failure a lease exists to
|
|
156
|
+
// prevent, reintroduced by the filter that reads it.
|
|
157
|
+
const filtered = options?.claimed === undefined
|
|
158
|
+
? byPark
|
|
159
|
+
: byPark.filter((e) => (e.claim !== undefined && !e.claim.expired) === options.claimed);
|
|
153
160
|
// Both orders sort on a key that cannot move under a paging caller — see
|
|
154
161
|
// the contract comment on `listDurableRuns`.
|
|
155
162
|
const orderBy = options?.orderBy ?? 'runId';
|
|
@@ -239,4 +246,88 @@ export async function listDurableRuns(store, scope, options) {
|
|
|
239
246
|
assertContiguousListingScope(scope, 'listDurableRuns');
|
|
240
247
|
return store.listDurableRuns(scope, options);
|
|
241
248
|
}
|
|
249
|
+
/**
|
|
250
|
+
* Take working possession of a run, refusing when the store cannot arbitrate.
|
|
251
|
+
*
|
|
252
|
+
* The refusal is the entire safety property. `claimRun` is optional on the
|
|
253
|
+
* contract, and the natural way to reach an absent optional method is to skip
|
|
254
|
+
* it — which here means every worker proceeds, believing it holds a run
|
|
255
|
+
* nobody arbitrated. Two workers then restore the same checkpoint, both run
|
|
256
|
+
* the tools, and both write under one run id; half the work vanishes with no
|
|
257
|
+
* error anywhere.
|
|
258
|
+
*
|
|
259
|
+
* So a store with no claim support does not get "claimed by default". It gets
|
|
260
|
+
* an error naming the deployment shape it cannot support. A single-writer
|
|
261
|
+
* host never calls this and is unaffected.
|
|
262
|
+
*
|
|
263
|
+
* Returns `null` — not an error — when another holder has the run. That is
|
|
264
|
+
* the ordinary outcome of two readers on one queue, and a caller loops to the
|
|
265
|
+
* next run rather than handling a fault.
|
|
266
|
+
*/
|
|
267
|
+
export async function claimRun(store, scope, options) {
|
|
268
|
+
if (typeof store.claimRun !== 'function') {
|
|
269
|
+
throw new NamzuError({
|
|
270
|
+
code: 'capability_unavailable',
|
|
271
|
+
message: 'claimRun: the injected checkpoint store does not implement `claimRun`, so it cannot arbitrate between two workers taking the same run. Refusing rather than proceeding unclaimed — proceeding would let two workers restore one checkpoint, both execute its tools, and both write under one run id, which loses half the work and reports nothing. Supply a store that implements it, or run a single writer per run.',
|
|
272
|
+
details: { runId: scope.runId },
|
|
273
|
+
});
|
|
274
|
+
}
|
|
275
|
+
if (!Number.isFinite(options.ttlMs) || options.ttlMs <= 0) {
|
|
276
|
+
throw new NamzuError({
|
|
277
|
+
code: 'invalid_config',
|
|
278
|
+
message: `claimRun: ttlMs must be a positive number of milliseconds, got ${String(options.ttlMs)}. A lease that expires immediately is a lease every worker can take at once, which is the condition this call exists to prevent.`,
|
|
279
|
+
details: { runId: scope.runId, ttlMs: options.ttlMs },
|
|
280
|
+
});
|
|
281
|
+
}
|
|
282
|
+
return store.claimRun(scope, options);
|
|
283
|
+
}
|
|
284
|
+
/**
|
|
285
|
+
* Give a claim up early, refusing when the store cannot arbitrate.
|
|
286
|
+
*
|
|
287
|
+
* Refuses for the same reason as {@link claimRun}: a host that believes it is
|
|
288
|
+
* releasing a claim on a store that has none is a host that believes the
|
|
289
|
+
* whole mechanism is running.
|
|
290
|
+
*/
|
|
291
|
+
export async function releaseRun(store, scope, fence) {
|
|
292
|
+
if (typeof store.releaseRun !== 'function') {
|
|
293
|
+
throw new NamzuError({
|
|
294
|
+
code: 'capability_unavailable',
|
|
295
|
+
message: 'releaseRun: the injected checkpoint store does not implement `releaseRun`. A release that silently does nothing would leave the run held until its lease expires while the caller believes it is back on the queue.',
|
|
296
|
+
details: { runId: scope.runId },
|
|
297
|
+
});
|
|
298
|
+
}
|
|
299
|
+
return store.releaseRun(scope, fence);
|
|
300
|
+
}
|
|
301
|
+
/**
|
|
302
|
+
* The refusal a store raises when a write presents a superseded fence.
|
|
303
|
+
*
|
|
304
|
+
* Shared so both shipped stores say the same thing, and so a host writing its
|
|
305
|
+
* own backend raises something a caller can branch on rather than a message
|
|
306
|
+
* string. This is the moment a stalled worker learns it lost the run — the
|
|
307
|
+
* only moment it CAN learn, since from the inside a pause and a partition
|
|
308
|
+
* both look like time not passing.
|
|
309
|
+
*/
|
|
310
|
+
export function fencedOut(scope, presented, current) {
|
|
311
|
+
return new NamzuError({
|
|
312
|
+
code: 'storage_error',
|
|
313
|
+
message: `writeCheckpoint: refusing a write for run ${scope.runId} fenced at ${presented} — the run is now claimed at ${current}. Another worker took this run over, so this process no longer holds it and its work is not the record. Stop the run rather than retrying: the claim is gone, not busy.`,
|
|
314
|
+
details: { runId: scope.runId, presentedFence: presented, currentFence: current },
|
|
315
|
+
retryable: false,
|
|
316
|
+
});
|
|
317
|
+
}
|
|
318
|
+
/**
|
|
319
|
+
* Whether a recorded claim still holds at `now`, and the summary a listing
|
|
320
|
+
* reports for it.
|
|
321
|
+
*/
|
|
322
|
+
export function toClaimSummary(claim, now) {
|
|
323
|
+
return {
|
|
324
|
+
holder: claim.holder,
|
|
325
|
+
fence: claim.fence,
|
|
326
|
+
expiresAt: claim.expiresAt,
|
|
327
|
+
// Judged here, once, against the store's clock. Left to the caller it
|
|
328
|
+
// would be judged against a different one, and a single page could
|
|
329
|
+
// then disagree with itself about which rows are available.
|
|
330
|
+
expired: now >= claim.expiresAt,
|
|
331
|
+
};
|
|
332
|
+
}
|
|
242
333
|
//# sourceMappingURL=listing.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"listing.js","sourceRoot":"","sources":["../../../src/store/run/listing.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAA;
|
|
1
|
+
{"version":3,"file":"listing.js","sourceRoot":"","sources":["../../../src/store/run/listing.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAA;AAkBxD,4CAA4C;AAC5C,MAAM,CAAC,MAAM,yBAAyB,GAAG,GAAG,CAAA;AAE5C;;;;;;;;GAQG;AACH,MAAM,UAAU,4BAA4B,CAAC,KAA6B,EAAE,MAAc;IACzF,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QACpE,MAAM,IAAI,UAAU,CAAC;YACpB,IAAI,EAAE,gBAAgB;YACtB,OAAO,EAAE,GAAG,MAAM,mMAAmM;YACrN,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE;SACjE,CAAC,CAAA;IACH,CAAC;AACF,CAAC;AAED,gFAAgF;AAChF,SAAS,cAAc,CAAC,OAAwB,EAAE,GAAW;IAC5D,OAAO,OAAO,CAAC,UAAU,KAAK,SAAS,IAAI,GAAG,IAAI,OAAO,CAAC,UAAU,CAAA;AACrE,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,WAAW,CAAC,OAAwB,EAAE,GAAW;IACzD,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS;QAAE,OAAO,UAAU,CAAA;IACvD,OAAO,cAAc,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,CAAA;AAChE,CAAC;AAED,SAAS,aAAa,CACrB,EAAuB,EACvB,OAAwB,EACxB,GAAW;IAEX,OAAO;QACN,KAAK,EAAE,WAAW,CAAC,OAAO,EAAE,GAAG,CAAC;QAChC,YAAY,EAAE,EAAE,CAAC,EAAE;QACnB,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI;QACjC,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,GAAG,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/E,GAAG,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC/E,CAAA;AACF,CAAC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,aAAa,CAC5B,WAA2C,EAC3C,GAAW;IAEX,IAAI,IAA6B,CAAA;IACjC,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAA;IACjB,IAAI,YAAY,GAAG,MAAM,CAAC,iBAAiB,CAAA;IAE3C,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,CAAA;QAC1B,IAAI,CAAC,OAAO;YAAE,SAAQ;QACtB,MAAM,OAAO,GAAG,aAAa,CAAC,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,CAAA;QAC/C,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QACrC,IAAI,IAAI,GAAG,QAAQ,IAAI,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,QAAQ,GAAG,YAAY,CAAC,EAAE,CAAC;YAC/E,IAAI,GAAG,OAAO,CAAA;YACd,QAAQ,GAAG,IAAI,CAAA;YACf,YAAY,GAAG,OAAO,CAAC,QAAQ,CAAA;QAChC,CAAC;IACF,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,MAAM,SAAS,GAA8B;IAC5C,QAAQ,EAAE,CAAC;IACX,OAAO,EAAE,CAAC;IACV,WAAW,EAAE,CAAC;CACd,CAAA;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,iBAAiB,CAChC,KAAyB,EACzB,WAA2C,EAC3C,GAAW;IAEX,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAA;IAEzC,IAAI,MAAM,GAAG,WAAW,CAAC,CAAC,CAAwB,CAAA;IAClD,yEAAyE;IACzE,kEAAkE;IAClE,sEAAsE;IACtE,yEAAyE;IACzE,wEAAwE;IACxE,qEAAqE;IACrE,IAAI,YAAgC,CAAA;IACpC,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE,CAAC;QAC9B,IAAI,EAAE,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS;YAAE,MAAM,GAAG,EAAE,CAAA;QAChD,IACC,EAAE,CAAC,YAAY,KAAK,SAAS;YAC7B,CAAC,YAAY,KAAK,SAAS,IAAI,EAAE,CAAC,YAAY,GAAG,YAAY,CAAC,EAC7D,CAAC;YACF,YAAY,GAAG,EAAE,CAAC,YAAY,CAAA;QAC/B,CAAC;IACF,CAAC;IAED,MAAM,IAAI,GAAG,aAAa,CAAC,WAAW,EAAE,GAAG,CAAC,CAAA;IAE5C,OAAO;QACN,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAChE,GAAG,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvD,eAAe,EAAE,WAAW,CAAC,MAAM;QACnC,kBAAkB,EAAE,MAAM,CAAC,EAAE;QAC7B,kBAAkB,EAAE,MAAM,CAAC,SAAS;QACpC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACzB,CAAA;AACF,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,mBAAmB,CAClC,OAAmC,EACnC,OAAgC;IAEhC,MAAM,MAAM,GAAG,OAAO,EAAE,IAAI,CAAA;IAC5B,MAAM,MAAM,GACX,MAAM,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAC1B,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9E,CAAC,CAAC,OAAO,CAAA;IAEX,sEAAsE;IACtE,wEAAwE;IACxE,wEAAwE;IACxE,qDAAqD;IACrD,MAAM,QAAQ,GACb,OAAO,EAAE,OAAO,KAAK,SAAS;QAC7B,CAAC,CAAC,MAAM;QACR,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,CAAA;IAEzF,yEAAyE;IACzE,6CAA6C;IAC7C,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,OAAO,CAAA;IAC3C,MAAM,OAAO,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC3C,WAAW,CAAC,OAAO,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CACrD,CAAA;IAED,MAAM,KAAK,GAAG,OAAO,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAC/F,MAAM,KAAK,GACV,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAA;IAChG,MAAM,IAAI,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAA;IAE/C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,IAAI,yBAAyB,CAAC,CAAC,CAAA;IAClF,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,GAAG,KAAK,CAAC,CAAA;IAC9C,MAAM,SAAS,GAAG,IAAI,GAAG,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,CAAA;IAEtD,OAAO;QACN,OAAO,EAAE,IAAI;QACb,iEAAiE;QACjE,8DAA8D;QAC9D,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YACjC,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,EAAE,MAAM,EAAE,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAoB,EAAE,OAAO,CAAC,CAAC,EAAE,CAAC;KACxF,CAAA;AACF,CAAC;AAcD,SAAS,OAAO,CAAC,KAAsB,EAAE,OAAwB;IAChE,IAAI,OAAO,KAAK,OAAO;QAAE,OAAO,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAA;IACnD,OAAO,KAAK,CAAC,YAAY,KAAK,SAAS;QACtC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC;QACrB,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,KAAK,CAAC,CAAA;AACxC,CAAC;AAED,SAAS,WAAW,CAAC,CAAU,EAAE,CAAU;IAC1C,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;IACrC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;IACrC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;AAC9C,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,YAAY,CAAC,GAAY;IACjC,OAAO,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAA;AACvC,CAAC;AAED,SAAS,YAAY,CAAC,MAAc,EAAE,OAAwB;IAC7D,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IACjC,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,GAAG,CAAC,CAAC,CAAA;IAC7C,IAAI,KAAK,GAAG,CAAC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,UAAU,CAAC;YACpB,IAAI,EAAE,gBAAgB;YACtB,OAAO,EAAE,qBAAqB,MAAM,6JAA6J;YACjM,OAAO,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE;SAC5B,CAAC,CAAA;IACH,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAA;IAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC,CAAA;IACrD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACvD,MAAM,IAAI,UAAU,CAAC;YACpB,IAAI,EAAE,gBAAgB;YACtB,OAAO,EAAE,4BAA4B,MAAM,wGAAwG;YACnJ,OAAO,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE;SAC5B,CAAC,CAAA;IACH,CAAC;IACD,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAA;AAC/C,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACpC,KAAsB,EACtB,KAA6B,EAC7B,OAAgC;IAEhC,IAAI,OAAO,KAAK,CAAC,eAAe,KAAK,UAAU,EAAE,CAAC;QACjD,MAAM,IAAI,UAAU,CAAC;YACpB,IAAI,EAAE,wBAAwB;YAC9B,OAAO,EACN,+VAA+V;YAChW,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE;SACrC,CAAC,CAAA;IACH,CAAC;IACD,4BAA4B,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAA;IACtD,OAAO,KAAK,CAAC,eAAe,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;AAC7C,CAAC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC7B,KAAsB,EACtB,KAAyB,EACzB,OAAwB;IAExB,IAAI,OAAO,KAAK,CAAC,QAAQ,KAAK,UAAU,EAAE,CAAC;QAC1C,MAAM,IAAI,UAAU,CAAC;YACpB,IAAI,EAAE,wBAAwB;YAC9B,OAAO,EACN,wZAAwZ;YACzZ,OAAO,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE;SAC/B,CAAC,CAAA;IACH,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,KAAK,IAAI,CAAC,EAAE,CAAC;QAC3D,MAAM,IAAI,UAAU,CAAC;YACpB,IAAI,EAAE,gBAAgB;YACtB,OAAO,EAAE,kEAAkE,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,kIAAkI;YAClO,OAAO,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE;SACrD,CAAC,CAAA;IACH,CAAC;IACD,OAAO,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;AACtC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC/B,KAAsB,EACtB,KAAyB,EACzB,KAAiB;IAEjB,IAAI,OAAO,KAAK,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;QAC5C,MAAM,IAAI,UAAU,CAAC;YACpB,IAAI,EAAE,wBAAwB;YAC9B,OAAO,EACN,qNAAqN;YACtN,OAAO,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE;SAC/B,CAAC,CAAA;IACH,CAAC;IACD,OAAO,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;AACtC,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,SAAS,CACxB,KAAyB,EACzB,SAAiB,EACjB,OAAe;IAEf,OAAO,IAAI,UAAU,CAAC;QACrB,IAAI,EAAE,eAAe;QACrB,OAAO,EAAE,6CAA6C,KAAK,CAAC,KAAK,cAAc,SAAS,gCAAgC,OAAO,yKAAyK;QACxS,OAAO,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,cAAc,EAAE,SAAS,EAAE,YAAY,EAAE,OAAO,EAAE;QACjF,SAAS,EAAE,KAAK;KAChB,CAAC,CAAA;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,KAAe,EAAE,GAAW;IAC1D,OAAO;QACN,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,sEAAsE;QACtE,mEAAmE;QACnE,4DAA4D;QAC5D,OAAO,EAAE,GAAG,IAAI,KAAK,CAAC,SAAS;KAC/B,CAAA;AACF,CAAC"}
|
|
@@ -167,6 +167,16 @@ export interface DurableRunEntry extends CheckpointRunScope {
|
|
|
167
167
|
readonly latestCheckpointAt: number;
|
|
168
168
|
/** Absent when the run has never parked. */
|
|
169
169
|
readonly park?: ParkSummary;
|
|
170
|
+
/**
|
|
171
|
+
* Absent when no process has ever claimed the run.
|
|
172
|
+
*
|
|
173
|
+
* A SIBLING of {@link DurableRunEntry.park}, not a member of
|
|
174
|
+
* {@link ParkState} — see the note on that union. A park is a question put
|
|
175
|
+
* to a human; a claim is a lease held by a process. A run can have both,
|
|
176
|
+
* neither, or either, and the state a queue worker needs most is parked
|
|
177
|
+
* AND unclaimed, which one union cannot say.
|
|
178
|
+
*/
|
|
179
|
+
readonly claim?: ClaimSummary;
|
|
170
180
|
}
|
|
171
181
|
/**
|
|
172
182
|
* Which order a listing comes back in.
|
|
@@ -198,6 +208,61 @@ export type DurableRunOrder =
|
|
|
198
208
|
* instead of a date nobody recorded.
|
|
199
209
|
*/
|
|
200
210
|
| 'createdAt';
|
|
211
|
+
/**
|
|
212
|
+
* A monotonically increasing number identifying one holding of a run's claim.
|
|
213
|
+
*
|
|
214
|
+
* The load-bearing word is *fencing*. A mutex answers "may I proceed", and a
|
|
215
|
+
* holder that stalls past its lease — a long GC pause, a suspended container,
|
|
216
|
+
* a partitioned network — answers it "yes" and then writes, long after
|
|
217
|
+
* somebody else legitimately took over. A fence answers a different question
|
|
218
|
+
* at the moment of the WRITE: "is the holding I belong to still the current
|
|
219
|
+
* one". Every claim of a run mints a number strictly greater than the last,
|
|
220
|
+
* so a store can reject a write from a superseded holder without knowing
|
|
221
|
+
* anything about processes, clocks or liveness.
|
|
222
|
+
*
|
|
223
|
+
* Not a random token, deliberately: randomness proves identity and cannot
|
|
224
|
+
* establish *order*, and order is the entire mechanism.
|
|
225
|
+
*/
|
|
226
|
+
export type ClaimFence = number;
|
|
227
|
+
/** A holding of a run's claim, as issued to the process that took it. */
|
|
228
|
+
export interface RunClaim {
|
|
229
|
+
/** Opaque caller-supplied identity — a worker id, a pod name. Evidence, not authority. */
|
|
230
|
+
readonly holder: string;
|
|
231
|
+
/** See {@link ClaimFence}. Present it on every durable write. */
|
|
232
|
+
readonly fence: ClaimFence;
|
|
233
|
+
/**
|
|
234
|
+
* Absolute epoch ms after which the claim may be taken by somebody else.
|
|
235
|
+
*
|
|
236
|
+
* Absolute rather than a duration for the same reason a park's deadline
|
|
237
|
+
* is: it has to survive the process that set it. A duration plus an
|
|
238
|
+
* in-process timer cannot — the holder is the thing that dies.
|
|
239
|
+
*/
|
|
240
|
+
readonly expiresAt: number;
|
|
241
|
+
}
|
|
242
|
+
/** A run's claim as a listing reports it. */
|
|
243
|
+
export interface ClaimSummary {
|
|
244
|
+
readonly holder: string;
|
|
245
|
+
readonly fence: ClaimFence;
|
|
246
|
+
readonly expiresAt: number;
|
|
247
|
+
/**
|
|
248
|
+
* Whether the claim had expired at the instant the listing was taken.
|
|
249
|
+
*
|
|
250
|
+
* A separate field rather than something the caller derives from
|
|
251
|
+
* `expiresAt`, because the caller would derive it against a DIFFERENT
|
|
252
|
+
* clock than the store used, and one page would then disagree with
|
|
253
|
+
* itself about which rows are available.
|
|
254
|
+
*/
|
|
255
|
+
readonly expired: boolean;
|
|
256
|
+
}
|
|
257
|
+
/** What a caller asks for when taking a run. */
|
|
258
|
+
export interface ClaimRunOptions {
|
|
259
|
+
/** Who is taking it. Recorded so an operator can see what holds a stuck run. */
|
|
260
|
+
readonly holder: string;
|
|
261
|
+
/** How long the holding is good for, in ms. */
|
|
262
|
+
readonly ttlMs: number;
|
|
263
|
+
/** Clock, for tests and so one operation judges every expiry against one instant. */
|
|
264
|
+
readonly now?: number;
|
|
265
|
+
}
|
|
201
266
|
/** Filters and paging for {@link CheckpointStore.listDurableRuns}. */
|
|
202
267
|
export interface ListDurableRunsOptions {
|
|
203
268
|
/**
|
|
@@ -212,6 +277,16 @@ export interface ListDurableRunsOptions {
|
|
|
212
277
|
* to include it.
|
|
213
278
|
*/
|
|
214
279
|
readonly park?: readonly ParkState[];
|
|
280
|
+
/**
|
|
281
|
+
* Keep only runs that are, or are not, currently held by a worker.
|
|
282
|
+
*
|
|
283
|
+
* `false` is the queue-reader's filter: give me the work nobody has. A
|
|
284
|
+
* claim that has expired counts as NOT held, because that is what expiry
|
|
285
|
+
* means and a reader that skipped expired claims would leave a dead
|
|
286
|
+
* worker's runs invisible forever — the exact failure the lease exists to
|
|
287
|
+
* prevent.
|
|
288
|
+
*/
|
|
289
|
+
readonly claimed?: boolean;
|
|
215
290
|
/** Page size. Defaults to 100, clamped to at least 1. */
|
|
216
291
|
readonly limit?: number;
|
|
217
292
|
/** Resume token from the previous page's {@link DurableRunPage.cursor}. */
|
|
@@ -266,8 +341,23 @@ export interface DurableRunPage {
|
|
|
266
341
|
* proceed.
|
|
267
342
|
*/
|
|
268
343
|
export interface CheckpointStore {
|
|
269
|
-
/**
|
|
270
|
-
|
|
344
|
+
/**
|
|
345
|
+
* Persist one checkpoint. Overwrites an existing checkpoint with the same id.
|
|
346
|
+
*
|
|
347
|
+
* @param fence the {@link ClaimFence} of the holding this write belongs
|
|
348
|
+
* to, when the run is claimed. A store that supports claims REFUSES a
|
|
349
|
+
* write whose fence is below the run's current one — that refusal is
|
|
350
|
+
* what makes a claim a lease rather than a suggestion, because a holder
|
|
351
|
+
* stalled past its expiry believes it still holds and is wrong only at
|
|
352
|
+
* the moment it writes.
|
|
353
|
+
*
|
|
354
|
+
* Omit it and the write is unfenced, which is exactly today's behaviour
|
|
355
|
+
* and correct for a single-writer deployment. A store MUST NOT start
|
|
356
|
+
* refusing unfenced writes because some other write carried a fence:
|
|
357
|
+
* that would make adding a claim to one worker break every worker that
|
|
358
|
+
* has not adopted it yet.
|
|
359
|
+
*/
|
|
360
|
+
writeCheckpoint(scope: CheckpointRunScope, checkpoint: IterationCheckpoint, fence?: ClaimFence): Promise<void>;
|
|
271
361
|
/** Load a single checkpoint by id. Returns `null` when it does not exist. */
|
|
272
362
|
readCheckpoint(scope: CheckpointRunScope, checkpointId: CheckpointId): Promise<IterationCheckpoint | null>;
|
|
273
363
|
/**
|
|
@@ -319,5 +409,91 @@ export interface CheckpointStore {
|
|
|
319
409
|
* @param options filters and paging. See {@link ListDurableRunsOptions}.
|
|
320
410
|
*/
|
|
321
411
|
listDurableRuns?(scope: CheckpointListingScope, options?: ListDurableRunsOptions): Promise<DurableRunPage>;
|
|
412
|
+
/**
|
|
413
|
+
* Take exclusive working possession of a run, or report that somebody
|
|
414
|
+
* else has it. OPTIONAL — see the optional-capability rule on this
|
|
415
|
+
* interface.
|
|
416
|
+
*
|
|
417
|
+
* Returns the holding on success and `null` when the run is currently
|
|
418
|
+
* held by somebody else. `null` is not an error: "another worker got
|
|
419
|
+
* there first" is the ordinary outcome of a queue with more than one
|
|
420
|
+
* reader, and a thrown exception would make the normal case look like a
|
|
421
|
+
* fault.
|
|
422
|
+
*
|
|
423
|
+
* ### What it is for
|
|
424
|
+
*
|
|
425
|
+
* Putting parked runs on a queue and letting more than one worker drain
|
|
426
|
+
* it. Without this, two workers restore the same checkpoint, both execute
|
|
427
|
+
* the run's tools, and both write checkpoints under one run id — each
|
|
428
|
+
* write minting a fresh checkpoint id, so two divergent chains land in
|
|
429
|
+
* one list and the pending lookup returns whichever wrote last. Half the
|
|
430
|
+
* work vanishes and nothing reports an error.
|
|
431
|
+
*
|
|
432
|
+
* ### The lease, and why it expires
|
|
433
|
+
*
|
|
434
|
+
* A claim is a LEASE, not a lock. A lock held by a process that dies is
|
|
435
|
+
* held forever, and the runs behind it are unreachable by anything except
|
|
436
|
+
* a human with a shell. The expiry is what makes a dead holder's work
|
|
437
|
+
* recoverable without one.
|
|
438
|
+
*
|
|
439
|
+
* The expiry is also why a fence exists. A holder does not know it has
|
|
440
|
+
* expired — a long pause, a suspended container and a partition all look
|
|
441
|
+
* from the inside like time not passing — so it wakes and writes as
|
|
442
|
+
* though it still holds. Liveness cannot be checked from here. What CAN
|
|
443
|
+
* be checked, at the write, is whether the holding that write belongs to
|
|
444
|
+
* is still the current one, and that is a comparison of two numbers.
|
|
445
|
+
*
|
|
446
|
+
* ### Reclaiming
|
|
447
|
+
*
|
|
448
|
+
* Calling this on a run whose claim has expired SUCCEEDS and mints a
|
|
449
|
+
* fence strictly greater than the expired holding's. The previous holder
|
|
450
|
+
* is not notified — it cannot be, that is the premise — it simply stops
|
|
451
|
+
* being able to write.
|
|
452
|
+
*
|
|
453
|
+
* Calling it again as the CURRENT holder also succeeds and extends the
|
|
454
|
+
* lease, minting a new fence. Renewal and reclamation are the same
|
|
455
|
+
* operation from the store's side, which is why there is no separate
|
|
456
|
+
* `renew`: two code paths that must agree about who holds a run is one
|
|
457
|
+
* more than can be kept correct.
|
|
458
|
+
*
|
|
459
|
+
* ### What a backend implementing this MUST guarantee
|
|
460
|
+
*
|
|
461
|
+
* These are the properties the fence comparison depends on. None of them
|
|
462
|
+
* is checkable from here, and every one of them was violated by the first
|
|
463
|
+
* built-in implementation, so they are written down rather than assumed:
|
|
464
|
+
*
|
|
465
|
+
* 1. **A fence exceeds every fence ever issued for the run** — including
|
|
466
|
+
* across a release, and across deletion of whatever recorded it. A
|
|
467
|
+
* counter that rewinds re-issues a number a stalled worker still
|
|
468
|
+
* believes it holds, and that worker's writes become legal again.
|
|
469
|
+
* 2. **Fences are unique.** The write check is `fence < current`, so two
|
|
470
|
+
* holders at one number are both admitted. Equality is permissive
|
|
471
|
+
* here, which makes a duplicate worse than a gap.
|
|
472
|
+
* 3. **The fence check is atomic with the write.** Reading the current
|
|
473
|
+
* fence and then writing is check-then-act, and the gap is a race. A
|
|
474
|
+
* database gets this free (`UPDATE … WHERE fence >= ?`); a filesystem
|
|
475
|
+
* does not, and the built-in disk store narrows rather than closes it.
|
|
476
|
+
* 4. **`holder` is unique per process.** It is evidence rather than
|
|
477
|
+
* authority, but it is the only thing distinguishing a RENEWAL from a
|
|
478
|
+
* theft — two workers sharing a holder string take a live, unexpired
|
|
479
|
+
* claim from each other instantly. Use something per-process, not a
|
|
480
|
+
* per-deployment name.
|
|
481
|
+
*/
|
|
482
|
+
claimRun?(scope: CheckpointRunScope, options: ClaimRunOptions): Promise<RunClaim | null>;
|
|
483
|
+
/**
|
|
484
|
+
* Give a claim up early. Idempotent: releasing a claim that already
|
|
485
|
+
* expired, was superseded, or never existed succeeds as a no-op.
|
|
486
|
+
*
|
|
487
|
+
* Presenting a stale fence releases NOTHING — a worker that stalled past
|
|
488
|
+
* its lease must not be able to hand away a run somebody else is now
|
|
489
|
+
* holding, and that is the same fencing comparison the write path makes.
|
|
490
|
+
*
|
|
491
|
+
* Optional to call, not optional to matter: a worker that finishes and
|
|
492
|
+
* releases returns the run to the queue immediately, where one that just
|
|
493
|
+
* exits leaves it stuck until the lease expires. That is a latency
|
|
494
|
+
* difference, never a correctness one, which is the property that lets a
|
|
495
|
+
* crashed worker be indistinguishable from a slow one.
|
|
496
|
+
*/
|
|
497
|
+
releaseRun?(scope: CheckpointRunScope, fence: ClaimFence): Promise<void>;
|
|
322
498
|
}
|
|
323
499
|
//# sourceMappingURL=checkpoint-store.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"checkpoint-store.d.ts","sourceRoot":"","sources":["../../../src/types/run/checkpoint-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAA;AAC9F,OAAO,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAA;AACjE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAA;AAElD;;;;;;;;GAQG;AACH,MAAM,WAAW,kBAAkB;IAClC,2CAA2C;IAC3C,QAAQ,EAAE,QAAQ,CAAA;IAClB,gDAAgD;IAChD,SAAS,EAAE,SAAS,CAAA;IACpB,wCAAwC;IACxC,SAAS,EAAE,SAAS,CAAA;IACpB,2CAA2C;IAC3C,KAAK,EAAE,KAAK,CAAA;IACZ;;;;OAIG;IACH,WAAW,CAAC,EAAE,KAAK,CAAA;CACnB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,WAAW,sBAAsB;IACtC,2DAA2D;IAC3D,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAA;IAC3B,mEAAmE;IACnE,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,CAAA;IAC9B,mDAAmD;IACnD,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,CAAA;CAC9B;AAED;;;;;;;;GAQG;AACH,MAAM,MAAM,SAAS;AACpB,mFAAmF;AACjF,aAAa;AACf,8EAA8E;GAC5E,SAAS;AACX,6EAA6E;GAC3E,UAAU,CAAA;AAEb;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,+EAA+E;AAC/E,MAAM,WAAW,WAAW;IAC3B,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAA;IACzB,yEAAyE;IACzE,QAAQ,CAAC,YAAY,EAAE,YAAY,CAAA;IACnC,gFAAgF;IAChF,QAAQ,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,CAAA;IACjD,wCAAwC;IACxC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,kDAAkD;IAClD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;IAC5B,gEAAgE;IAChE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAC5B;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,WAAW,eAAgB,SAAQ,kBAAkB;IAC1D;;;;;;;;;;;;OAYG;IACH,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAA;IAE9B,qEAAqE;IACrE,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAA;IAChC,+EAA+E;IAC/E,QAAQ,CAAC,kBAAkB,EAAE,YAAY,CAAA;IACzC,iEAAiE;IACjE,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAA;IACnC,4CAA4C;IAC5C,QAAQ,CAAC,IAAI,CAAC,EAAE,WAAW,CAAA;
|
|
1
|
+
{"version":3,"file":"checkpoint-store.d.ts","sourceRoot":"","sources":["../../../src/types/run/checkpoint-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAA;AAC9F,OAAO,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAA;AACjE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAA;AAElD;;;;;;;;GAQG;AACH,MAAM,WAAW,kBAAkB;IAClC,2CAA2C;IAC3C,QAAQ,EAAE,QAAQ,CAAA;IAClB,gDAAgD;IAChD,SAAS,EAAE,SAAS,CAAA;IACpB,wCAAwC;IACxC,SAAS,EAAE,SAAS,CAAA;IACpB,2CAA2C;IAC3C,KAAK,EAAE,KAAK,CAAA;IACZ;;;;OAIG;IACH,WAAW,CAAC,EAAE,KAAK,CAAA;CACnB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,WAAW,sBAAsB;IACtC,2DAA2D;IAC3D,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAA;IAC3B,mEAAmE;IACnE,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,CAAA;IAC9B,mDAAmD;IACnD,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,CAAA;CAC9B;AAED;;;;;;;;GAQG;AACH,MAAM,MAAM,SAAS;AACpB,mFAAmF;AACjF,aAAa;AACf,8EAA8E;GAC5E,SAAS;AACX,6EAA6E;GAC3E,UAAU,CAAA;AAEb;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,+EAA+E;AAC/E,MAAM,WAAW,WAAW;IAC3B,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAA;IACzB,yEAAyE;IACzE,QAAQ,CAAC,YAAY,EAAE,YAAY,CAAA;IACnC,gFAAgF;IAChF,QAAQ,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,CAAA;IACjD,wCAAwC;IACxC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,kDAAkD;IAClD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;IAC5B,gEAAgE;IAChE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAC5B;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,WAAW,eAAgB,SAAQ,kBAAkB;IAC1D;;;;;;;;;;;;OAYG;IACH,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAA;IAE9B,qEAAqE;IACrE,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAA;IAChC,+EAA+E;IAC/E,QAAQ,CAAC,kBAAkB,EAAE,YAAY,CAAA;IACzC,iEAAiE;IACjE,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAA;IACnC,4CAA4C;IAC5C,QAAQ,CAAC,IAAI,CAAC,EAAE,WAAW,CAAA;IAE3B;;;;;;;;OAQG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,CAAA;CAC7B;AAED;;;;;;;;GAQG;AACH,MAAM,MAAM,eAAe;AAC1B;;;GAGG;AACD,OAAO;AACT;;;;;;;;;;;;;GAaG;GACD,WAAW,CAAA;AAEd;;;;;;;;;;;;;;GAcG;AACH,MAAM,MAAM,UAAU,GAAG,MAAM,CAAA;AAE/B,yEAAyE;AACzE,MAAM,WAAW,QAAQ;IACxB,0FAA0F;IAC1F,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,iEAAiE;IACjE,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAA;IAC1B;;;;;;OAMG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;CAC1B;AAED,6CAA6C;AAC7C,MAAM,WAAW,YAAY;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAA;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B;;;;;;;OAOG;IACH,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;CACzB;AAED,gDAAgD;AAChD,MAAM,WAAW,eAAe;IAC/B,gFAAgF;IAChF,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,+CAA+C;IAC/C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,qFAAqF;IACrF,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CACrB;AAED,sEAAsE;AACtE,MAAM,WAAW,sBAAsB;IACtC;;;;OAIG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,eAAe,CAAA;IAElC;;;;OAIG;IACH,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,SAAS,EAAE,CAAA;IACpC;;;;;;;;OAQG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAA;IAC1B,yDAAyD;IACzD,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;IACvB,2EAA2E;IAC3E,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB;;;;OAIG;IACH,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CACrB;AAED,2CAA2C;AAC3C,MAAM,WAAW,cAAc;IAC9B,QAAQ,CAAC,OAAO,EAAE,SAAS,eAAe,EAAE,CAAA;IAC5C;;;;OAIG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CACxB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,MAAM,WAAW,eAAe;IAC/B;;;;;;;;;;;;;;;OAeG;IACH,eAAe,CACd,KAAK,EAAE,kBAAkB,EACzB,UAAU,EAAE,mBAAmB,EAC/B,KAAK,CAAC,EAAE,UAAU,GAChB,OAAO,CAAC,IAAI,CAAC,CAAA;IAEhB,6EAA6E;IAC7E,cAAc,CACb,KAAK,EAAE,kBAAkB,EACzB,YAAY,EAAE,YAAY,GACxB,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAA;IAEtC;;;;OAIG;IACH,eAAe,CAAC,KAAK,EAAE,kBAAkB,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAAA;IAE1E,wEAAwE;IACxE,gBAAgB,CAAC,KAAK,EAAE,kBAAkB,EAAE,YAAY,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAEtF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAuCG;IACH,eAAe,CAAC,CACf,KAAK,EAAE,sBAAsB,EAC7B,OAAO,CAAC,EAAE,sBAAsB,GAC9B,OAAO,CAAC,cAAc,CAAC,CAAA;IAE1B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAqEG;IACH,QAAQ,CAAC,CAAC,KAAK,EAAE,kBAAkB,EAAE,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAA;IAExF;;;;;;;;;;;;;OAaG;IACH,UAAU,CAAC,CAAC,KAAK,EAAE,kBAAkB,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CACxE"}
|
package/package.json
CHANGED
package/src/public-runtime.ts
CHANGED
|
@@ -218,6 +218,11 @@ export {
|
|
|
218
218
|
paginateDurableRuns,
|
|
219
219
|
toDurableRunEntry,
|
|
220
220
|
} from './store/index.js'
|
|
221
|
+
// Cross-process possession of a run. `claimRun` REFUSES on a store that
|
|
222
|
+
// cannot arbitrate rather than proceeding unclaimed, because proceeding lets
|
|
223
|
+
// two workers restore one checkpoint, both run its tools and both write under
|
|
224
|
+
// one run id — which loses half the work and reports nothing.
|
|
225
|
+
export { claimRun, fencedOut, releaseRun, toClaimSummary } from './store/index.js'
|
|
221
226
|
|
|
222
227
|
export {
|
|
223
228
|
AgentRegistry,
|
|
@@ -11,7 +11,11 @@ import type {
|
|
|
11
11
|
PendingDecision,
|
|
12
12
|
} from '../../types/hitl/index.js'
|
|
13
13
|
import type { AssistantMessage } from '../../types/message/index.js'
|
|
14
|
-
import type {
|
|
14
|
+
import type {
|
|
15
|
+
CheckpointRunScope,
|
|
16
|
+
CheckpointStore,
|
|
17
|
+
ClaimFence,
|
|
18
|
+
} from '../../types/run/checkpoint-store.js'
|
|
15
19
|
import type { EmergencySaveData } from '../../types/run/emergency.js'
|
|
16
20
|
import type { CheckpointListEntry } from '../../types/run/replay.js'
|
|
17
21
|
import { ZERO_COST } from '../../utils/cost.js'
|
|
@@ -154,6 +158,22 @@ export class CheckpointManager {
|
|
|
154
158
|
/** See {@link setParkTtl}. */
|
|
155
159
|
private parkTtlMs?: number
|
|
156
160
|
|
|
161
|
+
/**
|
|
162
|
+
* The claim this run holds, presented on every checkpoint write.
|
|
163
|
+
*
|
|
164
|
+
* Unset means unfenced, which is correct for a single-writer deployment
|
|
165
|
+
* and is what every run did before claims existed. Set it and a write from
|
|
166
|
+
* a superseded holding is refused by the store.
|
|
167
|
+
*
|
|
168
|
+
* This existed nowhere for one release, and the omission was invisible in
|
|
169
|
+
* the worst way: the claim, the fence and the refusal were all built and
|
|
170
|
+
* tested, and no code path between a run and its store carried the number,
|
|
171
|
+
* so every checkpoint a RUN wrote went out unfenced. A capability that is
|
|
172
|
+
* complete except for the wire between its halves reads exactly like a
|
|
173
|
+
* working one.
|
|
174
|
+
*/
|
|
175
|
+
private claimFence?: ClaimFence
|
|
176
|
+
|
|
157
177
|
/**
|
|
158
178
|
* The run's attribution instant, stamped onto every checkpoint this
|
|
159
179
|
* manager writes.
|
|
@@ -234,7 +254,7 @@ export class CheckpointManager {
|
|
|
234
254
|
traceContext: this.traceSource?.(),
|
|
235
255
|
}
|
|
236
256
|
|
|
237
|
-
await this.store.writeCheckpoint(this.scope, checkpoint)
|
|
257
|
+
await this.store.writeCheckpoint(this.scope, checkpoint, this.claimFence)
|
|
238
258
|
this.lastCreatedId = checkpoint.id
|
|
239
259
|
return checkpoint
|
|
240
260
|
}
|
|
@@ -298,10 +318,27 @@ export class CheckpointManager {
|
|
|
298
318
|
...(ttl !== undefined && ttl > 0 ? { deadlineAt: parkedAt + ttl } : {}),
|
|
299
319
|
},
|
|
300
320
|
}
|
|
301
|
-
await this.store.writeCheckpoint(this.scope, parked)
|
|
321
|
+
await this.store.writeCheckpoint(this.scope, parked, this.claimFence)
|
|
302
322
|
return parked
|
|
303
323
|
}
|
|
304
324
|
|
|
325
|
+
/**
|
|
326
|
+
* Present this claim on every subsequent write. See {@link claimFence}.
|
|
327
|
+
*
|
|
328
|
+
* A setter rather than a constructor argument because a run is claimed at
|
|
329
|
+
* a different moment than it is constructed — a worker draining a queue
|
|
330
|
+
* takes the run, then builds the pipeline around it — and because a
|
|
331
|
+
* renewal mints a NEW fence mid-run that has to replace the old one.
|
|
332
|
+
*/
|
|
333
|
+
setClaimFence(fence: ClaimFence | undefined): void {
|
|
334
|
+
this.claimFence = fence
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
/** The claim currently presented on writes, if any. */
|
|
338
|
+
get presentedFence(): ClaimFence | undefined {
|
|
339
|
+
return this.claimFence
|
|
340
|
+
}
|
|
341
|
+
|
|
305
342
|
/** Default time-to-live applied to every park this manager records. */
|
|
306
343
|
setParkTtl(ttlMs: number | undefined): void {
|
|
307
344
|
this.parkTtlMs = ttlMs
|
|
@@ -329,7 +366,7 @@ export class CheckpointManager {
|
|
|
329
366
|
decision: { action: 'pause', reason: 'The approval request expired without an answer.' },
|
|
330
367
|
},
|
|
331
368
|
}
|
|
332
|
-
await this.store.writeCheckpoint(this.scope, expired)
|
|
369
|
+
await this.store.writeCheckpoint(this.scope, expired, this.claimFence)
|
|
333
370
|
return expired
|
|
334
371
|
}
|
|
335
372
|
|
|
@@ -352,7 +389,7 @@ export class CheckpointManager {
|
|
|
352
389
|
...checkpoint,
|
|
353
390
|
pending: { ...checkpoint.pending, resolvedAt: Date.now(), decision },
|
|
354
391
|
}
|
|
355
|
-
await this.store.writeCheckpoint(this.scope, resolved)
|
|
392
|
+
await this.store.writeCheckpoint(this.scope, resolved, this.claimFence)
|
|
356
393
|
return resolved
|
|
357
394
|
}
|
|
358
395
|
|
|
@@ -58,7 +58,7 @@ import type { AgentPersona } from '../../types/persona/index.js'
|
|
|
58
58
|
import type { LLMProvider } from '../../types/provider/index.js'
|
|
59
59
|
import type { TaskRouterConfig } from '../../types/router/index.js'
|
|
60
60
|
import type { ReviewAnswer } from '../../types/run/answer-review.js'
|
|
61
|
-
import type { CheckpointStore } from '../../types/run/checkpoint-store.js'
|
|
61
|
+
import type { CheckpointStore, ClaimFence } from '../../types/run/checkpoint-store.js'
|
|
62
62
|
import type {
|
|
63
63
|
AgentRunConfig,
|
|
64
64
|
PrepareStepChain,
|
|
@@ -413,6 +413,27 @@ export interface QueryParams {
|
|
|
413
413
|
*/
|
|
414
414
|
checkpointStore?: CheckpointStore
|
|
415
415
|
|
|
416
|
+
/**
|
|
417
|
+
* The fence of the claim this worker holds on the run, from `claimRun`.
|
|
418
|
+
*
|
|
419
|
+
* Presented on every checkpoint the run writes, so a worker that stalled
|
|
420
|
+
* past its lease is refused rather than writing into a run somebody else
|
|
421
|
+
* has taken over. Omit it for single-writer deployments, which is what
|
|
422
|
+
* every run did before claims existed.
|
|
423
|
+
*
|
|
424
|
+
* This hop did not exist for a release. The claim, the fence and the
|
|
425
|
+
* store-side refusal were all built and tested, and no path between a run
|
|
426
|
+
* and its store carried the number — so every checkpoint a RUN wrote went
|
|
427
|
+
* out unfenced while the tests, which called the store directly, all
|
|
428
|
+
* passed. A capability complete except for the wire between its halves
|
|
429
|
+
* reads exactly like a working one.
|
|
430
|
+
*
|
|
431
|
+
* It fences checkpoints and nothing else. {@link QueryParams.runStore}
|
|
432
|
+
* takes no fence, so two workers that both took one run still overwrite
|
|
433
|
+
* each other's run record, transcript and report — see the changeset.
|
|
434
|
+
*/
|
|
435
|
+
claimFence?: ClaimFence
|
|
436
|
+
|
|
416
437
|
/**
|
|
417
438
|
* Where this run records its own evidence — the run record, its messages,
|
|
418
439
|
* its transcript and its report. Defaults to the disk layout under the
|
|
@@ -1013,6 +1034,10 @@ export async function* query(params: QueryParams): AsyncGenerator<RunEvent, Run>
|
|
|
1013
1034
|
// And every park it records carries an absolute deadline, so an
|
|
1014
1035
|
// unanswered approval cannot outlive the worker that asked for it.
|
|
1015
1036
|
checkpointMgr.setParkTtl(params.runConfig.hitlParkTtlMs)
|
|
1037
|
+
// The claim this worker holds, if it took one. Without this hop the
|
|
1038
|
+
// fence exists, the refusal exists, and no checkpoint a RUN writes ever
|
|
1039
|
+
// carries a number — so a stalled worker is refused nowhere.
|
|
1040
|
+
checkpointMgr.setClaimFence(params.claimFence)
|
|
1016
1041
|
|
|
1017
1042
|
// A question raised from inside a tool becomes a real checkpoint
|
|
1018
1043
|
// here. It used to park under a synthetic id nothing ever wrote, so
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { PendingDecision } from '../../types/hitl/index.js'
|
|
2
|
-
import type { CheckpointStore } from '../../types/run/checkpoint-store.js'
|
|
2
|
+
import type { CheckpointStore, ClaimFence } from '../../types/run/checkpoint-store.js'
|
|
3
3
|
import type { Run } from '../../types/run/entity.js'
|
|
4
4
|
import type { RunState } from '../../types/run/state.js'
|
|
5
5
|
import { type QueryParams, drainQuery } from './index.js'
|
|
@@ -38,6 +38,15 @@ export interface ResumeRunParams
|
|
|
38
38
|
readonly scope: RunStateScope
|
|
39
39
|
/** Required to find the checkpoint; also threaded into the resumed run. */
|
|
40
40
|
readonly checkpointStore: CheckpointStore
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* The fence of the claim this worker took on the run before resuming.
|
|
44
|
+
*
|
|
45
|
+
* A resume is the one moment two workers are most likely to collide — it
|
|
46
|
+
* is what a queue reader does with a parked run — so this is the call that
|
|
47
|
+
* most needs to carry one.
|
|
48
|
+
*/
|
|
49
|
+
readonly claimFence?: ClaimFence
|
|
41
50
|
/**
|
|
42
51
|
* Resume a specific checkpoint instead of the one the store would pick.
|
|
43
52
|
* Absent means the parked checkpoint if there is one, else the newest.
|
|
@@ -67,7 +76,7 @@ export interface ResumeRunParams
|
|
|
67
76
|
* resumed past without the answer it is waiting for.
|
|
68
77
|
*/
|
|
69
78
|
export async function resumeRun(params: ResumeRunParams): Promise<ResumeOutcome> {
|
|
70
|
-
const { scope, checkpointStore, checkpointId, pendingDecision, ...rest } = params
|
|
79
|
+
const { scope, checkpointStore, checkpointId, pendingDecision, claimFence, ...rest } = params
|
|
71
80
|
|
|
72
81
|
const state = await loadRunState(checkpointStore, scope, checkpointId)
|
|
73
82
|
if (!state?.checkpointId) return { resumed: false, reason: 'no-checkpoint' }
|
|
@@ -86,6 +95,7 @@ export async function resumeRun(params: ResumeRunParams): Promise<ResumeOutcome>
|
|
|
86
95
|
runId: state.runId,
|
|
87
96
|
resumeFromCheckpoint: state.checkpointId,
|
|
88
97
|
checkpointStore,
|
|
98
|
+
...(claimFence !== undefined ? { claimFence } : {}),
|
|
89
99
|
...(pendingDecision ? { pendingDecision } : {}),
|
|
90
100
|
} as QueryParams)
|
|
91
101
|
|
package/src/store/index.ts
CHANGED
|
@@ -18,8 +18,12 @@ export { InMemoryCheckpointStore } from './run/checkpoint-memory.js'
|
|
|
18
18
|
// host has no use for is surface to keep correct forever for nobody.
|
|
19
19
|
export {
|
|
20
20
|
assertContiguousListingScope,
|
|
21
|
+
claimRun,
|
|
22
|
+
fencedOut,
|
|
21
23
|
listDurableRuns,
|
|
22
24
|
paginateDurableRuns,
|
|
25
|
+
releaseRun,
|
|
26
|
+
toClaimSummary,
|
|
23
27
|
toDurableRunEntry,
|
|
24
28
|
} from './run/listing.js'
|
|
25
29
|
|