@namzu/sdk 20.3.0 → 20.4.0

package/src/store/run/listing.ts

@@ -14,12 +14,16 @@ import type {
 	CheckpointListingScope,
 	CheckpointRunScope,
 	CheckpointStore,
+	ClaimFence,
+	ClaimRunOptions,
+	ClaimSummary,
 	DurableRunEntry,
 	DurableRunOrder,
 	DurableRunPage,
 	ListDurableRunsOptions,
 	ParkState,
 	ParkSummary,
+	RunClaim,
 } from '../../types/run/checkpoint-store.js'
 
 /** Page size when the caller names none. */
@@ -186,11 +190,20 @@ export function paginateDurableRuns(
 	options?: ListDurableRunsOptions,
 ): DurableRunPage {
 	const wanted = options?.park
-	const filtered =
+	const byPark =
 		wanted && wanted.length > 0
 			? entries.filter((e) => e.park !== undefined && wanted.includes(e.park.state))
 			: entries
 
+	// An expired claim counts as unheld. That is what expiry means, and a
+	// queue reader that treated an expired claim as held would leave a dead
+	// worker's runs invisible forever — the exact failure a lease exists to
+	// prevent, reintroduced by the filter that reads it.
+	const filtered =
+		options?.claimed === undefined
+			? byPark
+			: byPark.filter((e) => (e.claim !== undefined && !e.claim.expired) === options.claimed)
+
 	// Both orders sort on a key that cannot move under a paging caller — see
 	// the contract comment on `listDurableRuns`.
 	const orderBy = options?.orderBy ?? 'runId'
@@ -305,3 +318,105 @@ export async function listDurableRuns(
 	assertContiguousListingScope(scope, 'listDurableRuns')
 	return store.listDurableRuns(scope, options)
 }
+
+/**
+ * Take working possession of a run, refusing when the store cannot arbitrate.
+ *
+ * The refusal is the entire safety property. `claimRun` is optional on the
+ * contract, and the natural way to reach an absent optional method is to skip
+ * it — which here means every worker proceeds, believing it holds a run
+ * nobody arbitrated. Two workers then restore the same checkpoint, both run
+ * the tools, and both write under one run id; half the work vanishes with no
+ * error anywhere.
+ *
+ * So a store with no claim support does not get "claimed by default". It gets
+ * an error naming the deployment shape it cannot support. A single-writer
+ * host never calls this and is unaffected.
+ *
+ * Returns `null` — not an error — when another holder has the run. That is
+ * the ordinary outcome of two readers on one queue, and a caller loops to the
+ * next run rather than handling a fault.
+ */
+export async function claimRun(
+	store: CheckpointStore,
+	scope: CheckpointRunScope,
+	options: ClaimRunOptions,
+): Promise<RunClaim | null> {
+	if (typeof store.claimRun !== 'function') {
+		throw new NamzuError({
+			code: 'capability_unavailable',
+			message:
+				'claimRun: the injected checkpoint store does not implement `claimRun`, so it cannot arbitrate between two workers taking the same run. Refusing rather than proceeding unclaimed — proceeding would let two workers restore one checkpoint, both execute its tools, and both write under one run id, which loses half the work and reports nothing. Supply a store that implements it, or run a single writer per run.',
+			details: { runId: scope.runId },
+		})
+	}
+	if (!Number.isFinite(options.ttlMs) || options.ttlMs <= 0) {
+		throw new NamzuError({
+			code: 'invalid_config',
+			message: `claimRun: ttlMs must be a positive number of milliseconds, got ${String(options.ttlMs)}. A lease that expires immediately is a lease every worker can take at once, which is the condition this call exists to prevent.`,
+			details: { runId: scope.runId, ttlMs: options.ttlMs },
+		})
+	}
+	return store.claimRun(scope, options)
+}
+
+/**
+ * Give a claim up early, refusing when the store cannot arbitrate.
+ *
+ * Refuses for the same reason as {@link claimRun}: a host that believes it is
+ * releasing a claim on a store that has none is a host that believes the
+ * whole mechanism is running.
+ */
+export async function releaseRun(
+	store: CheckpointStore,
+	scope: CheckpointRunScope,
+	fence: ClaimFence,
+): Promise<void> {
+	if (typeof store.releaseRun !== 'function') {
+		throw new NamzuError({
+			code: 'capability_unavailable',
+			message:
+				'releaseRun: the injected checkpoint store does not implement `releaseRun`. A release that silently does nothing would leave the run held until its lease expires while the caller believes it is back on the queue.',
+			details: { runId: scope.runId },
+		})
+	}
+	return store.releaseRun(scope, fence)
+}
+
+/**
+ * The refusal a store raises when a write presents a superseded fence.
+ *
+ * Shared so both shipped stores say the same thing, and so a host writing its
+ * own backend raises something a caller can branch on rather than a message
+ * string. This is the moment a stalled worker learns it lost the run — the
+ * only moment it CAN learn, since from the inside a pause and a partition
+ * both look like time not passing.
+ */
+export function fencedOut(
+	scope: CheckpointRunScope,
+	presented: number,
+	current: number,
+): NamzuError {
+	return new NamzuError({
+		code: 'storage_error',
+		message: `writeCheckpoint: refusing a write for run ${scope.runId} fenced at ${presented} — the run is now claimed at ${current}. Another worker took this run over, so this process no longer holds it and its work is not the record. Stop the run rather than retrying: the claim is gone, not busy.`,
+		details: { runId: scope.runId, presentedFence: presented, currentFence: current },
+		retryable: false,
+	})
+}
+
+/**
+ * Whether a recorded claim still holds at `now`, and the summary a listing
+ * reports for it.
+ */
+export function toClaimSummary(claim: RunClaim, now: number): ClaimSummary {
+	return {
+		holder: claim.holder,
+		fence: claim.fence,
+		expiresAt: claim.expiresAt,
+		// Judged here, once, against the store's clock. Left to the caller it
+		// would be judged against a different one, and a single page could
+		// then disagree with itself about which rows are available.
+		expired: now >= claim.expiresAt,
+	}
+}

package/src/types/run/checkpoint-store.ts

@@ -175,6 +175,17 @@ export interface DurableRunEntry extends CheckpointRunScope {
 	readonly latestCheckpointAt: number
 	/** Absent when the run has never parked. */
 	readonly park?: ParkSummary
+
+	/**
+	 * Absent when no process has ever claimed the run.
+	 *
+	 * A SIBLING of {@link DurableRunEntry.park}, not a member of
+	 * {@link ParkState} — see the note on that union. A park is a question put
+	 * to a human; a claim is a lease held by a process. A run can have both,
+	 * neither, or either, and the state a queue worker needs most is parked
+	 * AND unclaimed, which one union cannot say.
+	 */
+	readonly claim?: ClaimSummary
 }
 
 /**
@@ -208,6 +219,65 @@ export type DurableRunOrder =
 	 */
 	| 'createdAt'
 
+/**
+ * A monotonically increasing number identifying one holding of a run's claim.
+ *
+ * The load-bearing word is *fencing*. A mutex answers "may I proceed", and a
+ * holder that stalls past its lease — a long GC pause, a suspended container,
+ * a partitioned network — answers it "yes" and then writes, long after
+ * somebody else legitimately took over. A fence answers a different question
+ * at the moment of the WRITE: "is the holding I belong to still the current
+ * one". Every claim of a run mints a number strictly greater than the last,
+ * so a store can reject a write from a superseded holder without knowing
+ * anything about processes, clocks or liveness.
+ *
+ * Not a random token, deliberately: randomness proves identity and cannot
+ * establish *order*, and order is the entire mechanism.
+ */
+export type ClaimFence = number
+
+/** A holding of a run's claim, as issued to the process that took it. */
+export interface RunClaim {
+	/** Opaque caller-supplied identity — a worker id, a pod name. Evidence, not authority. */
+	readonly holder: string
+	/** See {@link ClaimFence}. Present it on every durable write. */
+	readonly fence: ClaimFence
+	/**
+	 * Absolute epoch ms after which the claim may be taken by somebody else.
+	 *
+	 * Absolute rather than a duration for the same reason a park's deadline
+	 * is: it has to survive the process that set it. A duration plus an
+	 * in-process timer cannot — the holder is the thing that dies.
+	 */
+	readonly expiresAt: number
+}
+
+/** A run's claim as a listing reports it. */
+export interface ClaimSummary {
+	readonly holder: string
+	readonly fence: ClaimFence
+	readonly expiresAt: number
+	/**
+	 * Whether the claim had expired at the instant the listing was taken.
+	 *
+	 * A separate field rather than something the caller derives from
+	 * `expiresAt`, because the caller would derive it against a DIFFERENT
+	 * clock than the store used, and one page would then disagree with
+	 * itself about which rows are available.
+	 */
+	readonly expired: boolean
+}
+
+/** What a caller asks for when taking a run. */
+export interface ClaimRunOptions {
+	/** Who is taking it. Recorded so an operator can see what holds a stuck run. */
+	readonly holder: string
+	/** How long the holding is good for, in ms. */
+	readonly ttlMs: number
+	/** Clock, for tests and so one operation judges every expiry against one instant. */
+	readonly now?: number
+}
+
 /** Filters and paging for {@link CheckpointStore.listDurableRuns}. */
 export interface ListDurableRunsOptions {
 	/**
@@ -223,6 +293,16 @@ export interface ListDurableRunsOptions {
 	 * to include it.
 	 */
 	readonly park?: readonly ParkState[]
+	/**
+	 * Keep only runs that are, or are not, currently held by a worker.
+	 *
+	 * `false` is the queue-reader's filter: give me the work nobody has. A
+	 * claim that has expired counts as NOT held, because that is what expiry
+	 * means and a reader that skipped expired claims would leave a dead
+	 * worker's runs invisible forever — the exact failure the lease exists to
+	 * prevent.
+	 */
+	readonly claimed?: boolean
 	/** Page size. Defaults to 100, clamped to at least 1. */
 	readonly limit?: number
 	/** Resume token from the previous page's {@link DurableRunPage.cursor}. */
@@ -279,8 +359,27 @@ export interface DurableRunPage {
  * proceed.
  */
 export interface CheckpointStore {
-	/** Persist one checkpoint. Overwrites an existing checkpoint with the same id. */
-	writeCheckpoint(scope: CheckpointRunScope, checkpoint: IterationCheckpoint): Promise<void>
+	/**
+	 * Persist one checkpoint. Overwrites an existing checkpoint with the same id.
+	 *
+	 * @param fence the {@link ClaimFence} of the holding this write belongs
+	 *   to, when the run is claimed. A store that supports claims REFUSES a
+	 *   write whose fence is below the run's current one — that refusal is
+	 *   what makes a claim a lease rather than a suggestion, because a holder
+	 *   stalled past its expiry believes it still holds and is wrong only at
+	 *   the moment it writes.
+	 *
+	 *   Omit it and the write is unfenced, which is exactly today's behaviour
+	 *   and correct for a single-writer deployment. A store MUST NOT start
+	 *   refusing unfenced writes because some other write carried a fence:
+	 *   that would make adding a claim to one worker break every worker that
+	 *   has not adopted it yet.
+	 */
+	writeCheckpoint(
+		scope: CheckpointRunScope,
+		checkpoint: IterationCheckpoint,
+		fence?: ClaimFence,
+	): Promise<void>
 
 	/** Load a single checkpoint by id. Returns `null` when it does not exist. */
 	readCheckpoint(
@@ -342,4 +441,92 @@ export interface CheckpointStore {
 		scope: CheckpointListingScope,
 		options?: ListDurableRunsOptions,
 	): Promise<DurableRunPage>
+
+	/**
+	 * Take exclusive working possession of a run, or report that somebody
+	 * else has it. OPTIONAL — see the optional-capability rule on this
+	 * interface.
+	 *
+	 * Returns the holding on success and `null` when the run is currently
+	 * held by somebody else. `null` is not an error: "another worker got
+	 * there first" is the ordinary outcome of a queue with more than one
+	 * reader, and a thrown exception would make the normal case look like a
+	 * fault.
+	 *
+	 * ### What it is for
+	 *
+	 * Putting parked runs on a queue and letting more than one worker drain
+	 * it. Without this, two workers restore the same checkpoint, both execute
+	 * the run's tools, and both write checkpoints under one run id — each
+	 * write minting a fresh checkpoint id, so two divergent chains land in
+	 * one list and the pending lookup returns whichever wrote last. Half the
+	 * work vanishes and nothing reports an error.
+	 *
+	 * ### The lease, and why it expires
+	 *
+	 * A claim is a LEASE, not a lock. A lock held by a process that dies is
+	 * held forever, and the runs behind it are unreachable by anything except
+	 * a human with a shell. The expiry is what makes a dead holder's work
+	 * recoverable without one.
+	 *
+	 * The expiry is also why a fence exists. A holder does not know it has
+	 * expired — a long pause, a suspended container and a partition all look
+	 * from the inside like time not passing — so it wakes and writes as
+	 * though it still holds. Liveness cannot be checked from here. What CAN
+	 * be checked, at the write, is whether the holding that write belongs to
+	 * is still the current one, and that is a comparison of two numbers.
+	 *
+	 * ### Reclaiming
+	 *
+	 * Calling this on a run whose claim has expired SUCCEEDS and mints a
+	 * fence strictly greater than the expired holding's. The previous holder
+	 * is not notified — it cannot be, that is the premise — it simply stops
+	 * being able to write.
+	 *
+	 * Calling it again as the CURRENT holder also succeeds and extends the
+	 * lease, minting a new fence. Renewal and reclamation are the same
+	 * operation from the store's side, which is why there is no separate
+	 * `renew`: two code paths that must agree about who holds a run is one
+	 * more than can be kept correct.
+	 *
+	 * ### What a backend implementing this MUST guarantee
+	 *
+	 * These are the properties the fence comparison depends on. None of them
+	 * is checkable from here, and every one of them was violated by the first
+	 * built-in implementation, so they are written down rather than assumed:
+	 *
+	 *  1. **A fence exceeds every fence ever issued for the run** — including
+	 *     across a release, and across deletion of whatever recorded it. A
+	 *     counter that rewinds re-issues a number a stalled worker still
+	 *     believes it holds, and that worker's writes become legal again.
+	 *  2. **Fences are unique.** The write check is `fence < current`, so two
+	 *     holders at one number are both admitted. Equality is permissive
+	 *     here, which makes a duplicate worse than a gap.
+	 *  3. **The fence check is atomic with the write.** Reading the current
+	 *     fence and then writing is check-then-act, and the gap is a race. A
+	 *     database gets this free (`UPDATE … WHERE fence >= ?`); a filesystem
+	 *     does not, and the built-in disk store narrows rather than closes it.
+	 *  4. **`holder` is unique per process.** It is evidence rather than
+	 *     authority, but it is the only thing distinguishing a RENEWAL from a
+	 *     theft — two workers sharing a holder string take a live, unexpired
+	 *     claim from each other instantly. Use something per-process, not a
+	 *     per-deployment name.
+	 */
+	claimRun?(scope: CheckpointRunScope, options: ClaimRunOptions): Promise<RunClaim | null>
+
+	/**
+	 * Give a claim up early. Idempotent: releasing a claim that already
+	 * expired, was superseded, or never existed succeeds as a no-op.
+	 *
+	 * Presenting a stale fence releases NOTHING — a worker that stalled past
+	 * its lease must not be able to hand away a run somebody else is now
+	 * holding, and that is the same fencing comparison the write path makes.
+	 *
+	 * Optional to call, not optional to matter: a worker that finishes and
+	 * releases returns the run to the queue immediately, where one that just
+	 * exits leaves it stuck until the lease expires. That is a latency
+	 * difference, never a correctness one, which is the property that lets a
+	 * crashed worker be indistinguishable from a slow one.
+	 */
+	releaseRun?(scope: CheckpointRunScope, fence: ClaimFence): Promise<void>
 }