@namzu/sdk 20.3.0 → 20.4.0

package/src/store/run/checkpoint-disk.ts

@@ -7,14 +7,24 @@ import type {
 	CheckpointListingScope,
 	CheckpointRunScope,
 	CheckpointStore,
+	ClaimFence,
+	ClaimRunOptions,
 	DurableRunEntry,
 	DurableRunPage,
 	ListDurableRunsOptions,
+	RunClaim,
 } from '../../types/run/checkpoint-store.js'
 import type { RunStoreConfig } from '../../types/run/index.js'
 import type { ProjectId } from '../../types/session/ids.js'
+import { acquireClaim, currentFence, readClaim, releaseClaim } from './claim-disk.js'
 import { RunDiskStore, readCheckpointsIn } from './disk.js'
-import { assertContiguousListingScope, paginateDurableRuns, toDurableRunEntry } from './listing.js'
+import {
+	assertContiguousListingScope,
+	fencedOut,
+	paginateDurableRuns,
+	toClaimSummary,
+	toDurableRunEntry,
+} from './listing.js'
 
 /**
  * The attribution a disk store's own layout does not record.
@@ -84,11 +94,47 @@ export class DiskCheckpointStore implements CheckpointStore {
 		return promise
 	}
 
-	async writeCheckpoint(scope: CheckpointRunScope, checkpoint: IterationCheckpoint): Promise<void> {
+	async writeCheckpoint(
+		scope: CheckpointRunScope,
+		checkpoint: IterationCheckpoint,
+		fence?: ClaimFence,
+	): Promise<void> {
 		const store = await this.bind(scope)
+		if (fence !== undefined) {
+			// Read at the moment of the write, not at the start of the run.
+			// A holder that stalled past its lease believes it still holds,
+			// and this is the only point at which it can be told otherwise.
+			// Names only, no parsing. The fence is the file name, so a corrupt
+			// body cannot make this check skip itself — which the previous shape
+			// did, at the one site whose entire job is refusing.
+			const current = await currentFence(this.runDir(scope))
+			if (fence < current) throw fencedOut(scope, fence, current)
+		}
 		await store.writeCheckpoint(checkpoint)
 	}
 
+	async claimRun(scope: CheckpointRunScope, options: ClaimRunOptions): Promise<RunClaim | null> {
+		return acquireClaim(this.runDir(scope), options)
+	}
+
+	async releaseRun(scope: CheckpointRunScope, fence: ClaimFence): Promise<void> {
+		await releaseClaim(this.runDir(scope), fence)
+	}
+
+	/**
+	 * The run's directory, resolved the same way `RunDiskStore.initRun` does.
+	 *
+	 * Duplicated rather than shared because the claim path must be derivable
+	 * WITHOUT binding a store — binding creates the directory, and a claim
+	 * read is a read. Kept beside the layout comment on `listDurableRuns` so
+	 * the two stay together if the layout ever moves.
+	 */
+	private runDir(scope: CheckpointRunScope): string {
+		return scope.parentRunId
+			? join(this.config.baseDir, scope.parentRunId, 'children', scope.runId)
+			: join(this.config.baseDir, scope.runId)
+	}
+
 	async readCheckpoint(
 		scope: CheckpointRunScope,
 		checkpointId: CheckpointId,
@@ -163,21 +209,40 @@ export class DiskCheckpointStore implements CheckpointStore {
 			const runDir = join(this.config.baseDir, runId)
 
 			const own = toDurableRunEntry({ ...attribution, runId }, await readCheckpointsIn(runDir), now)
-			if (own) entries.push(own)
+			if (own) entries.push(await this.withClaim(own, runDir, now))
 
 			for (const childId of await this.readRunDirs(join(runDir, 'children'))) {
+				const childDir = join(runDir, 'children', childId)
 				const child = toDurableRunEntry(
 					{ ...attribution, runId: childId, parentRunId: runId },
-					await readCheckpointsIn(join(runDir, 'children', childId)),
+					await readCheckpointsIn(childDir),
 					now,
 				)
-				if (child) entries.push(child)
+				if (child) entries.push(await this.withClaim(child, childDir, now))
 			}
 		}
 
 		return paginateDurableRuns(entries, options)
 	}
 
+	/**
+	 * Attach the run's claim to its listing row, judged against the page's
+	 * own clock so one page cannot disagree with itself about availability.
+	 */
+	private async withClaim(
+		entry: DurableRunEntry,
+		runDir: string,
+		now: number,
+	): Promise<DurableRunEntry> {
+		const claim = await readClaim(runDir)
+		// A holding whose body could not be read still appears on the row,
+		// carrying an expiry of 0 so it reads as available. Dropping the field
+		// entirely — which is what happened before — put the run under
+		// `claimed: false` by looking unclaimed rather than by being
+		// reclaimable, so a queue reader was told a wedged run was free work.
+		return claim ? { ...entry, claim: toClaimSummary(claim, now) } : entry
+	}
+
 	/** Directory names under `dir`, or none when `dir` does not exist. */
 	private async readRunDirs(dir: string): Promise<RunId[]> {
 		try {

package/src/store/run/checkpoint-memory.ts

@@ -3,11 +3,20 @@ import type {
 	CheckpointListingScope,
 	CheckpointRunScope,
 	CheckpointStore,
+	ClaimFence,
+	ClaimRunOptions,
 	DurableRunEntry,
 	DurableRunPage,
 	ListDurableRunsOptions,
+	RunClaim,
 } from '../../types/run/checkpoint-store.js'
-import { assertContiguousListingScope, paginateDurableRuns, toDurableRunEntry } from './listing.js'
+import {
+	assertContiguousListingScope,
+	fencedOut,
+	paginateDurableRuns,
+	toClaimSummary,
+	toDurableRunEntry,
+} from './listing.js'
 
 /**
  * Process-local {@link CheckpointStore}, keyed by the full five-layer scope.
@@ -35,7 +44,111 @@ export class InMemoryCheckpointStore implements CheckpointStore {
 		return [scope.tenantId, scope.projectId, scope.sessionId, scope.runId].join('/')
 	}
 
-	async writeCheckpoint(scope: CheckpointRunScope, checkpoint: IterationCheckpoint): Promise<void> {
+	/** `tenant/project/session/run` → the run's current holding, if any. */
+	private readonly claims = new Map<string, RunClaim>()
+
+	/**
+	 * The highest fence ever issued per run, kept separately from the claim.
+	 *
+	 * The claim is removed on release; this is not. That separation is the
+	 * whole point: the first version deleted the claim and then computed the
+	 * next fence from it, so releasing rewound the counter to 1 and a worker
+	 * stalled at fence 1 could write beside a new holder also at fence 1 —
+	 * and the documented `finally { releaseRun() }` did it on every pass.
+	 *
+	 * The disk store gets this property from file names that persist. In
+	 * memory the equivalent is a high-water mark nothing clears, and the two
+	 * must agree, because this class is what a host reads when writing a
+	 * backend of its own.
+	 */
+	private readonly highWater = new Map<string, ClaimFence>()
+
+	async claimRun(scope: CheckpointRunScope, options: ClaimRunOptions): Promise<RunClaim | null> {
+		const key = this.key(scope)
+		const now = options.now ?? Date.now()
+		const held = this.claims.get(key)
+
+		// Held by somebody else and still live. Not an error: two readers on
+		// one queue is the ordinary case.
+		if (held && now < held.expiresAt && held.holder !== options.holder) return null
+
+		// A reclaim of an expired holding and a renewal by the current holder
+		// are the same write. The fence advances either way, so a previous
+		// holder that wakes up is fenced out in both cases — a renewal that
+		// kept the fence would leave a stalled twin able to write.
+		// Counted from the high-water mark, never from the live claim. A
+		// released run has no claim, and computing from that absence is what
+		// rewound the counter to 1 on every release.
+		const fence = (this.highWater.get(key) ?? 0) + 1
+		const claim: RunClaim = { holder: options.holder, fence, expiresAt: now + options.ttlMs }
+		this.highWater.set(key, fence)
+		this.claims.set(key, claim)
+		return claim
+	}
+
+	async releaseRun(scope: CheckpointRunScope, fence: ClaimFence): Promise<void> {
+		const key = this.key(scope)
+		const held = this.claims.get(key)
+		// A stale fence releases nothing. A worker that stalled past its lease
+		// must not be able to hand away a run somebody else now holds.
+		//
+		// The high-water mark deliberately survives this. Dropping the claim
+		// returns the run to the queue; forgetting the number it reached would
+		// re-issue a fence a stalled worker still believes it holds.
+		if (!held || held.fence !== fence) return
+		this.claims.delete(key)
+		// And the counter steps past the released holding, because the disk
+		// store's release does: it appends a TOMBSTONE at `fence + 1`, which
+		// raises the maximum name and therefore the fence every later write is
+		// checked against. Without this step a holder that released could still
+		// write with the fence it just gave up — refused on disk, accepted here.
+		//
+		// The consequence is visible in the numbering: a released run's next
+		// claim is `fence + 2` in both stores, because the tombstone consumed
+		// one. That is parity, not an off-by-one.
+		this.highWater.set(key, fence + 1)
+	}
+
+	async writeCheckpoint(
+		scope: CheckpointRunScope,
+		checkpoint: IterationCheckpoint,
+		fence?: ClaimFence,
+	): Promise<void> {
+		const key = this.key(scope)
+		// An unfenced write is allowed even on a claimed run: a host adopting
+		// claims on one worker must not break the workers that have not
+		// adopted them. A fenced write is checked, and that check is what
+		// makes the lease real.
+		if (fence !== undefined) {
+			// Against the HIGH-WATER MARK, not the live claim, and the difference
+			// is a silent loss rather than a duplicate.
+			//
+			// `releaseRun` deletes the claim. Reading `claims` here meant that
+			// between a release and the next take there was no holding to
+			// compare against, so `held` was `undefined` and every fence was
+			// accepted however stale. w1 stalls at fence 1; w2 reclaims at 2,
+			// finishes the work, releases cleanly; w1 wakes and writes with
+			// fence 1 — accepted — and its checkpoint carries a fresh
+			// `createdAt`, so it sorts newest and the next resume restores w1's
+			// stale history. w2's completed work is gone, with no error
+			// anywhere.
+			//
+			// The disk store refuses that write, because `currentFence` reads
+			// file names and the release tombstone raised the maximum. The
+			// minting side here already counted from the high-water mark; only
+			// the enforcement side was left reading the claim, so the two
+			// shipped stores disagreed at the one point that decides whether a
+			// lease is real.
+			const current = this.highWater.get(key) ?? 0
+			if (fence < current) throw fencedOut(scope, fence, current)
+		}
+		return this.writeUnchecked(scope, checkpoint)
+	}
+
+	private async writeUnchecked(
+		scope: CheckpointRunScope,
+		checkpoint: IterationCheckpoint,
+	): Promise<void> {
 		const key = this.key(scope)
 		let run = this.runs.get(key)
 		if (!run) {
@@ -93,7 +206,9 @@ export class InMemoryCheckpointStore implements CheckpointStore {
 			if (scope.sessionId !== undefined && runScope.sessionId !== scope.sessionId) continue
 
 			const entry = toDurableRunEntry(runScope, [...checkpoints.values()], now)
-			if (entry) entries.push(entry)
+			if (!entry) continue
+			const claim = this.claims.get(key)
+			entries.push(claim ? { ...entry, claim: toClaimSummary(claim, now) } : entry)
 		}
 
 		return paginateDurableRuns(entries, options)