@pikku/core 0.12.129 → 0.12.133

package/dist/services/lease-service.js

@@ -0,0 +1,110 @@
+export class LeaseTakenError extends Error {
+    key;
+    constructor(key) {
+        super(`Lease ${key} is held by another holder`);
+        this.key = key;
+        this.name = 'LeaseTakenError';
+    }
+}
+/**
+ * Thrown by `holdLease` when the lease lapsed or passed to another holder while
+ * the body ran, so its result was not produced under the lease. The body's
+ * `signal` is aborted with it as soon as the loss is seen.
+ */
+export class LeaseLostError extends Error {
+    key;
+    constructor(key) {
+        super(`Lease ${key} was lost before the body finished`);
+        this.key = key;
+        this.name = 'LeaseLostError';
+    }
+}
+/** Resolves after `ms`, or as soon as `signal` aborts. */
+const sleep = (ms, signal) => new Promise((resolve) => {
+    if (signal.aborted)
+        return resolve();
+    const done = () => {
+        clearTimeout(timer);
+        signal.removeEventListener('abort', done);
+        resolve();
+    };
+    const timer = setTimeout(done, ms);
+    timer.unref?.();
+    signal.addEventListener('abort', done);
+});
+/** How many renewals fit in one lease, so a failed one still leaves others before it runs out. */
+export const RENEWALS_PER_LEASE = 3;
+/** How long a holder waits between renewals of a `ttlMs` lease. */
+export const leaseRenewalIntervalMs = (ttlMs) => Math.max(1, ttlMs / RENEWALS_PER_LEASE);
+/**
+ * Renews a held lease every third of `ttlMs` until stopped, so a renewal that
+ * fails still leaves two more before the lease runs out.
+ *
+ * `renew` resolves `true` while the lease is still this holder's and `false`
+ * once another holder has it. A throw proves neither, so the lease is only
+ * counted lost once renewals have kept failing past its expiry on this
+ * process's clock.
+ *
+ * `stop` waits for a renewal in flight, so a release made after it cannot be
+ * undone by a late one.
+ */
+export const keepLeaseAlive = (key, ttlMs, renew) => {
+    const lost = new AbortController();
+    const stopped = new AbortController();
+    const renewing = (async () => {
+        let heldUntil = Date.now() + ttlMs;
+        while (!lost.signal.aborted) {
+            await sleep(leaseRenewalIntervalMs(ttlMs), stopped.signal);
+            if (stopped.signal.aborted)
+                return;
+            const renewedAt = Date.now();
+            const held = await renew().catch(() => undefined);
+            if (held) {
+                heldUntil = renewedAt + ttlMs;
+            }
+            else if (held === false || Date.now() >= heldUntil) {
+                lost.abort(new LeaseLostError(key));
+            }
+        }
+    })();
+    return {
+        signal: lost.signal,
+        stop: async () => {
+            stopped.abort();
+            await renewing;
+        },
+    };
+};
+/**
+ * Runs `fn` holding `key` on `leases`, renewing the lease every third of
+ * `ttlMs` until it settles. `ttlMs` is how long a crashed holder keeps the key
+ * from everyone else. Never waits for a taken key: that throws
+ * `LeaseTakenError`.
+ */
+export const holdLease = async (leases, key, fn, ttlMs = 30_000) => {
+    const acquired = await leases.acquire(key, crypto.randomUUID(), ttlMs);
+    if (!acquired)
+        throw new LeaseTakenError(key);
+    let lease = acquired;
+    const renew = async () => {
+        const next = await leases.refresh(lease, ttlMs);
+        if (next)
+            lease = next;
+        return next !== null;
+    };
+    const renewal = keepLeaseAlive(key, ttlMs, renew);
+    try {
+        const result = await fn(lease, renewal.signal);
+        await renewal.stop();
+        // The last renewal can be a third of a lease old, so check once more
+        // before vouching that the body ran alone.
+        const stillHeld = !renewal.signal.aborted && (await renew().catch(() => false));
+        if (!stillHeld)
+            throw new LeaseLostError(key);
+        return result;
+    }
+    finally {
+        await renewal.stop();
+        await leases.release(lease).catch(() => { });
+    }
+};

package/dist/services/trigger-source-store.d.ts

@@ -3,15 +3,22 @@ export type TriggerSourceKind = 'webhook';
 export type DeclaredTriggerSource = {
     name: string;
     kind: TriggerSourceKind;
+    /** Where the running app is served and how it labels provider endpoints: what enabling a source registers it with. */
+    baseUrl?: string;
+    labelPrefix?: string;
 };
 /**
  * What was registered with the provider for one declared trigger source.
  * `state` is what the last `setup` returned — provider identifiers only; any
  * secret it issued lives in the credential service.
  */
-export type TriggerSourceRow = DeclaredTriggerSource & {
+export type TriggerSourceRow = Required<Pick<DeclaredTriggerSource, 'name' | 'kind'>> & {
+    baseUrl: string | null;
+    labelPrefix: string | null;
     /** False once the declaration has gone from code without a teardown: an orphan. */
     declared: boolean;
+    /** Only an enabled source is registered with its provider or receives events. Off until someone turns it on. */
+    enabled: boolean;
     status: string | null;
     state: WebhookSourceState | null;
     /** The last step's instructions or error. */
@@ -25,11 +32,12 @@ export type TriggerSourceResult = {
 };
 /** What each declared trigger source registered with its provider. */
 export interface TriggerSourceStore {
-    /** Upserts the declared sources and marks the rest undeclared. */
+    /** Upserts the declared sources, keeping a known address when none is given, and marks the rest undeclared. */
     syncTriggerSources(sources: DeclaredTriggerSource[]): Promise<void>;
     listTriggerSources(): Promise<TriggerSourceRow[]>;
     getTriggerSource(name: string): Promise<TriggerSourceRow | null>;
     recordTriggerSource(name: string, result: TriggerSourceResult): Promise<void>;
+    setTriggerSourceEnabled(name: string, enabled: boolean): Promise<void>;
     deleteTriggerSource(name: string): Promise<void>;
 }
 export declare class InMemoryTriggerSourceStore implements TriggerSourceStore {
@@ -38,8 +46,12 @@ export declare class InMemoryTriggerSourceStore implements TriggerSourceStore {
     listTriggerSources(): Promise<{
         name: string;
         kind: TriggerSourceKind;
+        baseUrl: string | null;
+        labelPrefix: string | null;
         /** False once the declaration has gone from code without a teardown: an orphan. */
         declared: boolean;
+        /** Only an enabled source is registered with its provider or receives events. Off until someone turns it on. */
+        enabled: boolean;
         status: string | null;
         state: WebhookSourceState | null;
         /** The last step's instructions or error. */
@@ -49,8 +61,12 @@ export declare class InMemoryTriggerSourceStore implements TriggerSourceStore {
     getTriggerSource(name: string): Promise<{
         name: string;
         kind: TriggerSourceKind;
+        baseUrl: string | null;
+        labelPrefix: string | null;
         /** False once the declaration has gone from code without a teardown: an orphan. */
         declared: boolean;
+        /** Only an enabled source is registered with its provider or receives events. Off until someone turns it on. */
+        enabled: boolean;
         status: string | null;
         state: WebhookSourceState | null;
         /** The last step's instructions or error. */
@@ -58,5 +74,6 @@ export declare class InMemoryTriggerSourceStore implements TriggerSourceStore {
         updatedAt: string | null;
     } | null>;
     recordTriggerSource(name: string, result: TriggerSourceResult): Promise<void>;
+    setTriggerSourceEnabled(name: string, enabled: boolean): Promise<void>;
     deleteTriggerSource(name: string): Promise<void>;
 }

package/dist/services/trigger-source-store.js

@@ -5,16 +5,23 @@ export class InMemoryTriggerSourceStore {
         for (const row of this.rows.values()) {
             row.declared = names.has(row.name);
         }
-        for (const { name, kind } of sources) {
+        for (const { name, kind, baseUrl, labelPrefix } of sources) {
             const row = this.rows.get(name);
             if (row) {
                 row.kind = kind;
+                if (baseUrl !== undefined)
+                    row.baseUrl = baseUrl;
+                if (labelPrefix !== undefined)
+                    row.labelPrefix = labelPrefix;
                 continue;
             }
             this.rows.set(name, {
                 name,
                 kind,
+                baseUrl: baseUrl ?? null,
+                labelPrefix: labelPrefix ?? null,
                 declared: true,
+                enabled: false,
                 status: null,
                 state: null,
                 detail: null,
@@ -41,6 +48,12 @@ export class InMemoryTriggerSourceStore {
         row.detail = result.detail ?? null;
         row.updatedAt = new Date().toISOString();
     }
+    async setTriggerSourceEnabled(name, enabled) {
+        const row = this.rows.get(name);
+        if (!row)
+            throw new Error(`Unknown trigger source: ${name}`);
+        row.enabled = enabled;
+    }
     async deleteTriggerSource(name) {
         this.rows.delete(name);
     }

package/dist/services/workflow-service.d.ts

@@ -16,7 +16,7 @@ export interface WorkflowService {
     getRunTimeline(id: string): Promise<RunTimeline | null>;
     reconstructRunStateAt(id: string, at?: number | Date): Promise<ReconstructedRunState | null>;
     updateRunStatus(id: string, status: WorkflowStatus, output?: any, error?: SerializedError): Promise<void>;
-    withRunLock<T>(id: string, fn: () => Promise<T>): Promise<T>;
+    withRunLease<T>(id: string, fn: () => Promise<T>): Promise<T>;
     close(): Promise<void>;
     resumeWorkflow(runId: string): Promise<void>;
     /**

package/dist/testing/index.d.ts

@@ -1,3 +1,4 @@
 export { defineServiceTests } from './service-tests.js';
 export type { ServiceTestConfig } from './service-tests.js';
+export type { WorkflowFencingHarness } from './service-tests/workflow-fencing-tests.js';
 export { clearPikkuRuntimeState } from '../test-utils.js';

package/dist/testing/service-tests/lease-service-tests.d.ts

@@ -0,0 +1,3 @@
+import { type LeaseService } from '../../services/lease-service.js';
+/** Conformance suite for `leaseService`. Each test gets a fresh service. */
+export declare const defineLeaseServiceTests: (name: string, create: () => Promise<LeaseService>) => void;

package/dist/testing/service-tests/lease-service-tests.js

@@ -0,0 +1,138 @@
+import { describe, test } from 'node:test';
+import assert from 'node:assert/strict';
+import { holdLease, } from '../../services/lease-service.js';
+const HOUR = 60 * 60 * 1000;
+/**
+ * A lease a test lets lapse, and the lease `holdLease` renews. Both leave room
+ * for a real database's round-trips: a lease has to outlive the statement that
+ * writes it and the one that reads it back.
+ */
+const SHORT = 500;
+const RENEWED = 1_000;
+const wait = (ms) => new Promise((r) => setTimeout(r, ms));
+/** Conformance suite for `leaseService`. Each test gets a fresh service. */
+export const defineLeaseServiceTests = (name, create) => {
+    describe(`LeaseService [${name}]`, () => {
+        test('one holder at a time', async () => {
+            const leases = await create();
+            const a = await leases.acquire('stage', 'a', HOUR);
+            assert.equal(a?.holder, 'a');
+            assert.equal(a?.token, 1);
+            assert.equal(await leases.acquire('stage', 'b', HOUR), null);
+            assert.equal((await leases.get('stage'))?.holder, 'a');
+        });
+        test('keys are independent', async () => {
+            const leases = await create();
+            await leases.acquire('one', 'a', HOUR);
+            assert.equal((await leases.acquire('two', 'b', HOUR))?.holder, 'b');
+        });
+        test('the holder re-acquires under the same token', async () => {
+            const leases = await create();
+            const first = await leases.acquire('stage', 'a', SHORT);
+            const again = await leases.acquire('stage', 'a', HOUR);
+            assert.equal(again?.token, first?.token);
+            assert.ok(again.expiresAt > first.expiresAt);
+        });
+        test('a lapsed lease passes to the next holder with a higher token', async () => {
+            const leases = await create();
+            const a = await leases.acquire('stage', 'a', SHORT);
+            await wait(2 * SHORT);
+            const b = await leases.acquire('stage', 'b', HOUR);
+            assert.equal(b?.holder, 'b');
+            assert.equal(b?.token, a.token + 1);
+            assert.equal(await leases.refresh(a, HOUR), null);
+        });
+        test('release frees the key and the token keeps rising', async () => {
+            const leases = await create();
+            const a = await leases.acquire('stage', 'a', HOUR);
+            await leases.release(a);
+            assert.equal(await leases.get('stage'), null);
+            const b = await leases.acquire('stage', 'b', HOUR);
+            assert.equal(b?.token, a.token + 1);
+        });
+        test('a stale lease cannot refresh or release the new holder', async () => {
+            const leases = await create();
+            const a = await leases.acquire('stage', 'a', SHORT);
+            await wait(2 * SHORT);
+            await leases.acquire('stage', 'b', HOUR);
+            await leases.release(a);
+            assert.equal((await leases.get('stage'))?.holder, 'b');
+        });
+        test('refresh extends a live lease', async () => {
+            const leases = await create();
+            const a = await leases.acquire('stage', 'a', SHORT);
+            const refreshed = await leases.refresh(a, HOUR);
+            assert.equal(refreshed?.token, a.token);
+            await wait(2 * SHORT);
+            assert.equal(await leases.acquire('stage', 'b', HOUR), null);
+        });
+        test('concurrent acquires admit exactly one holder', async () => {
+            const leases = await create();
+            const results = await Promise.all(['a', 'b', 'c', 'd', 'e'].map((holder) => leases.acquire('stage', holder, HOUR)));
+            assert.equal(results.filter(Boolean).length, 1);
+        });
+        test('holdLease hands the body its lease and frees the key', async () => {
+            const leases = await create();
+            let seen;
+            const result = await holdLease(leases, 'stage', async (lease) => {
+                seen = lease;
+                return 42;
+            });
+            assert.equal(result, 42);
+            assert.equal(seen?.key, 'stage');
+            assert.equal(await leases.get('stage'), null);
+        });
+        test('holdLease refuses a taken key', async () => {
+            const leases = await create();
+            await leases.acquire('stage', 'a', HOUR);
+            await assert.rejects(holdLease(leases, 'stage', async () => { }), (err) => err.name === 'LeaseTakenError');
+        });
+        test('holdLease frees the key when the body throws', async () => {
+            const leases = await create();
+            await assert.rejects(holdLease(leases, 'stage', async () => {
+                throw new Error('boom');
+            }), /boom/);
+            assert.equal(await leases.get('stage'), null);
+        });
+        test('holdLease refuses to vouch for a body that outlived its lease', async () => {
+            const leases = await create();
+            await assert.rejects(holdLease(leases, 'stage', async (lease) => {
+                await leases.release(lease);
+                assert.ok(await leases.acquire('stage', 'other', HOUR));
+                await wait(RENEWED);
+                return 'unprotected';
+            }, RENEWED), (err) => err.name === 'LeaseLostError');
+            assert.equal((await leases.get('stage'))?.holder, 'other');
+        });
+        test('holdLease notices a lease lost after its last renewal', async () => {
+            const leases = await create();
+            await assert.rejects(holdLease(leases, 'stage', async (lease) => {
+                await leases.release(lease);
+                assert.ok(await leases.acquire('stage', 'other', HOUR));
+                return 'finished before any renewal ran';
+            }), (err) => err.name === 'LeaseLostError');
+            assert.equal((await leases.get('stage'))?.holder, 'other');
+        });
+        test('holdLease aborts the body once its lease is lost', async () => {
+            const leases = await create();
+            let aborted = false;
+            await assert.rejects(holdLease(leases, 'stage', async (lease, signal) => {
+                await leases.release(lease);
+                assert.ok(await leases.acquire('stage', 'other', HOUR));
+                await new Promise((resolve) => {
+                    signal.addEventListener('abort', () => resolve());
+                    setTimeout(resolve, 5 * RENEWED);
+                });
+                aborted = signal.aborted;
+            }, RENEWED), (err) => err.name === 'LeaseLostError');
+            assert.equal(aborted, true, 'the body was never told its lease was gone');
+        });
+        test('holdLease keeps the lease alive past its ttl', async () => {
+            const leases = await create();
+            await holdLease(leases, 'stage', async () => {
+                await wait(2 * RENEWED);
+                assert.ok(await leases.get('stage'), 'the lease lapsed mid-body');
+            }, RENEWED);
+        });
+    });
+};

package/dist/testing/service-tests/workflow-fencing-tests.d.ts

@@ -0,0 +1,18 @@
+import type { PikkuWorkflowService } from '../../wirings/workflow/pikku-workflow-service.js';
+/** A workflow service under test, and the means to kill the worker holding a step. */
+export type WorkflowFencingHarness = {
+    service: PikkuWorkflowService;
+    /**
+     * Make the step's lease read as lapsed, as it would once its worker died, on
+     * whichever clock the store judges leases by.
+     */
+    lapseLease: (runId: string, stepName: string) => Promise<void>;
+};
+/**
+ * Conformance suite for a workflow service that fences a step's writes to the
+ * claim that made them. Each test gets a fresh harness.
+ *
+ * Every scenario has a stale attempt — claimed, then its lease lapsed and the
+ * step claimed again — and the attempt that replaced it.
+ */
+export declare const defineWorkflowFencingTests: (name: string, create: () => Promise<WorkflowFencingHarness>) => void;

package/dist/testing/service-tests/workflow-fencing-tests.js

@@ -0,0 +1,216 @@
+import { describe, test } from 'node:test';
+import assert from 'node:assert/strict';
+import { RPCNotFoundError } from '../../wirings/rpc/rpc-runner.js';
+import { pikkuState } from '../../pikku-state.js';
+import { ConsoleLogger } from '../../services/logger-console.js';
+import { LogLevel } from '../../services/logger.js';
+const STEP = 'charge';
+const RPC = 'charge:card';
+const LEASE_MS = 60_000;
+const RACES = 25;
+const createSilentLogger = () => {
+    const logger = new ConsoleLogger();
+    logger.setLevel(LogLevel.critical);
+    return logger;
+};
+const superseded = (e) => e.name === 'WorkflowStepSupersededError';
+/**
+ * Conformance suite for a workflow service that fences a step's writes to the
+ * claim that made them. Each test gets a fresh harness.
+ *
+ * Every scenario has a stale attempt — claimed, then its lease lapsed and the
+ * step claimed again — and the attempt that replaced it.
+ */
+export const defineWorkflowFencingTests = (name, create) => {
+    describe(`WorkflowService fencing [${name}]`, () => {
+        const claim = ({ service }, runId) => service['claimStepForExecution'](runId, STEP, RPC, LEASE_MS);
+        const startRun = async ({ service }) => {
+            const runId = await service.createRun('fenced', {}, false, 'hash', {
+                type: 'test',
+            });
+            await service.insertStepState(runId, STEP, RPC, {}, { retries: 5 });
+            return runId;
+        };
+        /** Claim the step, lose its worker, and claim it again. */
+        const supersede = async (h, runId) => {
+            const stale = await claim(h, runId);
+            assert.ok(stale, 'the first dispatch did not get the step');
+            await h.lapseLease(runId, STEP);
+            const current = await claim(h, runId);
+            assert.ok(current, 'the lapsed step was not claimed again');
+            assert.equal(current.attemptCount, stale.attemptCount + 1);
+            return { stale, current };
+        };
+        const stepOf = (h, runId) => h.service.getStepState(runId, STEP);
+        const leaseOf = async (h, runId) => (await stepOf(h, runId)).leaseExpiresAt?.getTime();
+        /** Runs `fn` with the engine's singletons quiet, then puts back what was there. */
+        const quietly = async (fn) => {
+            const previous = pikkuState(null, 'package', 'singletonServices');
+            pikkuState(null, 'package', 'singletonServices', {
+                ...previous,
+                logger: createSilentLogger(),
+            });
+            try {
+                return await fn();
+            }
+            finally {
+                pikkuState(null, 'package', 'singletonServices', previous);
+            }
+        };
+        /**
+         * Dispatch the step through the engine, with an RPC that loses the step to
+         * a newer claim while it runs and then does `outcome`.
+         */
+        const dispatchStale = (h, runId, outcome) => quietly(async () => {
+            let current = null;
+            const staleWorker = {
+                rpcWithWire: async () => {
+                    await h.lapseLease(runId, STEP);
+                    current = await claim(h, runId);
+                    return outcome();
+                },
+            };
+            const surfaced = await h.service
+                .executeWorkflowStep(runId, STEP, RPC, {}, staleWorker)
+                .then(() => undefined, (e) => e);
+            assert.ok(current, 'the step was never claimed from under the worker');
+            return { surfaced, current: current };
+        });
+        test("a stale attempt's result is refused and the current attempt's lands", async () => {
+            const h = await create();
+            const runId = await startRun(h);
+            const { stale, current } = await supersede(h, runId);
+            await assert.rejects(h.service.setStepResult(stale.stepId, 'stale', stale.attemptCount), superseded);
+            await h.service.setStepResult(current.stepId, 'fresh', current.attemptCount);
+            const step = await stepOf(h, runId);
+            assert.equal(step.status, 'succeeded');
+            assert.equal(step.result, 'fresh');
+            assert.equal(step.attemptCount, current.attemptCount);
+            const history = (await h.service.getRunHistory(runId)).filter((attempt) => attempt.stepName === STEP);
+            assert.equal(history.length, 2, 'each claim keeps its own attempt');
+            assert.equal(history[0].status, 'running', 'the stale attempt was recorded with an outcome it was refused');
+            assert.notEqual(history[0].result, 'stale');
+            assert.equal(history[1].status, 'succeeded');
+            assert.equal(history[1].result, 'fresh');
+        });
+        test("a stale attempt's error is refused and does not fail the step", async () => {
+            const h = await create();
+            const runId = await startRun(h);
+            const { stale, current } = await supersede(h, runId);
+            await assert.rejects(h.service.setStepError(stale.stepId, new Error('stale'), stale.attemptCount), superseded);
+            const step = await stepOf(h, runId);
+            assert.equal(step.status, 'running', 'the stale attempt failed the step');
+            assert.equal(step.error, undefined);
+            assert.equal(step.attemptCount, current.attemptCount);
+        });
+        test('a stale attempt cannot renew the lease, nor move the one that replaced it', async () => {
+            const h = await create();
+            const runId = await startRun(h);
+            const { stale, current } = await supersede(h, runId);
+            const held = await leaseOf(h, runId);
+            assert.equal(await h.service.refreshStepLease(stale.stepId, 10 * LEASE_MS, stale.attemptCount), false, 'the stale attempt was told it still holds the step');
+            assert.equal(await leaseOf(h, runId), held, 'the stale renewal landed');
+            assert.equal(await h.service.refreshStepLease(stale.stepId, null, stale.attemptCount), false);
+            assert.equal(await leaseOf(h, runId), held, 'the stale release landed');
+            assert.equal(await h.service.refreshStepLease(current.stepId, 10 * LEASE_MS, current.attemptCount), true);
+        });
+        test('a stale attempt cannot overwrite a step the current attempt already succeeded', async () => {
+            const h = await create();
+            const runId = await startRun(h);
+            const { stale, current } = await supersede(h, runId);
+            await h.service.setStepResult(current.stepId, 'fresh', current.attemptCount);
+            await assert.rejects(h.service.setStepResult(stale.stepId, 'stale', stale.attemptCount), superseded);
+            await assert.rejects(h.service.setStepError(stale.stepId, new Error('stale'), stale.attemptCount), superseded);
+            const step = await stepOf(h, runId);
+            assert.equal(step.status, 'succeeded');
+            assert.equal(step.result, 'fresh');
+            assert.equal(step.error, undefined);
+        });
+        test('a stale attempt cannot overwrite a step the current attempt already failed', async () => {
+            const h = await create();
+            const runId = await startRun(h);
+            const { stale, current } = await supersede(h, runId);
+            await h.service.setStepError(current.stepId, new Error('fresh'), current.attemptCount);
+            await assert.rejects(h.service.setStepResult(stale.stepId, 'stale', stale.attemptCount), superseded);
+            const step = await stepOf(h, runId);
+            assert.equal(step.status, 'failed');
+            assert.equal(step.error?.message, 'fresh');
+            assert.equal(step.result, undefined);
+        });
+        test('when both attempts write at once, only the current one ever lands', async () => {
+            const h = await create();
+            for (let i = 0; i < RACES; i++) {
+                const runId = await startRun(h);
+                const { stale, current } = await supersede(h, runId);
+                const staleWrite = () => i % 2 === 0
+                    ? h.service.setStepResult(stale.stepId, 'stale', stale.attemptCount)
+                    : h.service.setStepError(stale.stepId, new Error('stale'), stale.attemptCount);
+                const currentWrite = () => h.service.setStepResult(current.stepId, 'fresh', current.attemptCount);
+                const [staleOutcome, currentOutcome] = i % 3 === 0
+                    ? (await Promise.allSettled([currentWrite(), staleWrite()])).reverse()
+                    : await Promise.allSettled([staleWrite(), currentWrite()]);
+                assert.equal(staleOutcome.status, 'rejected', `race ${i}`);
+                assert.ok(superseded(staleOutcome.reason), `race ${i}: ${staleOutcome.reason}`);
+                assert.equal(currentOutcome.status, 'fulfilled', `race ${i}`);
+                const step = await stepOf(h, runId);
+                assert.equal(step.status, 'succeeded', `race ${i}`);
+                assert.equal(step.result, 'fresh', `race ${i}`);
+            }
+        });
+        test('a third claim fences out both attempts before it', async () => {
+            const h = await create();
+            const runId = await startRun(h);
+            const { stale: first, current: second } = await supersede(h, runId);
+            await h.lapseLease(runId, STEP);
+            const third = await claim(h, runId);
+            assert.ok(third);
+            assert.equal(third.attemptCount, second.attemptCount + 1);
+            const held = await leaseOf(h, runId);
+            for (const stale of [first, second]) {
+                await assert.rejects(h.service.setStepResult(stale.stepId, 'stale', stale.attemptCount), superseded, `attempt ${stale.attemptCount} recorded a result`);
+                await assert.rejects(h.service.setStepError(stale.stepId, new Error('stale'), stale.attemptCount), superseded, `attempt ${stale.attemptCount} recorded an error`);
+                assert.equal(await h.service.refreshStepLease(stale.stepId, 10 * LEASE_MS, stale.attemptCount), false, `attempt ${stale.attemptCount} renewed the lease`);
+            }
+            assert.equal(await leaseOf(h, runId), held);
+            await h.service.setStepResult(third.stepId, 'third', third.attemptCount);
+            const step = await stepOf(h, runId);
+            assert.equal(step.status, 'succeeded');
+            assert.equal(step.result, 'third');
+        });
+        test('a superseded dispatch whose RPC is missing neither fails the step nor suspends the run', async () => {
+            const h = await create();
+            const runId = await startRun(h);
+            const { surfaced, current } = await dispatchStale(h, runId, () => {
+                throw new RPCNotFoundError(RPC);
+            });
+            const step = await stepOf(h, runId);
+            assert.equal(step.status, 'running', 'the stale attempt failed the step');
+            assert.equal(step.attemptCount, current.attemptCount);
+            assert.equal((await h.service.getRun(runId))?.status, 'running', 'the stale attempt suspended the run');
+            assert.equal(surfaced, undefined, 'a superseded dispatch ends quietly');
+        });
+        test('a superseded dispatch whose RPC throws ends quietly and leaves the step to its new owner', async () => {
+            const h = await create();
+            const runId = await startRun(h);
+            const { surfaced, current } = await dispatchStale(h, runId, () => {
+                throw new Error('the card was declined');
+            });
+            assert.equal(surfaced, undefined, 'the stale failure surfaced');
+            const step = await stepOf(h, runId);
+            assert.equal(step.status, 'running', 'the stale attempt failed the step');
+            assert.equal(step.error, undefined);
+            assert.equal(step.attemptCount, current.attemptCount);
+            assert.equal((await h.service.getRun(runId))?.status, 'running');
+        });
+        test('a superseded dispatch whose RPC succeeds does not record its result', async () => {
+            const h = await create();
+            const runId = await startRun(h);
+            const { surfaced, current } = await dispatchStale(h, runId, () => 'stale');
+            assert.equal(surfaced, undefined);
+            const step = await stepOf(h, runId);
+            assert.equal(step.status, 'running', 'the stale result landed');
+            assert.equal(step.result, undefined);
+            assert.equal(step.attemptCount, current.attemptCount);
+        });
+    });
+};

package/dist/testing/service-tests.d.ts

@@ -9,6 +9,8 @@ import type { SecretService } from '../services/secret-service.js';
 import type { CredentialService } from '../services/credential-service.js';
 import type { AgentRunService } from '../wirings/agent/agent.types.js';
 import type { SessionStore } from '../services/session-store.js';
+import type { LeaseService } from '../services/lease-service.js';
+import type { WorkflowFencingHarness } from './service-tests/workflow-fencing-tests.js';
 export interface ServiceTestConfig {
     name: string;
     services: {
@@ -36,6 +38,9 @@ export interface ServiceTestConfig {
             rotateKEK?(): Promise<number>;
         }>;
         sessionStore?: () => Promise<SessionStore>;
+        leaseService?: () => Promise<LeaseService>;
+        /** A workflow service that fences each step write to the claim that made it. */
+        workflowFencing?: () => Promise<WorkflowFencingHarness>;
     };
 }
 /**

package/dist/testing/service-tests.js

@@ -8,6 +8,8 @@ import { defineSecretServiceTests } from './service-tests/secret-service-tests.j
 import { defineCredentialServiceTests } from './service-tests/credential-service-tests.js';
 import { defineAgentRunServiceTests } from './service-tests/agent-run-service-tests.js';
 import { defineSessionStoreTests } from './service-tests/session-store-tests.js';
+import { defineLeaseServiceTests } from './service-tests/lease-service-tests.js';
+import { defineWorkflowFencingTests } from './service-tests/workflow-fencing-tests.js';
 /**
  * The shared conformance suite every storage backend runs.
  *
@@ -46,4 +48,10 @@ export function defineServiceTests(config) {
     if (services.sessionStore) {
         defineSessionStoreTests(name, services.sessionStore);
     }
+    if (services.leaseService) {
+        defineLeaseServiceTests(name, services.leaseService);
+    }
+    if (services.workflowFencing) {
+        defineWorkflowFencingTests(name, services.workflowFencing);
+    }
 }