@lostgradient/weft 0.7.0 → 0.9.0

package/dist/core/context/run-operation.js

@@ -16,6 +16,7 @@ import {
   resolveActivityTimeout
 } from "./activity-schedule-to-close.js";
 import { getInternals, hasContextInternals } from "./internals.js";
+import { getCachedRunActivityRequest } from "./run-operation-cached-request.js";
 import { isActivityCallOptions } from "./session-state.js";
 import { captureCallerStack } from "./validation.js";
 export function asConcreteContext(context) {
@@ -42,13 +43,27 @@ function parseRunArguments(activity, rest) {
   return { input: values[0], options };
 }
 export const readActivityRetryAttemptForTesting = readActivityRetryAttempt, readCompletedRetrySleepCountForTesting = readCompletedRetrySleepCount, completeActivityRetryAttemptForTesting = completeActivityRetryAttempt;
-function resolveScheduleToCloseBudget(internals, step, scheduleToCloseTimeout) {
+const ACTIVITY_RETRY_SLEEP_DEADLINE_LOCAL_PREFIX = "__weftActivityRetrySleepDeadline:";
+export function readOrInitActivityRetrySleepFireAt(context, operationId, duration) {
+  const internals = getInternals(context), localKey = `${ACTIVITY_RETRY_SLEEP_DEADLINE_LOCAL_PREFIX}${operationId}`, existing = internals.checkpointLocals[localKey];
+  if (typeof existing === "number" && Number.isFinite(existing))
+    return existing;
+  if (existing !== void 0)
+    throw Error(`Invalid checkpointed activity retry sleep deadline ${JSON.stringify(existing)} for "${operationId}".`);
+  const scheduledFireAt = internals.getNow() + duration;
+  internals.checkpointLocals = {
+    ...internals.checkpointLocals,
+    [localKey]: scheduledFireAt
+  };
+  return scheduledFireAt;
+}
+export function resolveScheduleToCloseBudget(internals, retryStateKey, scheduleToCloseTimeout) {
   const budgetMs = parseScheduleToCloseBudgetMs(scheduleToCloseTimeout);
   if (budgetMs === void 0)
     return;
   return {
     budgetMs,
-    dispatchedAt: readOrInitActivityDispatchedAt(internals, step, internals.getNow())
+    dispatchedAt: readOrInitActivityDispatchedAt(internals, retryStateKey, internals.getNow())
   };
 }
 function getActivityName(activity, explicitName) {
@@ -59,39 +74,7 @@ function getActivityName(activity, explicitName) {
 function getActivityFunction(activity) {
   return typeof activity === "function" ? activity : void 0;
 }
-function getCachedRunActivityRequest(internals, step, activityName, input) {
-  const hasCachedResult = internals.accumulatedResults?.has(step) ?? !1;
-  if (hasCachedResult) {
-    const cachedResult = internals.accumulatedResults?.get(step);
-    internals.stepIndex += readCompletedRetrySleepCount(internals, step);
-    if (internals.explainMode)
-      console.log(`[weft] ctx.run(${activityName}) \u2192 Returning cached result from step ${step}`);
-    return {
-      request: { type: "activity", operationId: "", activityName, input },
-      step,
-      hasCachedResult,
-      cachedResult,
-      retryAttempt: 1
-    };
-  }
-  const retryAttempt = readActivityRetryAttempt(internals, step);
-  if (retryAttempt !== void 0) {
-    internals.stepIndex += retryAttempt - 2;
-    return {
-      request: { type: "activity", operationId: "", activityName, input },
-      step,
-      hasCachedResult: !1,
-      retryAttempt
-    };
-  }
-  return {
-    request: { type: "activity", operationId: "", activityName, input },
-    step,
-    hasCachedResult,
-    retryAttempt: 1
-  };
-}
-function createFreshRunActivityRequest(internals, step, activityName, activityFunction, input, options) {
+function createFreshRunActivityRequest(internals, step, activityName, activityFunction, input, options, activityStateKey, workflowExecutionToken) {
   const queue = options?.queue ?? "default";
   if (internals.explainMode) {
     console.log(`[weft] ctx.run(${activityName}, ${JSON.stringify(input)})`);
@@ -103,6 +86,8 @@ function createFreshRunActivityRequest(internals, step, activityName, activityFu
     operationId: crypto.randomUUID(),
     activityName,
     step,
+    ...activityStateKey === void 0 ? {} : { activityStateKey },
+    ...workflowExecutionToken === void 0 ? {} : { workflowExecutionToken },
     ...activityFunction !== void 0 ? { fn: activityFunction } : {},
     input,
     callerStack: captureCallerStack(),
@@ -130,7 +115,7 @@ function isNonRetryableActivityError(error, policy) {
 export function shouldRetryActivityError(error, policy, attempt) {
   return policy !== void 0 && attempt < policy.maxAttempts && !isNonRetryableActivityError(error, policy);
 }
-function prepareActivityRetryRequest(request, attempt) {
+export function prepareActivityRetryRequest(request, attempt) {
   if (attempt === 1)
     return request;
   return {
@@ -140,45 +125,76 @@ function prepareActivityRetryRequest(request, attempt) {
   };
 }
 export function createRunActivityRequest(context, activity, rest, explicitName) {
-  const { input, options } = parseRunArguments(activity, rest), activityName = getActivityName(activity, explicitName), activityFunction = getActivityFunction(activity), internals = getInternals(context), step = internals.stepIndex++, cachedRequest = getCachedRunActivityRequest(internals, step, activityName, input);
+  parseRunArguments(activity, rest);
+  const internals = getInternals(context), step = internals.stepIndex++;
+  return createRunActivityRequestAtStep(context, activity, rest, step, {
+    ...explicitName === void 0 ? {} : { explicitName },
+    advanceStepIndexForCachedRetryState: !0,
+    retryStateKey: step,
+    cacheResultStep: step
+  });
+}
+export function createRunActivityRequestAtStep(context, activity, rest, step, configuration = {}) {
+  const { input, options } = parseRunArguments(activity, rest), activityName = getActivityName(activity, configuration.explicitName), activityFunction = getActivityFunction(activity), internals = getInternals(context), retryStateKey = configuration.retryStateKey ?? step, cacheResultStep = configuration.cacheResultStep === !1 ? void 0 : configuration.cacheResultStep ?? step, cachedRequest = getCachedRunActivityRequest(internals, step, retryStateKey, cacheResultStep, activityName, input, {
+    advanceStepIndexForCachedRetryState: configuration.advanceStepIndexForCachedRetryState ?? !1
+  });
   if (cachedRequest.hasCachedResult)
     return cachedRequest;
-  const retryPolicy = resolveActivityRetryPolicy(activity, options), scheduleToCloseTimeout = resolveActivityScheduleToCloseTimeout(activity, options), effectiveTimeout = resolveActivityTimeout(activity, options), dispatchedOptions = effectiveTimeout === void 0 ? options : { ...options, timeout: effectiveTimeout };
+  const retryPolicy = resolveActivityRetryPolicy(activity, options), scheduleToCloseTimeout = resolveActivityScheduleToCloseTimeout(activity, options);
   return {
-    request: createFreshRunActivityRequest(internals, step, activityName, activityFunction, input, dispatchedOptions),
+    request: createFreshRunActivityRequest(internals, step, activityName, activityFunction, input, resolveDispatchedActivityOptions(activity, options), configuration.activityStateKey, context.workflowExecutionToken),
     step,
+    retryStateKey,
+    ...cacheResultStep === void 0 ? {} : { cacheResultStep },
     hasCachedResult: !1,
     retryAttempt: cachedRequest.retryAttempt,
     ...retryPolicy === void 0 ? {} : { retryPolicy },
     ...scheduleToCloseTimeout === void 0 ? {} : { scheduleToCloseTimeout }
   };
 }
+function resolveDispatchedActivityOptions(activity, options) {
+  const effectiveTimeout = resolveActivityTimeout(activity, options);
+  return effectiveTimeout === void 0 ? options : { ...options, timeout: effectiveTimeout };
+}
 export function* runActivityWithRetry(context, activity, rest, explicitName) {
+  return yield* runPreparedActivityWithRetry(context, createRunActivityRequest(context, activity, rest, explicitName), defaultActivityRetrySleep(context));
+}
+export function* runActivityWithRetryAtStep(context, activity, rest, step, configuration = {}) {
+  return yield* runPreparedActivityWithRetry(context, createRunActivityRequestAtStep(context, activity, rest, step, configuration), configuration.retrySleep ?? defaultActivityRetrySleep(context));
+}
+function defaultActivityRetrySleep(context) {
+  return function* sleepForRetry(duration) {
+    yield* context.sleep(duration);
+  };
+}
+function* runPreparedActivityWithRetry(context, prepared, retrySleep) {
   const {
     request,
-    step,
+    retryStateKey,
+    cacheResultStep,
     hasCachedResult,
     cachedResult,
     retryAttempt,
     retryPolicy,
     scheduleToCloseTimeout
-  } = createRunActivityRequest(context, activity, rest, explicitName);
+  } = prepared;
   if (hasCachedResult)
     return cachedResult;
-  const internals = getInternals(context), budget = retryPolicy === void 0 ? void 0 : resolveScheduleToCloseBudget(internals, step, scheduleToCloseTimeout);
+  const internals = getInternals(context), budget = retryPolicy === void 0 ? void 0 : resolveScheduleToCloseBudget(internals, retryStateKey, scheduleToCloseTimeout);
   let attempt = retryAttempt;
   if (attempt > 1) {
     if (retryPolicy === void 0)
       throw Error(`Missing activity retry policy for checkpointed retry attempt ${attempt}`);
-    yield* context.sleep(calculateBackoff(attempt - 1, retryPolicy));
+    yield* retrySleep(calculateBackoff(attempt - 1, retryPolicy), attempt);
   }
   while (!0) {
     if (attempt > 1)
       assertScheduleToCloseBudgetNotExhausted(budget, request.activityName, internals.getNow());
     try {
       const result = yield prepareActivityRetryRequest(request, attempt);
-      context.accumulatedResults.set(step, result);
-      completeActivityRetryAttempt(internals, step, attempt - 1);
+      if (cacheResultStep !== void 0)
+        context.accumulatedResults.set(cacheResultStep, result);
+      completeActivityRetryAttempt(internals, retryStateKey, attempt - 1);
       return result;
     } catch (error) {
       if (!shouldRetryActivityError(error, retryPolicy, attempt))
@@ -186,8 +202,8 @@ export function* runActivityWithRetry(context, activity, rest, explicitName) {
       const backoff = calculateBackoff(attempt, retryPolicy), now = internals.getNow();
       assertScheduleToCloseBudgetNotExhausted(budget, request.activityName, now, now + backoff);
       const nextAttempt = attempt + 1;
-      writeActivityRetryAttempt(internals, step, nextAttempt);
-      yield* context.sleep(backoff);
+      writeActivityRetryAttempt(internals, retryStateKey, nextAttempt);
+      yield* retrySleep(backoff, nextAttempt);
       attempt = nextAttempt;
     }
   }

package/dist/core/context/speculative-child.js

@@ -24,6 +24,7 @@ function assignOptionalContextOptions(options, internals) {
 function createSpeculativeChildOptions(parent, internals) {
   const options = {
     workflowId: parent.workflowId,
+    ...parent.workflowExecutionToken === void 0 ? {} : { workflowExecutionToken: parent.workflowExecutionToken },
     workflowType: parent.workflowType,
     startedAt: parent.startedAt,
     abortController: internals.abortController,

package/dist/core/context/types.d.ts

@@ -150,6 +150,7 @@ export interface StreamSink {
  */
 export interface ContextOptions {
     workflowId: string;
+    workflowExecutionToken?: string;
     workflowType: string;
     startedAt: number;
     abortController: AbortController;

package/dist/core/engine/activity-heartbeat-tracking.d.ts

@@ -1,19 +1,10 @@
 import type { ActivityContext } from '../types.ts';
 import type { EngineInternals } from './internals.ts';
-/**
- * Per-step heartbeat tracking for {@link ActivityContext.lastHeartbeatDetails}.
- *
- * The store is keyed `workflowId -> step` (not `workflowId` alone) so a retry of
- * an activity step reads the heartbeat ITS prior attempt recorded — and so a
- * later step's first attempt never inherits an earlier step's heartbeat, and
- * concurrent `ctx.all` activities never clobber one another. The step is stable
- * across retry attempts (assigned once at `stepIndex++`). Held only in engine
- * memory and cleared by workflowId (the outer key) on terminal cleanup and purge.
- */
-/** Record the heartbeat the current attempt of a step sent. */
-export declare function recordLastHeartbeatForStep(internals: EngineInternals, workflowId: string, step: number, details: unknown): void;
-/** Read the heartbeat a prior attempt of this step recorded, or `undefined`. */
-export declare function readLastHeartbeatForStep(internals: EngineInternals, workflowId: string, step: number): unknown;
+export type ActivityHeartbeatKey = number | string;
+/** Record the heartbeat the current attempt of an activity sent. */
+export declare function recordLastHeartbeatForStep(internals: EngineInternals, workflowId: string, step: ActivityHeartbeatKey, details: unknown): void;
+/** Read the heartbeat a prior attempt of this activity recorded, or `undefined`. */
+export declare function readLastHeartbeatForStep(internals: EngineInternals, workflowId: string, step: ActivityHeartbeatKey): unknown;
 /**
  * Drop the heartbeat tracked for a single step once that step has completed
  * successfully — after inline verify, and (on the idempotency path) after the
@@ -26,7 +17,7 @@ export declare function readLastHeartbeatForStep(internals: EngineInternals, wor
  * resumable-batch heartbeat a retry depends on, so this is only ever called from
  * the non-speculative success path.
  */
-export declare function clearLastHeartbeatForStep(internals: EngineInternals, workflowId: string, step: number): void;
+export declare function clearLastHeartbeatForStep(internals: EngineInternals, workflowId: string, step: ActivityHeartbeatKey): void;
 /**
  * #493: in development mode, emit a COARSE warning when an inline activity RETRY
  * (`attempt > 1`) starts with no `lastHeartbeatDetails` — the closest runtime
@@ -53,7 +44,7 @@ export declare function clearLastHeartbeatForStep(internals: EngineInternals, wo
  * routing model would need this gate to key on the resolved per-dispatch mode
  * instead.
  */
-export declare function warnIfRetryMissingHeartbeat(internals: EngineInternals, workflowId: string, step: number, attempt: number): void;
+export declare function warnIfRetryMissingHeartbeat(internals: EngineInternals, workflowId: string, step: ActivityHeartbeatKey, attempt: number): void;
 /**
  * Build the {@link ActivityContext} handed to an inline activity function. The
  * signal comes from the per-workflow AbortController (so workflow cancellation
@@ -61,4 +52,4 @@ export declare function warnIfRetryMissingHeartbeat(internals: EngineInternals,
  * tracking above; `completeAsync` is supplied by the caller (it owns the
  * async-completion token machinery).
  */
-export declare function buildActivityContext(internals: EngineInternals, workflowId: string, step: number, signal: AbortSignal, completeAsync: () => never): ActivityContext;
+export declare function buildActivityContext(internals: EngineInternals, workflowId: string, workflowExecutionToken: string | undefined, step: ActivityHeartbeatKey, attempt: number, signal: AbortSignal, completeAsync: () => never): ActivityContext;

package/dist/core/engine/activity-heartbeat-tracking.js

@@ -1,4 +1,10 @@
 import { DevelopmentWarningEvent } from "../events.js";
+function activityHeartbeatKeyLabel(key) {
+  return typeof key === "number" ? `step ${String(key)}` : `activity state key ${key}`;
+}
+function activityHeartbeatFieldPath(key) {
+  return typeof key === "number" ? `step.${String(key)}.lastHeartbeatDetails` : `activityStateKey.${key}.lastHeartbeatDetails`;
+}
 export function recordLastHeartbeatForStep(internals, workflowId, step, details) {
   let byStep = internals.lastHeartbeatDetailsByStep.get(workflowId);
   if (byStep === void 0) {
@@ -27,11 +33,15 @@ export function warnIfRetryMissingHeartbeat(internals, workflowId, step, attempt
     return;
   if (readLastHeartbeatForStep(internals, workflowId, step) !== void 0)
     return;
-  internals.engine.dispatchEvent(new DevelopmentWarningEvent(workflowId, `Activity retry (attempt ${attempt}) at step ${step} has no lastHeartbeatDetails. Either the previous attempt never recorded heartbeat details (it never called heartbeat(), or called it with no details), or the engine process restarted and discarded the in-memory heartbeat (it is not durable). The resumable-batch pattern only resumes across in-process retries; design the activity to restart cleanly when lastHeartbeatDetails is undefined.`, [`step.${step}.lastHeartbeatDetails`]));
+  internals.engine.dispatchEvent(new DevelopmentWarningEvent(workflowId, `Activity retry (attempt ${attempt}) at ${activityHeartbeatKeyLabel(step)} has no lastHeartbeatDetails. Either the previous attempt never recorded heartbeat details (it never called heartbeat(), or called it with no details), or the engine process restarted and discarded the in-memory heartbeat (it is not durable). The resumable-batch pattern only resumes across in-process retries; design the activity to restart cleanly when lastHeartbeatDetails is undefined.`, [activityHeartbeatFieldPath(step)]));
 }
-export function buildActivityContext(internals, workflowId, step, signal, completeAsync) {
+export function buildActivityContext(internals, workflowId, workflowExecutionToken, step, attempt, signal, completeAsync) {
   return {
     signal,
+    ...workflowExecutionToken !== void 0 && {
+      workflowExecutionToken,
+      activityAttemptToken: `${workflowExecutionToken}:activity:${String(step)}:${String(attempt)}`
+    },
     lastHeartbeatDetails: readLastHeartbeatForStep(internals, workflowId, step),
     heartbeat: (details) => {
       internals.heartbeatDetails.set(workflowId, details);

package/dist/core/engine/activity-reconciliation.d.ts

@@ -102,6 +102,7 @@ export declare function resolveStartedActivityReconciliationRecord(internals: En
 }>;
 export declare function writeActivityReconciliationTransition(storage: Storage, reference: ActivityReconciliationReference, expectedRecord: ActivityReconciliationRecord, nextRecord: ActivityReconciliationRecord): Promise<void>;
 export declare function stageActivityReconciliationTransitionWithAtomicWorkflowCommit(internals: EngineInternals, workflowId: string, reference: ActivityReconciliationReference, expectedRecord: ActivityReconciliationRecord, nextRecord: ActivityReconciliationRecord): void;
+export declare function commitActivityReconciliationTransitionWithFencedWrite(internals: EngineInternals, reference: ActivityReconciliationReference, expectedRecord: ActivityReconciliationRecord, nextRecord: ActivityReconciliationRecord): Promise<void>;
 export declare function normalizePreDispatchVerificationResult(result: ActivityVerificationResult): 'not-completed' | 'completed-result-unavailable' | 'indeterminate' | {
     result: unknown;
 };

package/dist/core/engine/activity-reconciliation.js

@@ -7,6 +7,7 @@ import { decode, encode } from "../codec.js";
 import { assertPayloadWithinLimit } from "../payload-size.js";
 import { WeftError } from "../weft-error.js";
 import { stageAtomicWorkflowCommitSideEffects } from "./checkpoint-side-effects.js";
+import { commitFencedEngineWrite } from "./fenced-write.js";
 
 export class ActivityReconciliationCapabilityError extends WeftError {
   constructor() {
@@ -83,7 +84,7 @@ export async function resolveStartedActivityReconciliationRecord(internals, work
       ownerId: crypto.randomUUID(),
       updatedAt: internals.options.getNow()
     };
-    await writeActivityReconciliationTransition(internals.storage, reference, record, nextRecord);
+    await commitActivityReconciliationTransitionWithFencedWrite(internals, reference, record, nextRecord);
     return nextRecord;
   }
   if (normalized === "completed-result-unavailable")
@@ -92,7 +93,7 @@ export async function resolveStartedActivityReconciliationRecord(internals, work
     throw new ActivityReconciliationIndeterminateError(`Activity "${operation.activityName}" reconciliation is indeterminate.`);
   validateActivityResultForReconciliation(normalized.result, internals.options.payloadSizePolicy.maxBytes);
   const completedRecord = createCompletedActivityReconciliationRecord(record, normalized.result, internals.options.getNow());
-  await writeActivityReconciliationTransition(internals.storage, reference, record, completedRecord);
+  await commitActivityReconciliationTransitionWithFencedWrite(internals, reference, record, completedRecord);
   return { completedResult: normalized.result };
 }
 export async function writeActivityReconciliationTransition(storage, reference, expectedRecord, nextRecord) {
@@ -103,6 +104,10 @@ export async function writeActivityReconciliationTransition(storage, reference,
 export function stageActivityReconciliationTransitionWithAtomicWorkflowCommit(internals, workflowId, reference, expectedRecord, nextRecord) {
   stageAtomicWorkflowCommitSideEffects(internals, workflowId, buildActivityReconciliationTransitionSideEffects(reference, expectedRecord, nextRecord));
 }
+export async function commitActivityReconciliationTransitionWithFencedWrite(internals, reference, expectedRecord, nextRecord) {
+  const sideEffects = buildActivityReconciliationTransitionSideEffects(reference, expectedRecord, nextRecord);
+  await commitFencedEngineWrite(internals, sideEffects.operations, sideEffects.conditions, () => new ActivityReconciliationConflictError("Activity reconciliation completion lost compare-and-set ownership."));
+}
 function buildActivityReconciliationTransitionSideEffects(reference, expectedRecord, nextRecord) {
   return {
     conditions: [{ key: reference.key, expectedValue: encode(expectedRecord) }],

package/dist/core/engine/anonymous-signal-sequence.js

@@ -79,21 +79,21 @@ async function scanNextAnonymousSignalSequence(internals, workflowId) {
     if (sequence !== null && sequence >= nextSequence)
       nextSequence = sequence + 1;
   }
+  if (!Number.isSafeInteger(nextSequence))
+    throw Error(`Anonymous signal sequence overflow for workflow "${workflowId}": computed ${nextSequence}`);
   return nextSequence;
 }
 function extractAnonymousSignalSequence(key) {
-  const marker = ":anonymous%3A", markerIndex = key.lastIndexOf(marker);
-  if (markerIndex === -1)
+  const idComponent = key.slice(key.lastIndexOf(":") + 1), marker = "anonymous%3A";
+  if (!idComponent.startsWith(marker))
     return null;
-  const sequenceStart = markerIndex + marker.length, sequenceEnd = key.indexOf("%3A", sequenceStart);
+  const sequenceStart = marker.length, sequenceEnd = idComponent.indexOf("%3A", sequenceStart);
   if (sequenceEnd === -1)
     return null;
-  const sequence = Number(key.slice(sequenceStart, sequenceEnd));
+  const sequence = Number(idComponent.slice(sequenceStart, sequenceEnd));
   return Number.isSafeInteger(sequence) && sequence >= 0 ? sequence : null;
 }
 function decodeSignalSequence(bytes) {
-  if (bytes === null)
-    return 0;
   const value = decode(bytes);
   if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0)
     throw Error("Stored anonymous signal sequence must be a non-negative safe integer");

package/dist/core/engine/async-activity-completion.d.ts

@@ -26,9 +26,6 @@
  * arrives after token recovery but before replay has adopted the workflow
  * generator, the engine buffers the completion or failure outcome and drains it
  * when replay reaches the same deterministic token.
- *
- * Internal-only. Imported from `src/core/engine/**` and the `ActivityContext`
- * construction path.
  */
 import type { OperationOutcome } from '../types.ts';
 import { WeftError } from '../weft-error.ts';
@@ -101,13 +98,14 @@ export type PendingAsyncActivityResolution = {
 /**
  * Derive the durable, deterministic task token for an async activity.
  *
- * The token is anchored to the workflow id, the workflow step index, and the
+ * The token is anchored to the workflow id, the activity state key, and the
  * dispatch attempt — all of which are stable across replay — so a workflow that
  * crashes while parked on an async activity mints the identical token after
- * recovery. `operationId` is deliberately excluded because it is regenerated on
- * every yield and would change on replay.
+ * recovery. Plain `ctx.run()` uses the workflow step as the state key.
+ * `operationId` is deliberately excluded because it is regenerated on every
+ * yield and would change on replay.
  */
-export declare function deriveAsyncActivityToken(workflowId: string, step: number, attempt: number): string;
+export declare function deriveAsyncActivityToken(workflowId: string, step: number | string, attempt: number): string;
 /**
  * Register a deferred activity: record it in memory and durably, then announce
  * the token via {@link ActivityAsyncPendingEvent}. Idempotent on `token`: if the

package/dist/core/engine/async-activity-completion.js

@@ -3,6 +3,8 @@ import { decode, encode } from "../codec.js";
 import { ActivityAsyncPendingEvent } from "../events.js";
 import { assertPayloadWithinLimit } from "../payload-size.js";
 import { WeftError } from "../weft-error.js";
+import { stageAtomicWorkflowCommitSideEffects } from "./checkpoint-side-effects.js";
+import { commitFencedEngineWrite } from "./fenced-write.js";
 const ASYNC_ACTIVITY_TOKEN_PREFIX = "async-act:v1", ASYNC_ACTIVITY_KEY_PREFIX = "async-act:v1:";
 export function asyncActivityWorkflowPrefix(workflowId) {
   return `${ASYNC_ACTIVITY_KEY_PREFIX}${encodeStorageKeyComponent(workflowId)}:`;
@@ -33,7 +35,7 @@ function isPersistedAsyncActivity(value) {
 export function deriveAsyncActivityToken(workflowId, step, attempt) {
   return `${ASYNC_ACTIVITY_TOKEN_PREFIX}:${workflowId}:${step}:${attempt}`;
 }
-function persistPendingAsyncActivity(storage, pending) {
+function buildPersistPendingAsyncActivityOperation(pending) {
   const record = {
     version: 1,
     token: pending.token,
@@ -44,12 +46,16 @@ function persistPendingAsyncActivity(storage, pending) {
     attempt: pending.attempt,
     createdAt: pending.createdAt
   };
-  return storage.put(KEYS.asyncActivity(pending.workflowId, pending.token), encode(record));
+  return {
+    type: "put",
+    key: KEYS.asyncActivity(pending.workflowId, pending.token),
+    value: encode(record)
+  };
 }
 export async function registerPendingAsyncActivity(internals, pending) {
   const alreadyRegistered = internals.pendingAsyncActivities.has(pending.token);
   internals.pendingAsyncActivities.set(pending.token, pending);
-  await persistPendingAsyncActivity(internals.storage, pending);
+  await commitFencedEngineWrite(internals, [buildPersistPendingAsyncActivityOperation(pending)], [], () => Error(`Async activity registration for token "${pending.token}" lost its precondition.`));
   if (!alreadyRegistered)
     internals.engine.dispatchEvent(new ActivityAsyncPendingEvent(pending.token, pending.operationId, pending.workflowId, pending.activityName, pending.attempt));
 }
@@ -87,12 +93,10 @@ async function consumePendingAsyncActivity(internals, token) {
   if (!pending)
     throw new AsyncActivityTokenNotFoundError(token);
   internals.pendingAsyncActivities.delete(token);
-  try {
-    await internals.storage.delete(KEYS.asyncActivity(pending.workflowId, token));
-  } catch (error) {
-    internals.pendingAsyncActivities.set(token, pending);
-    throw error;
-  }
+  stageAtomicWorkflowCommitSideEffects(internals, pending.workflowId, {
+    conditions: [],
+    operations: [{ type: "delete", key: KEYS.asyncActivity(pending.workflowId, token) }]
+  });
   return pending;
 }
 function shouldBufferPendingAsyncActivityResolution(internals, workflowId) {

package/dist/core/engine/bulk-operations-purge.js

@@ -10,6 +10,7 @@ import { buildIndexOperations } from "../search-attributes.js";
 import { buildWorkflowTagIndexOperations, normalizeWorkflowTags } from "../workflow-tags.js";
 import { asyncActivityWorkflowPrefix } from "./async-activity-completion.js";
 import { forgetCommittedCheckpointBytes } from "./checkpoint-commit-snapshots.js";
+import { commitFencedEngineWrite } from "./fenced-write.js";
 import { streamWorkflowStates } from "./listing.js";
 import { createTerminalCleanupTimerId } from "./state-utilities.js";
 import {
@@ -131,7 +132,7 @@ function getWorkflowRetentionDeadline(internals, state) {
 }
 export async function purgeWorkflow(internals, state, cleanupWaiters) {
   const deleteOperations = await collectWorkflowPurgeDeleteOperations(internals, state);
-  await internals.storage.batch(deleteOperations);
+  await commitFencedEngineWrite(internals, deleteOperations, [], () => Error(`Purge commit for workflow "${state.id}" lost its precondition.`));
   clearPurgedWorkflowInMemoryState(internals, state.id, cleanupWaiters);
 }
 export async function collectWorkflowPurgeDeleteOperations(internals, state) {

package/dist/core/engine/bulk-operations.js

@@ -1,7 +1,6 @@
 import {
   KEYS,
   requireStorageCapability,
-  storageConditionalBatch,
   storageHas
 } from "../../storage/interface.js";
 import { assertScopedBulkWorkflowFilter } from "../bulk-workflow-filter.js";
@@ -24,6 +23,10 @@ import {
   withBulkAuditEvent
 } from "./bulk-operations-shared.js";
 import { BulkDeleteRequiresTerminalWorkflowsError } from "./errors.js";
+import {
+  assertLeaseHeldForEngineWork,
+  commitFencedEngineWriteAllowingPreconditionFailure
+} from "./fenced-write.js";
 import { BULK_OPERATION_BATCH_SIZE } from "./listing.js";
 import { createTerminalCleanupTimerId } from "./state-utilities.js";
 import { loadWorkflowState, runSerializedWorkflowStateWrite } from "./storage-io.js";
@@ -59,20 +62,13 @@ async function runBulkCancellation(internals, filter, options = {}) {
   const workflowIdsToCancel = preparation.workflowIds, bulkConcurrency = resolveBulkOperationConcurrency(options);
   let cancelled = 0;
   const errors = [], cancellationResults = await runBulkWorkflowPool(workflowIdsToCancel, bulkConcurrency, async (workflowId) => {
-    try {
-      await internals.engine.cancel(workflowId);
-      if ((await loadWorkflowState(internals, workflowId))?.status === "cancelled")
-        return { status: "cancelled" };
-      return {
-        status: "failed",
-        error: { id: workflowId, error: "Workflow no longer cancellable" }
-      };
-    } catch (error) {
-      return {
-        status: "failed",
-        error: toBulkOperationError(internals, workflowId, error)
-      };
-    }
+    await internals.engine.cancel(workflowId);
+    if ((await loadWorkflowState(internals, workflowId))?.status === "cancelled")
+      return { status: "cancelled" };
+    return {
+      status: "failed",
+      error: { id: workflowId, error: "Workflow no longer cancellable" }
+    };
   });
   for (const cancellationResult of cancellationResults) {
     if (cancellationResult.status === "rejected") {
@@ -104,27 +100,15 @@ async function runBulkFailedWorkflowRetry(internals, filter, options = {}) {
   const bulkConcurrency = resolveBulkOperationConcurrency(options);
   let retried = 0;
   const errors = [], retryResults = await runBulkWorkflowPool(preparation.workflowIds, bulkConcurrency, async (workflowId) => {
-    try {
-      await retryFailedWorkflow(internals, workflowId);
-      return { status: "retried" };
-    } catch (error) {
-      return {
-        status: "failed",
-        error: toBulkOperationError(internals, workflowId, error)
-      };
-    }
+    await retryFailedWorkflow(internals, workflowId);
+    return { status: "retried" };
   });
   for (const retryResult of retryResults) {
     if (retryResult.status === "rejected") {
       errors.push(toBulkOperationError(internals, retryResult.item, retryResult.reason));
       continue;
     }
-    const { value } = retryResult;
-    if (value.status === "retried") {
-      retried += 1;
-      continue;
-    }
-    errors.push(value.error);
+    retried += 1;
   }
   const result = { retried, failed: errors.length, errors };
   if (!shouldPersistBulkAudit(options))
@@ -132,6 +116,7 @@ async function runBulkFailedWorkflowRetry(internals, filter, options = {}) {
   return withBulkAuditEvent(internals, preparation, options, result, retried);
 }
 export async function retryFailedAll(internals, filter, options = {}) {
+  assertLeaseHeldForEngineWork(internals);
   return runBulkFailedWorkflowRetry(internals, filter, options);
 }
 async function retryFailedWorkflow(internals, workflowId) {
@@ -194,12 +179,9 @@ async function reactivateFailedWorkflowFromCheckpointSerialized(internals, workf
   throw Error(`Workflow concurrency admission for "${lastConcurrencyStateKey ?? workflowId}" changed too many times while retrying failed workflow "${workflowId}"`);
 }
 async function commitFailedWorkflowReactivation(internals, operations, conditions) {
-  if (conditions.length === 0) {
-    await internals.storage.batch(operations);
-    return !0;
-  }
-  requireStorageCapability(internals.storage, "conditionalBatch", "retry failed workflow");
-  return storageConditionalBatch(internals.storage, conditions, operations);
+  if (conditions.length > 0)
+    requireStorageCapability(internals.storage, "conditionalBatch", "retry failed workflow");
+  return commitFencedEngineWriteAllowingPreconditionFailure(internals, operations, conditions);
 }
 function buildReactivatedWorkflowState(internals, state) {
   const reactivatedState = {

package/dist/core/engine/callback-checkpoint-persistence.d.ts

@@ -0,0 +1,3 @@
+import type { ContextOperationRequest } from '../context.ts';
+import type { Engine } from './index.ts';
+export declare function persistCheckpointForDataOperation<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>, workflowId: string, operation: ContextOperationRequest): Promise<void>;

package/dist/core/engine/callback-checkpoint-persistence.js

@@ -0,0 +1,25 @@
+import { HISTORY_CIRCUIT_BREAKER_REASON } from "../types.js";
+import { validateAttributeValueSizes } from "./attributes-tags.js";
+import { createTerminationCallbacks } from "./callback-creators-core.js";
+import {
+  appendTimelineBatchOperations,
+  persistCheckpoint,
+  pruneCheckpointHistory
+} from "./checkpoint-io.js";
+import { getInternals } from "./internals.js";
+import { swallowPromiseRejection } from "./strategy-helpers.js";
+import { terminateWorkflow } from "./termination.js";
+export function persistCheckpointForDataOperation(engine, workflowId, operation) {
+  return persistCheckpoint(getInternals(engine), workflowId, operation, void 0, {
+    appendTimelineBatchOperations: (id, checkpointOperation, step, timestamp, operations) => appendTimelineBatchOperations(getInternals(engine), id, checkpointOperation, step, timestamp, operations),
+    swallowPromiseRejection: (promise) => {
+      swallowPromiseRejection(promise);
+    },
+    validateAttributeValueSizes,
+    pruneCheckpointHistory: (id, step) => pruneCheckpointHistory(getInternals(engine), id, step),
+    dispatchEvent: (event) => {
+      engine.dispatchEvent(event);
+    },
+    enforceHistoryCircuitBreaker: (id) => terminateWorkflow(getInternals(engine), id, "timed-out", createTerminationCallbacks(engine), HISTORY_CIRCUIT_BREAKER_REASON)
+  }, { timeline: "preserve-pending" });
+}

package/dist/core/engine/callback-creators-bundles.js

@@ -3,6 +3,7 @@ import {
   broadcast as broadcastFromInternals,
   dispatchPendingUpdateReceived as dispatchPendingUpdateReceivedFromBroadcast
 } from "./broadcast.js";
+import { persistCheckpointForDataOperation } from "./callback-checkpoint-persistence.js";
 import {
   createBroadcastCallbacks,
   createGuardCallbacks,
@@ -97,7 +98,9 @@ export function createConditionOperationCallbacks(engine) {
 }
 export function createDataOperationCallbacks(engine) {
   return {
-    runOperationWithResult: (workflowId, operation, execute) => runOperationWithResultForEngine(engine, workflowId, operation, execute)
+    runOperationWithResult: (workflowId, operation, execute) => runOperationWithResultForEngine(engine, workflowId, operation, execute),
+    persistCheckpoint: (workflowId, operation) => persistCheckpointForDataOperation(engine, workflowId, operation),
+    getActivityOperationCallbacks: () => createActivityOperationCallbacks(engine)
   };
 }
 export function createStateOperationCallbacks(engine) {
@@ -119,7 +122,7 @@ export function createTimeOperationCallbacks(engine) {
     loadWorkflowState: (workflowId) => loadWorkflowState(getInternals(engine), workflowId),
     failWorkflow: (workflowId, error) => failWorkflow(getInternals(engine), workflowId, error, createTerminationCallbacks(engine)),
     runSerializedWorkflowStateWrite: (workflowId, writeOperation) => runSerializedWorkflowStateWrite(getInternals(engine), workflowId, writeOperation),
-    beginWorkflowExecution: (workflowId, workflowType, input, checkpoint, executionDeadline, executionStateOwnerId, registration) => beginWorkflowExecution(getInternals(engine), workflowId, workflowType, input, checkpoint, executionDeadline, executionStateOwnerId, registration, createLifecycleCallbacks(engine)),
+    beginWorkflowExecution: (workflowId, workflowExecutionToken, workflowType, input, checkpoint, executionDeadline, executionStateOwnerId, registration) => beginWorkflowExecution(getInternals(engine), workflowId, workflowExecutionToken, workflowType, input, checkpoint, executionDeadline, executionStateOwnerId, registration, createLifecycleCallbacks(engine)),
     workflowVersionTupleFromState: (state) => workflowVersionTupleFromState(getInternals(engine), state, createLifecycleCallbacks(engine)),
     setWorkflowStartHeaders: (workflowId, headers) => setWorkflowStartHeaders(getInternals(engine), workflowId, headers, createLifecycleCallbacks(engine)),
     loadWorkflowStartHeaders: (workflowId) => loadWorkflowStartHeaders(getInternals(engine), workflowId, createLifecycleCallbacks(engine)),