@lostgradient/weft 0.7.0 → 0.8.0

package/dist/core/context/run-operation.js

@@ -16,6 +16,7 @@ import {
   resolveActivityTimeout
 } from "./activity-schedule-to-close.js";
 import { getInternals, hasContextInternals } from "./internals.js";
+import { getCachedRunActivityRequest } from "./run-operation-cached-request.js";
 import { isActivityCallOptions } from "./session-state.js";
 import { captureCallerStack } from "./validation.js";
 export function asConcreteContext(context) {
@@ -42,13 +43,27 @@ function parseRunArguments(activity, rest) {
   return { input: values[0], options };
 }
 export const readActivityRetryAttemptForTesting = readActivityRetryAttempt, readCompletedRetrySleepCountForTesting = readCompletedRetrySleepCount, completeActivityRetryAttemptForTesting = completeActivityRetryAttempt;
-function resolveScheduleToCloseBudget(internals, step, scheduleToCloseTimeout) {
+const ACTIVITY_RETRY_SLEEP_DEADLINE_LOCAL_PREFIX = "__weftActivityRetrySleepDeadline:";
+export function readOrInitActivityRetrySleepFireAt(context, operationId, duration) {
+  const internals = getInternals(context), localKey = `${ACTIVITY_RETRY_SLEEP_DEADLINE_LOCAL_PREFIX}${operationId}`, existing = internals.checkpointLocals[localKey];
+  if (typeof existing === "number" && Number.isFinite(existing))
+    return existing;
+  if (existing !== void 0)
+    throw Error(`Invalid checkpointed activity retry sleep deadline ${JSON.stringify(existing)} for "${operationId}".`);
+  const scheduledFireAt = internals.getNow() + duration;
+  internals.checkpointLocals = {
+    ...internals.checkpointLocals,
+    [localKey]: scheduledFireAt
+  };
+  return scheduledFireAt;
+}
+export function resolveScheduleToCloseBudget(internals, retryStateKey, scheduleToCloseTimeout) {
   const budgetMs = parseScheduleToCloseBudgetMs(scheduleToCloseTimeout);
   if (budgetMs === void 0)
     return;
   return {
     budgetMs,
-    dispatchedAt: readOrInitActivityDispatchedAt(internals, step, internals.getNow())
+    dispatchedAt: readOrInitActivityDispatchedAt(internals, retryStateKey, internals.getNow())
   };
 }
 function getActivityName(activity, explicitName) {
@@ -59,39 +74,7 @@ function getActivityName(activity, explicitName) {
 function getActivityFunction(activity) {
   return typeof activity === "function" ? activity : void 0;
 }
-function getCachedRunActivityRequest(internals, step, activityName, input) {
-  const hasCachedResult = internals.accumulatedResults?.has(step) ?? !1;
-  if (hasCachedResult) {
-    const cachedResult = internals.accumulatedResults?.get(step);
-    internals.stepIndex += readCompletedRetrySleepCount(internals, step);
-    if (internals.explainMode)
-      console.log(`[weft] ctx.run(${activityName}) \u2192 Returning cached result from step ${step}`);
-    return {
-      request: { type: "activity", operationId: "", activityName, input },
-      step,
-      hasCachedResult,
-      cachedResult,
-      retryAttempt: 1
-    };
-  }
-  const retryAttempt = readActivityRetryAttempt(internals, step);
-  if (retryAttempt !== void 0) {
-    internals.stepIndex += retryAttempt - 2;
-    return {
-      request: { type: "activity", operationId: "", activityName, input },
-      step,
-      hasCachedResult: !1,
-      retryAttempt
-    };
-  }
-  return {
-    request: { type: "activity", operationId: "", activityName, input },
-    step,
-    hasCachedResult,
-    retryAttempt: 1
-  };
-}
-function createFreshRunActivityRequest(internals, step, activityName, activityFunction, input, options) {
+function createFreshRunActivityRequest(internals, step, activityName, activityFunction, input, options, activityStateKey) {
   const queue = options?.queue ?? "default";
   if (internals.explainMode) {
     console.log(`[weft] ctx.run(${activityName}, ${JSON.stringify(input)})`);
@@ -103,6 +86,7 @@ function createFreshRunActivityRequest(internals, step, activityName, activityFu
     operationId: crypto.randomUUID(),
     activityName,
     step,
+    ...activityStateKey === void 0 ? {} : { activityStateKey },
     ...activityFunction !== void 0 ? { fn: activityFunction } : {},
     input,
     callerStack: captureCallerStack(),
@@ -130,7 +114,7 @@ function isNonRetryableActivityError(error, policy) {
 export function shouldRetryActivityError(error, policy, attempt) {
   return policy !== void 0 && attempt < policy.maxAttempts && !isNonRetryableActivityError(error, policy);
 }
-function prepareActivityRetryRequest(request, attempt) {
+export function prepareActivityRetryRequest(request, attempt) {
   if (attempt === 1)
     return request;
   return {
@@ -140,45 +124,76 @@ function prepareActivityRetryRequest(request, attempt) {
   };
 }
 export function createRunActivityRequest(context, activity, rest, explicitName) {
-  const { input, options } = parseRunArguments(activity, rest), activityName = getActivityName(activity, explicitName), activityFunction = getActivityFunction(activity), internals = getInternals(context), step = internals.stepIndex++, cachedRequest = getCachedRunActivityRequest(internals, step, activityName, input);
+  parseRunArguments(activity, rest);
+  const internals = getInternals(context), step = internals.stepIndex++;
+  return createRunActivityRequestAtStep(context, activity, rest, step, {
+    ...explicitName === void 0 ? {} : { explicitName },
+    advanceStepIndexForCachedRetryState: !0,
+    retryStateKey: step,
+    cacheResultStep: step
+  });
+}
+export function createRunActivityRequestAtStep(context, activity, rest, step, configuration = {}) {
+  const { input, options } = parseRunArguments(activity, rest), activityName = getActivityName(activity, configuration.explicitName), activityFunction = getActivityFunction(activity), internals = getInternals(context), retryStateKey = configuration.retryStateKey ?? step, cacheResultStep = configuration.cacheResultStep === !1 ? void 0 : configuration.cacheResultStep ?? step, cachedRequest = getCachedRunActivityRequest(internals, step, retryStateKey, cacheResultStep, activityName, input, {
+    advanceStepIndexForCachedRetryState: configuration.advanceStepIndexForCachedRetryState ?? !1
+  });
   if (cachedRequest.hasCachedResult)
     return cachedRequest;
-  const retryPolicy = resolveActivityRetryPolicy(activity, options), scheduleToCloseTimeout = resolveActivityScheduleToCloseTimeout(activity, options), effectiveTimeout = resolveActivityTimeout(activity, options), dispatchedOptions = effectiveTimeout === void 0 ? options : { ...options, timeout: effectiveTimeout };
+  const retryPolicy = resolveActivityRetryPolicy(activity, options), scheduleToCloseTimeout = resolveActivityScheduleToCloseTimeout(activity, options);
   return {
-    request: createFreshRunActivityRequest(internals, step, activityName, activityFunction, input, dispatchedOptions),
+    request: createFreshRunActivityRequest(internals, step, activityName, activityFunction, input, resolveDispatchedActivityOptions(activity, options), configuration.activityStateKey),
     step,
+    retryStateKey,
+    ...cacheResultStep === void 0 ? {} : { cacheResultStep },
     hasCachedResult: !1,
     retryAttempt: cachedRequest.retryAttempt,
     ...retryPolicy === void 0 ? {} : { retryPolicy },
     ...scheduleToCloseTimeout === void 0 ? {} : { scheduleToCloseTimeout }
   };
 }
+function resolveDispatchedActivityOptions(activity, options) {
+  const effectiveTimeout = resolveActivityTimeout(activity, options);
+  return effectiveTimeout === void 0 ? options : { ...options, timeout: effectiveTimeout };
+}
 export function* runActivityWithRetry(context, activity, rest, explicitName) {
+  return yield* runPreparedActivityWithRetry(context, createRunActivityRequest(context, activity, rest, explicitName), defaultActivityRetrySleep(context));
+}
+export function* runActivityWithRetryAtStep(context, activity, rest, step, configuration = {}) {
+  return yield* runPreparedActivityWithRetry(context, createRunActivityRequestAtStep(context, activity, rest, step, configuration), configuration.retrySleep ?? defaultActivityRetrySleep(context));
+}
+function defaultActivityRetrySleep(context) {
+  return function* sleepForRetry(duration) {
+    yield* context.sleep(duration);
+  };
+}
+function* runPreparedActivityWithRetry(context, prepared, retrySleep) {
   const {
     request,
-    step,
+    retryStateKey,
+    cacheResultStep,
     hasCachedResult,
     cachedResult,
     retryAttempt,
     retryPolicy,
     scheduleToCloseTimeout
-  } = createRunActivityRequest(context, activity, rest, explicitName);
+  } = prepared;
   if (hasCachedResult)
     return cachedResult;
-  const internals = getInternals(context), budget = retryPolicy === void 0 ? void 0 : resolveScheduleToCloseBudget(internals, step, scheduleToCloseTimeout);
+  const internals = getInternals(context), budget = retryPolicy === void 0 ? void 0 : resolveScheduleToCloseBudget(internals, retryStateKey, scheduleToCloseTimeout);
   let attempt = retryAttempt;
   if (attempt > 1) {
     if (retryPolicy === void 0)
       throw Error(`Missing activity retry policy for checkpointed retry attempt ${attempt}`);
-    yield* context.sleep(calculateBackoff(attempt - 1, retryPolicy));
+    yield* retrySleep(calculateBackoff(attempt - 1, retryPolicy), attempt);
   }
   while (!0) {
     if (attempt > 1)
       assertScheduleToCloseBudgetNotExhausted(budget, request.activityName, internals.getNow());
     try {
       const result = yield prepareActivityRetryRequest(request, attempt);
-      context.accumulatedResults.set(step, result);
-      completeActivityRetryAttempt(internals, step, attempt - 1);
+      if (cacheResultStep !== void 0)
+        context.accumulatedResults.set(cacheResultStep, result);
+      completeActivityRetryAttempt(internals, retryStateKey, attempt - 1);
       return result;
     } catch (error) {
       if (!shouldRetryActivityError(error, retryPolicy, attempt))
@@ -186,8 +201,8 @@ export function* runActivityWithRetry(context, activity, rest, explicitName) {
       const backoff = calculateBackoff(attempt, retryPolicy), now = internals.getNow();
       assertScheduleToCloseBudgetNotExhausted(budget, request.activityName, now, now + backoff);
       const nextAttempt = attempt + 1;
-      writeActivityRetryAttempt(internals, step, nextAttempt);
-      yield* context.sleep(backoff);
+      writeActivityRetryAttempt(internals, retryStateKey, nextAttempt);
+      yield* retrySleep(backoff, nextAttempt);
       attempt = nextAttempt;
     }
   }

package/dist/core/engine/activity-heartbeat-tracking.d.ts

@@ -1,19 +1,10 @@
 import type { ActivityContext } from '../types.ts';
 import type { EngineInternals } from './internals.ts';
-/**
- * Per-step heartbeat tracking for {@link ActivityContext.lastHeartbeatDetails}.
- *
- * The store is keyed `workflowId -> step` (not `workflowId` alone) so a retry of
- * an activity step reads the heartbeat ITS prior attempt recorded — and so a
- * later step's first attempt never inherits an earlier step's heartbeat, and
- * concurrent `ctx.all` activities never clobber one another. The step is stable
- * across retry attempts (assigned once at `stepIndex++`). Held only in engine
- * memory and cleared by workflowId (the outer key) on terminal cleanup and purge.
- */
-/** Record the heartbeat the current attempt of a step sent. */
-export declare function recordLastHeartbeatForStep(internals: EngineInternals, workflowId: string, step: number, details: unknown): void;
-/** Read the heartbeat a prior attempt of this step recorded, or `undefined`. */
-export declare function readLastHeartbeatForStep(internals: EngineInternals, workflowId: string, step: number): unknown;
+export type ActivityHeartbeatKey = number | string;
+/** Record the heartbeat the current attempt of an activity sent. */
+export declare function recordLastHeartbeatForStep(internals: EngineInternals, workflowId: string, step: ActivityHeartbeatKey, details: unknown): void;
+/** Read the heartbeat a prior attempt of this activity recorded, or `undefined`. */
+export declare function readLastHeartbeatForStep(internals: EngineInternals, workflowId: string, step: ActivityHeartbeatKey): unknown;
 /**
  * Drop the heartbeat tracked for a single step once that step has completed
  * successfully — after inline verify, and (on the idempotency path) after the
@@ -26,7 +17,7 @@ export declare function readLastHeartbeatForStep(internals: EngineInternals, wor
  * resumable-batch heartbeat a retry depends on, so this is only ever called from
  * the non-speculative success path.
  */
-export declare function clearLastHeartbeatForStep(internals: EngineInternals, workflowId: string, step: number): void;
+export declare function clearLastHeartbeatForStep(internals: EngineInternals, workflowId: string, step: ActivityHeartbeatKey): void;
 /**
  * #493: in development mode, emit a COARSE warning when an inline activity RETRY
  * (`attempt > 1`) starts with no `lastHeartbeatDetails` — the closest runtime
@@ -53,7 +44,7 @@ export declare function clearLastHeartbeatForStep(internals: EngineInternals, wo
  * routing model would need this gate to key on the resolved per-dispatch mode
  * instead.
  */
-export declare function warnIfRetryMissingHeartbeat(internals: EngineInternals, workflowId: string, step: number, attempt: number): void;
+export declare function warnIfRetryMissingHeartbeat(internals: EngineInternals, workflowId: string, step: ActivityHeartbeatKey, attempt: number): void;
 /**
  * Build the {@link ActivityContext} handed to an inline activity function. The
  * signal comes from the per-workflow AbortController (so workflow cancellation
@@ -61,4 +52,4 @@ export declare function warnIfRetryMissingHeartbeat(internals: EngineInternals,
  * tracking above; `completeAsync` is supplied by the caller (it owns the
  * async-completion token machinery).
  */
-export declare function buildActivityContext(internals: EngineInternals, workflowId: string, step: number, signal: AbortSignal, completeAsync: () => never): ActivityContext;
+export declare function buildActivityContext(internals: EngineInternals, workflowId: string, step: ActivityHeartbeatKey, signal: AbortSignal, completeAsync: () => never): ActivityContext;

package/dist/core/engine/activity-heartbeat-tracking.js

@@ -1,4 +1,10 @@
 import { DevelopmentWarningEvent } from "../events.js";
+function activityHeartbeatKeyLabel(key) {
+  return typeof key === "number" ? `step ${String(key)}` : `activity state key ${key}`;
+}
+function activityHeartbeatFieldPath(key) {
+  return typeof key === "number" ? `step.${String(key)}.lastHeartbeatDetails` : `activityStateKey.${key}.lastHeartbeatDetails`;
+}
 export function recordLastHeartbeatForStep(internals, workflowId, step, details) {
   let byStep = internals.lastHeartbeatDetailsByStep.get(workflowId);
   if (byStep === void 0) {
@@ -27,7 +33,7 @@ export function warnIfRetryMissingHeartbeat(internals, workflowId, step, attempt
     return;
   if (readLastHeartbeatForStep(internals, workflowId, step) !== void 0)
     return;
-  internals.engine.dispatchEvent(new DevelopmentWarningEvent(workflowId, `Activity retry (attempt ${attempt}) at step ${step} has no lastHeartbeatDetails. Either the previous attempt never recorded heartbeat details (it never called heartbeat(), or called it with no details), or the engine process restarted and discarded the in-memory heartbeat (it is not durable). The resumable-batch pattern only resumes across in-process retries; design the activity to restart cleanly when lastHeartbeatDetails is undefined.`, [`step.${step}.lastHeartbeatDetails`]));
+  internals.engine.dispatchEvent(new DevelopmentWarningEvent(workflowId, `Activity retry (attempt ${attempt}) at ${activityHeartbeatKeyLabel(step)} has no lastHeartbeatDetails. Either the previous attempt never recorded heartbeat details (it never called heartbeat(), or called it with no details), or the engine process restarted and discarded the in-memory heartbeat (it is not durable). The resumable-batch pattern only resumes across in-process retries; design the activity to restart cleanly when lastHeartbeatDetails is undefined.`, [activityHeartbeatFieldPath(step)]));
 }
 export function buildActivityContext(internals, workflowId, step, signal, completeAsync) {
   return {

package/dist/core/engine/activity-reconciliation.d.ts

@@ -102,6 +102,7 @@ export declare function resolveStartedActivityReconciliationRecord(internals: En
 }>;
 export declare function writeActivityReconciliationTransition(storage: Storage, reference: ActivityReconciliationReference, expectedRecord: ActivityReconciliationRecord, nextRecord: ActivityReconciliationRecord): Promise<void>;
 export declare function stageActivityReconciliationTransitionWithAtomicWorkflowCommit(internals: EngineInternals, workflowId: string, reference: ActivityReconciliationReference, expectedRecord: ActivityReconciliationRecord, nextRecord: ActivityReconciliationRecord): void;
+export declare function commitActivityReconciliationTransitionWithFencedWrite(internals: EngineInternals, reference: ActivityReconciliationReference, expectedRecord: ActivityReconciliationRecord, nextRecord: ActivityReconciliationRecord): Promise<void>;
 export declare function normalizePreDispatchVerificationResult(result: ActivityVerificationResult): 'not-completed' | 'completed-result-unavailable' | 'indeterminate' | {
     result: unknown;
 };

package/dist/core/engine/activity-reconciliation.js

@@ -7,6 +7,7 @@ import { decode, encode } from "../codec.js";
 import { assertPayloadWithinLimit } from "../payload-size.js";
 import { WeftError } from "../weft-error.js";
 import { stageAtomicWorkflowCommitSideEffects } from "./checkpoint-side-effects.js";
+import { commitFencedEngineWrite } from "./fenced-write.js";
 
 export class ActivityReconciliationCapabilityError extends WeftError {
   constructor() {
@@ -83,7 +84,7 @@ export async function resolveStartedActivityReconciliationRecord(internals, work
       ownerId: crypto.randomUUID(),
       updatedAt: internals.options.getNow()
     };
-    await writeActivityReconciliationTransition(internals.storage, reference, record, nextRecord);
+    await commitActivityReconciliationTransitionWithFencedWrite(internals, reference, record, nextRecord);
     return nextRecord;
   }
   if (normalized === "completed-result-unavailable")
@@ -92,7 +93,7 @@ export async function resolveStartedActivityReconciliationRecord(internals, work
     throw new ActivityReconciliationIndeterminateError(`Activity "${operation.activityName}" reconciliation is indeterminate.`);
   validateActivityResultForReconciliation(normalized.result, internals.options.payloadSizePolicy.maxBytes);
   const completedRecord = createCompletedActivityReconciliationRecord(record, normalized.result, internals.options.getNow());
-  await writeActivityReconciliationTransition(internals.storage, reference, record, completedRecord);
+  await commitActivityReconciliationTransitionWithFencedWrite(internals, reference, record, completedRecord);
   return { completedResult: normalized.result };
 }
 export async function writeActivityReconciliationTransition(storage, reference, expectedRecord, nextRecord) {
@@ -103,6 +104,10 @@ export async function writeActivityReconciliationTransition(storage, reference,
 export function stageActivityReconciliationTransitionWithAtomicWorkflowCommit(internals, workflowId, reference, expectedRecord, nextRecord) {
   stageAtomicWorkflowCommitSideEffects(internals, workflowId, buildActivityReconciliationTransitionSideEffects(reference, expectedRecord, nextRecord));
 }
+export async function commitActivityReconciliationTransitionWithFencedWrite(internals, reference, expectedRecord, nextRecord) {
+  const sideEffects = buildActivityReconciliationTransitionSideEffects(reference, expectedRecord, nextRecord);
+  await commitFencedEngineWrite(internals, sideEffects.operations, sideEffects.conditions, () => new ActivityReconciliationConflictError("Activity reconciliation completion lost compare-and-set ownership."));
+}
 function buildActivityReconciliationTransitionSideEffects(reference, expectedRecord, nextRecord) {
   return {
     conditions: [{ key: reference.key, expectedValue: encode(expectedRecord) }],

package/dist/core/engine/anonymous-signal-sequence.js

@@ -79,16 +79,18 @@ async function scanNextAnonymousSignalSequence(internals, workflowId) {
     if (sequence !== null && sequence >= nextSequence)
       nextSequence = sequence + 1;
   }
+  if (!Number.isSafeInteger(nextSequence))
+    throw Error(`Anonymous signal sequence overflow for workflow "${workflowId}": computed ${nextSequence}`);
   return nextSequence;
 }
 function extractAnonymousSignalSequence(key) {
-  const marker = ":anonymous%3A", markerIndex = key.lastIndexOf(marker);
-  if (markerIndex === -1)
+  const idComponent = key.slice(key.lastIndexOf(":") + 1), marker = "anonymous%3A";
+  if (!idComponent.startsWith(marker))
     return null;
-  const sequenceStart = markerIndex + marker.length, sequenceEnd = key.indexOf("%3A", sequenceStart);
+  const sequenceStart = marker.length, sequenceEnd = idComponent.indexOf("%3A", sequenceStart);
   if (sequenceEnd === -1)
     return null;
-  const sequence = Number(key.slice(sequenceStart, sequenceEnd));
+  const sequence = Number(idComponent.slice(sequenceStart, sequenceEnd));
   return Number.isSafeInteger(sequence) && sequence >= 0 ? sequence : null;
 }
 function decodeSignalSequence(bytes) {

package/dist/core/engine/async-activity-completion.d.ts

@@ -26,9 +26,6 @@
  * arrives after token recovery but before replay has adopted the workflow
  * generator, the engine buffers the completion or failure outcome and drains it
  * when replay reaches the same deterministic token.
- *
- * Internal-only. Imported from `src/core/engine/**` and the `ActivityContext`
- * construction path.
  */
 import type { OperationOutcome } from '../types.ts';
 import { WeftError } from '../weft-error.ts';
@@ -101,13 +98,14 @@ export type PendingAsyncActivityResolution = {
 /**
  * Derive the durable, deterministic task token for an async activity.
  *
- * The token is anchored to the workflow id, the workflow step index, and the
+ * The token is anchored to the workflow id, the activity state key, and the
  * dispatch attempt — all of which are stable across replay — so a workflow that
  * crashes while parked on an async activity mints the identical token after
- * recovery. `operationId` is deliberately excluded because it is regenerated on
- * every yield and would change on replay.
+ * recovery. Plain `ctx.run()` uses the workflow step as the state key.
+ * `operationId` is deliberately excluded because it is regenerated on every
+ * yield and would change on replay.
  */
-export declare function deriveAsyncActivityToken(workflowId: string, step: number, attempt: number): string;
+export declare function deriveAsyncActivityToken(workflowId: string, step: number | string, attempt: number): string;
 /**
  * Register a deferred activity: record it in memory and durably, then announce
  * the token via {@link ActivityAsyncPendingEvent}. Idempotent on `token`: if the

package/dist/core/engine/async-activity-completion.js

@@ -3,6 +3,8 @@ import { decode, encode } from "../codec.js";
 import { ActivityAsyncPendingEvent } from "../events.js";
 import { assertPayloadWithinLimit } from "../payload-size.js";
 import { WeftError } from "../weft-error.js";
+import { stageAtomicWorkflowCommitSideEffects } from "./checkpoint-side-effects.js";
+import { commitFencedEngineWrite } from "./fenced-write.js";
 const ASYNC_ACTIVITY_TOKEN_PREFIX = "async-act:v1", ASYNC_ACTIVITY_KEY_PREFIX = "async-act:v1:";
 export function asyncActivityWorkflowPrefix(workflowId) {
   return `${ASYNC_ACTIVITY_KEY_PREFIX}${encodeStorageKeyComponent(workflowId)}:`;
@@ -33,7 +35,7 @@ function isPersistedAsyncActivity(value) {
 export function deriveAsyncActivityToken(workflowId, step, attempt) {
   return `${ASYNC_ACTIVITY_TOKEN_PREFIX}:${workflowId}:${step}:${attempt}`;
 }
-function persistPendingAsyncActivity(storage, pending) {
+function buildPersistPendingAsyncActivityOperation(pending) {
   const record = {
     version: 1,
     token: pending.token,
@@ -44,12 +46,16 @@ function persistPendingAsyncActivity(storage, pending) {
     attempt: pending.attempt,
     createdAt: pending.createdAt
   };
-  return storage.put(KEYS.asyncActivity(pending.workflowId, pending.token), encode(record));
+  return {
+    type: "put",
+    key: KEYS.asyncActivity(pending.workflowId, pending.token),
+    value: encode(record)
+  };
 }
 export async function registerPendingAsyncActivity(internals, pending) {
   const alreadyRegistered = internals.pendingAsyncActivities.has(pending.token);
   internals.pendingAsyncActivities.set(pending.token, pending);
-  await persistPendingAsyncActivity(internals.storage, pending);
+  await commitFencedEngineWrite(internals, [buildPersistPendingAsyncActivityOperation(pending)], [], () => Error(`Async activity registration for token "${pending.token}" lost its precondition.`));
   if (!alreadyRegistered)
     internals.engine.dispatchEvent(new ActivityAsyncPendingEvent(pending.token, pending.operationId, pending.workflowId, pending.activityName, pending.attempt));
 }
@@ -87,12 +93,10 @@ async function consumePendingAsyncActivity(internals, token) {
   if (!pending)
     throw new AsyncActivityTokenNotFoundError(token);
   internals.pendingAsyncActivities.delete(token);
-  try {
-    await internals.storage.delete(KEYS.asyncActivity(pending.workflowId, token));
-  } catch (error) {
-    internals.pendingAsyncActivities.set(token, pending);
-    throw error;
-  }
+  stageAtomicWorkflowCommitSideEffects(internals, pending.workflowId, {
+    conditions: [],
+    operations: [{ type: "delete", key: KEYS.asyncActivity(pending.workflowId, token) }]
+  });
   return pending;
 }
 function shouldBufferPendingAsyncActivityResolution(internals, workflowId) {

package/dist/core/engine/bulk-operations-purge.js

@@ -10,6 +10,7 @@ import { buildIndexOperations } from "../search-attributes.js";
 import { buildWorkflowTagIndexOperations, normalizeWorkflowTags } from "../workflow-tags.js";
 import { asyncActivityWorkflowPrefix } from "./async-activity-completion.js";
 import { forgetCommittedCheckpointBytes } from "./checkpoint-commit-snapshots.js";
+import { commitFencedEngineWrite } from "./fenced-write.js";
 import { streamWorkflowStates } from "./listing.js";
 import { createTerminalCleanupTimerId } from "./state-utilities.js";
 import {
@@ -131,7 +132,7 @@ function getWorkflowRetentionDeadline(internals, state) {
 }
 export async function purgeWorkflow(internals, state, cleanupWaiters) {
   const deleteOperations = await collectWorkflowPurgeDeleteOperations(internals, state);
-  await internals.storage.batch(deleteOperations);
+  await commitFencedEngineWrite(internals, deleteOperations, [], () => Error(`Purge commit for workflow "${state.id}" lost its precondition.`));
   clearPurgedWorkflowInMemoryState(internals, state.id, cleanupWaiters);
 }
 export async function collectWorkflowPurgeDeleteOperations(internals, state) {

package/dist/core/engine/bulk-operations.js

@@ -1,7 +1,6 @@
 import {
   KEYS,
   requireStorageCapability,
-  storageConditionalBatch,
   storageHas
 } from "../../storage/interface.js";
 import { assertScopedBulkWorkflowFilter } from "../bulk-workflow-filter.js";
@@ -24,6 +23,10 @@ import {
   withBulkAuditEvent
 } from "./bulk-operations-shared.js";
 import { BulkDeleteRequiresTerminalWorkflowsError } from "./errors.js";
+import {
+  assertLeaseHeldForEngineWork,
+  commitFencedEngineWriteAllowingPreconditionFailure
+} from "./fenced-write.js";
 import { BULK_OPERATION_BATCH_SIZE } from "./listing.js";
 import { createTerminalCleanupTimerId } from "./state-utilities.js";
 import { loadWorkflowState, runSerializedWorkflowStateWrite } from "./storage-io.js";
@@ -132,6 +135,7 @@ async function runBulkFailedWorkflowRetry(internals, filter, options = {}) {
   return withBulkAuditEvent(internals, preparation, options, result, retried);
 }
 export async function retryFailedAll(internals, filter, options = {}) {
+  assertLeaseHeldForEngineWork(internals);
   return runBulkFailedWorkflowRetry(internals, filter, options);
 }
 async function retryFailedWorkflow(internals, workflowId) {
@@ -194,12 +198,9 @@ async function reactivateFailedWorkflowFromCheckpointSerialized(internals, workf
   throw Error(`Workflow concurrency admission for "${lastConcurrencyStateKey ?? workflowId}" changed too many times while retrying failed workflow "${workflowId}"`);
 }
 async function commitFailedWorkflowReactivation(internals, operations, conditions) {
-  if (conditions.length === 0) {
-    await internals.storage.batch(operations);
-    return !0;
-  }
-  requireStorageCapability(internals.storage, "conditionalBatch", "retry failed workflow");
-  return storageConditionalBatch(internals.storage, conditions, operations);
+  if (conditions.length > 0)
+    requireStorageCapability(internals.storage, "conditionalBatch", "retry failed workflow");
+  return commitFencedEngineWriteAllowingPreconditionFailure(internals, operations, conditions);
 }
 function buildReactivatedWorkflowState(internals, state) {
   const reactivatedState = {

package/dist/core/engine/callback-checkpoint-persistence.d.ts

@@ -0,0 +1,3 @@
+import type { ContextOperationRequest } from '../context.ts';
+import type { Engine } from './index.ts';
+export declare function persistCheckpointForDataOperation<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>, workflowId: string, operation: ContextOperationRequest): Promise<void>;

package/dist/core/engine/callback-checkpoint-persistence.js

@@ -0,0 +1,25 @@
+import { HISTORY_CIRCUIT_BREAKER_REASON } from "../types.js";
+import { validateAttributeValueSizes } from "./attributes-tags.js";
+import { createTerminationCallbacks } from "./callback-creators-core.js";
+import {
+  appendTimelineBatchOperations,
+  persistCheckpoint,
+  pruneCheckpointHistory
+} from "./checkpoint-io.js";
+import { getInternals } from "./internals.js";
+import { swallowPromiseRejection } from "./strategy-helpers.js";
+import { terminateWorkflow } from "./termination.js";
+export function persistCheckpointForDataOperation(engine, workflowId, operation) {
+  return persistCheckpoint(getInternals(engine), workflowId, operation, void 0, {
+    appendTimelineBatchOperations: (id, checkpointOperation, step, timestamp, operations) => appendTimelineBatchOperations(getInternals(engine), id, checkpointOperation, step, timestamp, operations),
+    swallowPromiseRejection: (promise) => {
+      swallowPromiseRejection(promise);
+    },
+    validateAttributeValueSizes,
+    pruneCheckpointHistory: (id, step) => pruneCheckpointHistory(getInternals(engine), id, step),
+    dispatchEvent: (event) => {
+      engine.dispatchEvent(event);
+    },
+    enforceHistoryCircuitBreaker: (id) => terminateWorkflow(getInternals(engine), id, "timed-out", createTerminationCallbacks(engine), HISTORY_CIRCUIT_BREAKER_REASON)
+  }, { timeline: "preserve-pending" });
+}

package/dist/core/engine/callback-creators-bundles.js

@@ -3,6 +3,7 @@ import {
   broadcast as broadcastFromInternals,
   dispatchPendingUpdateReceived as dispatchPendingUpdateReceivedFromBroadcast
 } from "./broadcast.js";
+import { persistCheckpointForDataOperation } from "./callback-checkpoint-persistence.js";
 import {
   createBroadcastCallbacks,
   createGuardCallbacks,
@@ -97,7 +98,9 @@ export function createConditionOperationCallbacks(engine) {
 }
 export function createDataOperationCallbacks(engine) {
   return {
-    runOperationWithResult: (workflowId, operation, execute) => runOperationWithResultForEngine(engine, workflowId, operation, execute)
+    runOperationWithResult: (workflowId, operation, execute) => runOperationWithResultForEngine(engine, workflowId, operation, execute),
+    persistCheckpoint: (workflowId, operation) => persistCheckpointForDataOperation(engine, workflowId, operation),
+    getActivityOperationCallbacks: () => createActivityOperationCallbacks(engine)
   };
 }
 export function createStateOperationCallbacks(engine) {

package/dist/core/engine/checkpoint-io.d.ts

@@ -6,6 +6,9 @@ type PendingTimelineEntryValue = {
     startedAt: number;
     entry: WorkflowTimelineEntry;
 };
+export type PersistCheckpointOptions = {
+    timeline?: 'record-operation' | 'preserve-pending';
+};
 type PersistCheckpointCallbacks = {
     appendTimelineBatchOperations: (workflowId: string, operation: ContextOperationRequest, step: number, timestamp: number, operations: BatchOperation[]) => PendingTimelineEntryValue;
     swallowPromiseRejection: (promise: Promise<void>) => void;
@@ -26,7 +29,7 @@ type DevelopmentCheckpointCallbacks = {
 };
 export declare function appendTimelineBatchOperations(internals: EngineInternals, workflowId: string, operation: ContextOperationRequest, step: number, timestamp: number, operations: BatchOperation[]): PendingTimelineEntryValue;
 /** Persist a workflow checkpoint, history entry, timeline record, and event log record. */
-export declare function persistCheckpoint(internals: EngineInternals, workflowId: string, operation: ContextOperationRequest, workerCheckpointBytes: ArrayBuffer | undefined, callbacks: PersistCheckpointCallbacks): Promise<void>;
+export declare function persistCheckpoint(internals: EngineInternals, workflowId: string, operation: ContextOperationRequest, workerCheckpointBytes: ArrayBuffer | undefined, callbacks: PersistCheckpointCallbacks, options?: PersistCheckpointOptions): Promise<void>;
 /** Delete the single checkpoint history entry that overflows the retention limit. */
 export declare function pruneCheckpointHistory(internals: EngineInternals, workflowId: string, currentStep: number): Promise<void>;
 /** Validate checkpoint serialization in development mode and dispatch warnings. */

package/dist/core/engine/checkpoint-io.js

@@ -60,13 +60,13 @@ export function appendTimelineBatchOperations(internals, workflowId, operation,
     entry
   };
 }
-export async function persistCheckpoint(internals, workflowId, operation, workerCheckpointBytes, callbacks) {
+export async function persistCheckpoint(internals, workflowId, operation, workerCheckpointBytes, callbacks, options = {}) {
   if (internals.inlineStrategy?.getContext(workflowId))
-    await persistInlineCheckpoint(internals, workflowId, operation, callbacks);
+    await persistInlineCheckpoint(internals, workflowId, operation, callbacks, options);
   else if (workerCheckpointBytes && workerCheckpointBytes.byteLength > 0)
-    await persistWorkerCheckpoint(internals, workflowId, operation, workerCheckpointBytes, callbacks);
+    await persistWorkerCheckpoint(internals, workflowId, operation, workerCheckpointBytes, callbacks, options);
 }
-async function persistInlineCheckpoint(internals, workflowId, operation, callbacks) {
+async function persistInlineCheckpoint(internals, workflowId, operation, callbacks, options) {
   const current = internals.checkpoints.get(workflowId), context = internals.inlineStrategy?.getContext(workflowId);
   if (!current || !context)
     return;
@@ -76,13 +76,13 @@ async function persistInlineCheckpoint(internals, workflowId, operation, callbac
     ...pendingAttributeChanges !== void 0 ? { searchAttributes: pendingAttributeChanges } : {}
   }), pruned = pruneCheckpointReplayState(advanced, resolvePendingOperationStep(operation, context.stepIndex)), commit = createCheckpointCommit(internals, workflowId, pruned.checkpoint, serializeCheckpoint(attachTransientCheckpointReplayPayload(pruned.checkpoint, pruned.replayPayload)), pruned.replayPayload);
   appendAttributeOperations(workflowId, commit, previousAttributes, pendingAttributeChanges, hasPendingAttributeChangesValue, callbacks);
-  await commitCheckpoint(internals, workflowId, operation, commit, callbacks);
+  await commitCheckpoint(internals, workflowId, operation, commit, callbacks, options);
   if (hasPendingAttributeChangesValue)
     callbacks.dispatchEvent(new AttributesChangedEvent(workflowId, { ...pendingAttributeChanges }));
 }
-async function persistWorkerCheckpoint(internals, workflowId, operation, workerCheckpointBytes, callbacks) {
+async function persistWorkerCheckpoint(internals, workflowId, operation, workerCheckpointBytes, callbacks, options) {
   const serialized = new Uint8Array(workerCheckpointBytes), checkpoint = deserializeCheckpoint(serialized), workerReplayPayload = readCheckpointReplayPayload(checkpoint), pruned = pruneCheckpointReplayState(checkpoint, resolvePendingOperationStep(operation, checkpoint.step)), replayPayload = mergeCheckpointReplayPayloads(workerReplayPayload, pruned.replayPayload), prunedSerialized = serializeCheckpoint(attachTransientCheckpointReplayPayload(pruned.checkpoint, replayPayload));
-  await commitCheckpoint(internals, workflowId, operation, createCheckpointCommit(internals, workflowId, pruned.checkpoint, prunedSerialized, replayPayload), callbacks);
+  await commitCheckpoint(internals, workflowId, operation, createCheckpointCommit(internals, workflowId, pruned.checkpoint, prunedSerialized, replayPayload), callbacks, options);
 }
 function createCheckpointCommit(internals, workflowId, checkpoint, serialized, replayPayload) {
   const operations = [
@@ -116,9 +116,9 @@ function appendAttributeOperations(workflowId, commit, previousAttributes, pendi
   });
   commit.operations.push(...buildIndexOperations(workflowId, previousAttributes, commit.checkpoint.searchAttributes));
 }
-async function commitCheckpoint(internals, workflowId, operation, commit, callbacks) {
+async function commitCheckpoint(internals, workflowId, operation, commit, callbacks, options) {
   dispatchCheckpointSizeWarning(internals, workflowId, commit, callbacks);
-  const nextPendingTimelineEntry = callbacks.appendTimelineBatchOperations(workflowId, operation, commit.step, commit.timestamp, commit.operations), { newHead, timestamp } = appendCheckpointEventLog(internals, workflowId, commit), retentionWindow = internals.options.historyPolicy.retentionWindow, compaction = retentionWindow === null ? null : await appendCompactionOperations(internals.storage, workflowId, newHead.sequence, retentionWindow, commit.operations), pendingSideEffects = takePendingAtomicWorkflowCommitSideEffects(internals, workflowId);
+  const nextPendingTimelineEntry = options.timeline === "preserve-pending" ? preservePendingTimelineEntry(internals, workflowId, commit.operations) : callbacks.appendTimelineBatchOperations(workflowId, operation, commit.step, commit.timestamp, commit.operations), { newHead, timestamp } = appendCheckpointEventLog(internals, workflowId, commit), retentionWindow = internals.options.historyPolicy.retentionWindow, compaction = retentionWindow === null ? null : await appendCompactionOperations(internals.storage, workflowId, newHead.sequence, retentionWindow, commit.operations), pendingSideEffects = takePendingAtomicWorkflowCommitSideEffects(internals, workflowId);
   if (pendingSideEffects !== void 0)
     commit.operations.push(...pendingSideEffects.operations);
   const storageSupportsConditionalBatch = internals.storage.capabilities().conditionalBatch, sideEffectConditions = checkpointSideEffectConditions(pendingSideEffects, storageSupportsConditionalBatch), conditions = buildCheckpointCommitConditions(workflowId, commit, storageSupportsConditionalBatch, sideEffectConditions);
@@ -129,7 +129,10 @@ async function commitCheckpoint(internals, workflowId, operation, commit, callba
     clearPendingAtomicWorkflowCommitSideEffects(internals, workflowId);
   if (commit.expectedSerialized !== void 0)
     rememberCommittedCheckpointBytes(internals, workflowId, commit.serialized);
-  internals.pendingTimelineEntries.set(workflowId, nextPendingTimelineEntry);
+  if (nextPendingTimelineEntry === void 0)
+    internals.pendingTimelineEntries.delete(workflowId);
+  else
+    internals.pendingTimelineEntries.set(workflowId, nextPendingTimelineEntry);
   internals.checkpoints.set(workflowId, commit.checkpoint);
   internals.eventLogHeads.set(workflowId, newHead);
   if (compaction !== null)
@@ -146,6 +149,12 @@ async function commitCheckpoint(internals, workflowId, operation, commit, callba
   if (historyEventLimitBreached(internals, newHead.sequence))
     await callbacks.enforceHistoryCircuitBreaker(workflowId);
 }
+function preservePendingTimelineEntry(internals, workflowId, operations) {
+  const pendingTimelineOperation = buildPendingTimelineOperation(internals, workflowId);
+  if (pendingTimelineOperation)
+    operations.push(pendingTimelineOperation);
+  return internals.pendingTimelineEntries.get(workflowId);
+}
 function checkpointSideEffectConditions(pendingSideEffects, storageSupportsConditionalBatch) {
   if (!storageSupportsConditionalBatch)
     return [];