@lostgradient/weft 0.3.0 → 0.4.0

package/dist/core/engine/bulk-operations.js

@@ -1,5 +1,15 @@
+import {
+  KEYS,
+  requireStorageCapability,
+  storageConditionalBatch,
+  storageHas
+} from "../../storage/interface.js";
 import { assertScopedBulkWorkflowFilter } from "../bulk-workflow-filter.js";
-import { bulkMutateWorkflowTags } from "./attributes-tags.js";
+import { deserializeCheckpoint } from "../checkpoint.js";
+import { decode, encode } from "../codec.js";
+import { buildTimerBatchOperations } from "../scheduler.js";
+import { buildIndexOperations } from "../search-attributes.js";
+import { buildTerminalWorkflowIndexOperations, bulkMutateWorkflowTags } from "./attributes-tags.js";
 import { purgeInternal, purgeWorkflow } from "./bulk-operations-purge.js";
 import {
   buildActionableBulkWorkflowFilter,
@@ -7,6 +17,7 @@ import {
   collectTerminalWorkflowSnapshots,
   normalizeBulkOperationOptions,
   prepareBulkOperation,
+  resolveBulkOperationConcurrency,
   shouldPersistBulkAudit,
   toBulkOperationError,
   validateBulkConfirmation,
@@ -14,10 +25,27 @@ import {
 } from "./bulk-operations-shared.js";
 import { BulkDeleteRequiresTerminalWorkflowsError } from "./errors.js";
 import { BULK_OPERATION_BATCH_SIZE } from "./listing.js";
-import { loadWorkflowState } from "./storage-io.js";
-import { isTerminalWorkflowStatus } from "./validation.js";
+import { createTerminalCleanupTimerId } from "./state-utilities.js";
+import { loadWorkflowState, runSerializedWorkflowStateWrite } from "./storage-io.js";
+import { decodeWorkflowState, isTerminalWorkflowStatus } from "./validation.js";
+import { buildWorkflowConcurrencyStartOperations } from "./workflow-concurrency.js";
+import { buildWorkflowVisibilityIndexTransition } from "./workflow-indexes.js";
 export { purgeInternal, TERMINAL_CLEANUP_DELAY_MS } from "./bulk-operations-purge.js";
-const ACTIVE_WORKFLOW_STATUSES = ["pending", "running", "suspended"];
+const ACTIVE_WORKFLOW_STATUSES = ["pending", "running", "suspended"], FAILED_WORKFLOW_STATUSES = ["failed"], CHECKPOINT_RETRY_CONCURRENCY_ADMISSION_MAX_ATTEMPTS = 5;
+async function runBulkWorkflowPool(items, concurrencyLimit, operation) {
+  const results = [];
+  for (let batchStart = 0;batchStart < items.length; batchStart += concurrencyLimit) {
+    const batchItems = items.slice(batchStart, batchStart + concurrencyLimit);
+    results.push(...await Promise.all(batchItems.map(async (item) => {
+      try {
+        return { item, status: "fulfilled", value: await operation(item) };
+      } catch (reason) {
+        return { item, status: "rejected", reason };
+      }
+    })));
+  }
+  return results;
+}
 export async function purge(internals, filter, cleanupWaiters) {
   return purgeInternal(internals, filter, { expiredOnly: !1, now: internals.options.getNow() }, cleanupWaiters);
 }
@@ -28,20 +56,36 @@ async function runBulkCancellation(internals, filter, options = {}) {
   if (options.dryRun === !0)
     return preparation.preview;
   validateBulkConfirmation(options, preparation);
-  const workflowIdsToCancel = preparation.workflowIds;
+  const workflowIdsToCancel = preparation.workflowIds, bulkConcurrency = resolveBulkOperationConcurrency(options);
   let cancelled = 0;
-  const errors = [];
-  for (const workflowId of workflowIdsToCancel)
+  const errors = [], cancellationResults = await runBulkWorkflowPool(workflowIdsToCancel, bulkConcurrency, async (workflowId) => {
     try {
       await internals.engine.cancel(workflowId);
-      if ((await loadWorkflowState(internals, workflowId))?.status === "cancelled") {
-        cancelled += 1;
-        continue;
-      }
-      errors.push({ id: workflowId, error: "Workflow no longer cancellable" });
+      if ((await loadWorkflowState(internals, workflowId))?.status === "cancelled")
+        return { status: "cancelled" };
+      return {
+        status: "failed",
+        error: { id: workflowId, error: "Workflow no longer cancellable" }
+      };
     } catch (error) {
-      errors.push(toBulkOperationError(internals, workflowId, error));
+      return {
+        status: "failed",
+        error: toBulkOperationError(internals, workflowId, error)
+      };
+    }
+  });
+  for (const cancellationResult of cancellationResults) {
+    if (cancellationResult.status === "rejected") {
+      errors.push(toBulkOperationError(internals, cancellationResult.item, cancellationResult.reason));
+      continue;
+    }
+    const { value } = cancellationResult;
+    if (value.status === "cancelled") {
+      cancelled += 1;
+      continue;
     }
+    errors.push(value.error);
+  }
   const result = { cancelled, failed: errors.length, errors };
   if (!shouldPersistBulkAudit(options))
     return result;
@@ -50,6 +94,145 @@ async function runBulkCancellation(internals, filter, options = {}) {
 export async function cancelAll(internals, filter, options = {}) {
   return runBulkCancellation(internals, filter, options);
 }
+async function runBulkFailedWorkflowRetry(internals, filter, options = {}) {
+  options = normalizeBulkOperationOptions(options);
+  assertScopedBulkWorkflowFilter(filter);
+  const actionableFilter = buildActionableBulkWorkflowFilter(internals, filter, FAILED_WORKFLOW_STATUSES), preparation = await prepareBulkOperation(internals, "retry-failed", actionableFilter, filter, {}, options);
+  if (options.dryRun === !0)
+    return preparation.preview;
+  validateBulkConfirmation(options, preparation);
+  const bulkConcurrency = resolveBulkOperationConcurrency(options);
+  let retried = 0;
+  const errors = [], retryResults = await runBulkWorkflowPool(preparation.workflowIds, bulkConcurrency, async (workflowId) => {
+    try {
+      await retryFailedWorkflow(internals, workflowId);
+      return { status: "retried" };
+    } catch (error) {
+      return {
+        status: "failed",
+        error: toBulkOperationError(internals, workflowId, error)
+      };
+    }
+  });
+  for (const retryResult of retryResults) {
+    if (retryResult.status === "rejected") {
+      errors.push(toBulkOperationError(internals, retryResult.item, retryResult.reason));
+      continue;
+    }
+    const { value } = retryResult;
+    if (value.status === "retried") {
+      retried += 1;
+      continue;
+    }
+    errors.push(value.error);
+  }
+  const result = { retried, failed: errors.length, errors };
+  if (!shouldPersistBulkAudit(options))
+    return result;
+  return withBulkAuditEvent(internals, preparation, options, result, retried);
+}
+export async function retryFailedAll(internals, filter, options = {}) {
+  return runBulkFailedWorkflowRetry(internals, filter, options);
+}
+async function retryFailedWorkflow(internals, workflowId) {
+  const state = await loadWorkflowState(internals, workflowId);
+  if (state === null)
+    throw Error("Workflow no longer exists");
+  if (state.status !== "failed")
+    throw Error(`Workflow is ${state.status}, not failed`);
+  if (await internals.storage.get(KEYS.checkpoint(workflowId)) !== null) {
+    await reactivateFailedWorkflowFromCheckpoint(internals, state);
+    await internals.engine.resume(workflowId);
+    return;
+  }
+  await internals.engine.start(state.type, state.input, {
+    id: workflowId,
+    onTerminalConflict: "start-new",
+    ...state.tags !== void 0 ? { tags: state.tags } : {}
+  });
+}
+async function reactivateFailedWorkflowFromCheckpoint(internals, state) {
+  const reactivated = await runSerializedWorkflowStateWrite(internals, state.id, async () => reactivateFailedWorkflowFromCheckpointSerialized(internals, state.id));
+  if (reactivated.terminalCleanupTimerId !== void 0)
+    await internals.scheduler.cancel(reactivated.terminalCleanupTimerId, state.id);
+}
+async function reactivateFailedWorkflowFromCheckpointSerialized(internals, workflowId) {
+  let lastConcurrencyStateKey;
+  for (let attempt = 0;attempt < CHECKPOINT_RETRY_CONCURRENCY_ADMISSION_MAX_ATTEMPTS; attempt += 1) {
+    const currentStateBytes = await internals.storage.get(KEYS.workflow(workflowId));
+    if (currentStateBytes === null)
+      throw Error("Workflow no longer exists");
+    const currentState = decodeWorkflowState(currentStateBytes);
+    if (currentState.status !== "failed")
+      throw Error(`Workflow is ${currentState.status}, not failed`);
+    const currentCheckpointBytes = await internals.storage.get(KEYS.checkpoint(workflowId));
+    if (currentCheckpointBytes === null)
+      throw Error("Checkpoint no longer exists");
+    const checkpoint = deserializeCheckpoint(currentCheckpointBytes), registration = internals.registrations.get(currentState.type);
+    if (registration === void 0)
+      throw Error(`No workflow registered with name "${currentState.type}" (needed to retry "${workflowId}")`);
+    const concurrencyStartOperations = registration.concurrency === void 0 ? void 0 : await buildWorkflowConcurrencyStartOperations(internals, currentState.type, workflowId, currentState.input, registration.concurrency);
+    lastConcurrencyStateKey = concurrencyStartOperations?.stateKey ?? lastConcurrencyStateKey;
+    const reactivatedState = buildReactivatedWorkflowState(internals, currentState), currentAttributes = await loadSearchAttributes(internals, workflowId), operations = [
+      ...buildTerminalWorkflowIndexOperations(currentState, reactivatedState),
+      { type: "put", key: KEYS.workflow(workflowId), value: encode(reactivatedState) },
+      ...buildWorkflowVisibilityIndexTransition(workflowId, currentState, reactivatedState).batchOps,
+      ...buildRetrySearchAttributeOperations(workflowId, currentAttributes, checkpoint.searchAttributes),
+      ...reactivatedState.executionDeadline === void 0 ? [] : buildTimerBatchOperations({
+        id: `deadline:${workflowId}`,
+        workflowId,
+        fireAt: reactivatedState.executionDeadline,
+        kind: "execution-deadline"
+      }),
+      ...concurrencyStartOperations?.operations ?? []
+    ], conditions = concurrencyStartOperations?.conditions ?? [];
+    if (await commitFailedWorkflowReactivation(internals, operations, conditions))
+      return {
+        terminalCleanupTimerId: currentState.terminalCleanupToken === void 0 ? void 0 : createTerminalCleanupTimerId(!1, currentState.terminalCleanupToken)
+      };
+  }
+  throw Error(`Workflow concurrency admission for "${lastConcurrencyStateKey ?? workflowId}" changed too many times while retrying failed workflow "${workflowId}"`);
+}
+async function commitFailedWorkflowReactivation(internals, operations, conditions) {
+  if (conditions.length === 0) {
+    await internals.storage.batch(operations);
+    return !0;
+  }
+  requireStorageCapability(internals.storage, "conditionalBatch", "retry failed workflow");
+  return storageConditionalBatch(internals.storage, conditions, operations);
+}
+function buildReactivatedWorkflowState(internals, state) {
+  const reactivatedState = {
+    ...state,
+    status: "running",
+    updatedAt: internals.options.getNow()
+  };
+  delete reactivatedState.error;
+  delete reactivatedState.errorStack;
+  delete reactivatedState.failureCategory;
+  delete reactivatedState.result;
+  delete reactivatedState.terminationReason;
+  delete reactivatedState.terminalCleanupToken;
+  return reactivatedState;
+}
+async function loadSearchAttributes(internals, workflowId) {
+  const attributeBytes = await internals.storage.get(KEYS.attribute(workflowId));
+  if (attributeBytes === null)
+    return {};
+  return decode(attributeBytes);
+}
+function buildRetrySearchAttributeOperations(workflowId, currentAttributes, checkpointAttributes) {
+  const operations = buildIndexOperations(workflowId, currentAttributes, checkpointAttributes);
+  if (Object.keys(checkpointAttributes).length === 0)
+    operations.push({ type: "delete", key: KEYS.attribute(workflowId) });
+  else
+    operations.push({
+      type: "put",
+      key: KEYS.attribute(workflowId),
+      value: encode(checkpointAttributes)
+    });
+  return operations;
+}
 export async function signalAll(internals, filter, name, payload, maybeOptions) {
   const options = normalizeBulkOperationOptions(maybeOptions ?? {});
   assertScopedBulkWorkflowFilter(filter);
@@ -59,15 +242,21 @@ export async function signalAll(internals, filter, name, payload, maybeOptions)
   if (options.dryRun === !0)
     return preparation.preview;
   validateBulkConfirmation(options, preparation);
-  const workflowIdsToSignal = preparation.workflowIds;
+  const workflowIdsToSignal = preparation.workflowIds, bulkConcurrency = resolveBulkOperationConcurrency(options);
   let signalled = 0, failed = 0;
-  for (const workflowId of workflowIdsToSignal)
+  const signalResults = await runBulkWorkflowPool(workflowIdsToSignal, bulkConcurrency, async (workflowId) => {
     try {
       await internals.engine.signal(workflowId, name, payload);
-      signalled += 1;
+      return { status: "signalled" };
     } catch {
-      failed += 1;
+      return { status: "failed" };
     }
+  });
+  for (const signalResult of signalResults)
+    if (signalResult.status === "fulfilled" && signalResult.value.status === "signalled")
+      signalled += 1;
+    else
+      failed += 1;
   const result = { signalled, failed };
   if (!shouldPersistBulkAudit(options))
     return result;
@@ -77,18 +266,40 @@ async function runBulkDeletion(internals, filter, cleanupWaiters, options = {})
   options = normalizeBulkOperationOptions(options);
   assertScopedBulkWorkflowFilter(filter);
   const candidateWorkflowSnapshots = await collectTerminalWorkflowSnapshots(internals, filter), preparation = buildBulkOperationPreparation("delete", filter, {}, candidateWorkflowSnapshots, options);
-  if (options.dryRun === !0)
-    return preparation.preview;
+  if (options.dryRun === !0) {
+    const skippedTeardownPending = [];
+    for (const workflowId of preparation.workflowIds)
+      if (await storageHas(internals.storage, KEYS.teardownOwed(workflowId)))
+        skippedTeardownPending.push(workflowId);
+    return skippedTeardownPending.length > 0 ? { ...preparation.preview, skippedTeardownPending } : preparation.preview;
+  }
   validateBulkConfirmation(options, preparation);
+  const bulkConcurrency = resolveBulkOperationConcurrency(options);
   let deleted = 0;
+  const skippedTeardownPending = [];
   for (let batchStart = 0;batchStart < preparation.workflowIds.length; batchStart += BULK_OPERATION_BATCH_SIZE) {
-    const batchWorkflowIds = preparation.workflowIds.slice(batchStart, batchStart + BULK_OPERATION_BATCH_SIZE), workflowStatesToDelete = await loadTerminalWorkflowStatesForBatch(internals, batchWorkflowIds);
-    for (const workflowState of workflowStatesToDelete) {
+    const batchWorkflowIds = preparation.workflowIds.slice(batchStart, batchStart + BULK_OPERATION_BATCH_SIZE), workflowStatesToDelete = await loadTerminalWorkflowStatesForBatch(internals, batchWorkflowIds), deletableStates = [];
+    for (const workflowState of workflowStatesToDelete)
+      if (await storageHas(internals.storage, KEYS.teardownOwed(workflowState.id)))
+        skippedTeardownPending.push(workflowState.id);
+      else
+        deletableStates.push(workflowState);
+    const deletionResults = await runBulkWorkflowPool(deletableStates, bulkConcurrency, async (workflowState) => {
       await purgeWorkflow(internals, workflowState, cleanupWaiters);
-      deleted += 1;
-    }
+      return workflowState.id;
+    }), deletionErrors = [];
+    for (const deletionResult of deletionResults)
+      if (deletionResult.status === "fulfilled")
+        deleted += 1;
+      else
+        deletionErrors.push(deletionResult.reason);
+    if (deletionErrors.length > 0)
+      throw AggregateError(deletionErrors, `Bulk delete failed for ${deletionErrors.length} workflow(s) after deleting ${deleted} workflow(s)`);
   }
-  const result = { deleted };
+  const result = {
+    deleted,
+    ...skippedTeardownPending.length > 0 ? { skippedTeardownPending } : {}
+  };
   if (!shouldPersistBulkAudit(options))
     return result;
   return withBulkAuditEvent(internals, preparation, options, result, deleted);

package/dist/core/engine/callback-creators-bundles.d.ts

@@ -9,16 +9,25 @@ import { type OperationWithCallerStack } from './operations-router.ts';
 import type { StateOperationCallbacks } from './operations-state.ts';
 import type { StreamOperationCallbacks } from './operations-stream.ts';
 import type { TimeOperationCallbacks } from './operations-time.ts';
+import type { ConditionOperationCallbacks } from './operations-wait-condition.ts';
 import { type UpdateCallbacks } from './updates.ts';
 export declare function createChildWorkflowOperationCallbacks<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>): ChildWorkflowOperationCallbacks;
 export declare function createActivityOperationCallbacks<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>): ActivityOperationCallbacks;
 export declare function createCoordinationOperationCallbacks<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>): CoordinationOperationCallbacks;
+export declare function createConditionOperationCallbacks<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>): ConditionOperationCallbacks;
 export declare function createDataOperationCallbacks<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>): DataOperationCallbacks;
 export declare function createStateOperationCallbacks<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>): StateOperationCallbacks;
 export declare function createStreamOperationCallbacks<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>): StreamOperationCallbacks;
 export declare function createTimeOperationCallbacks<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>): TimeOperationCallbacks;
 export declare function createUpdateCallbacks<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>): UpdateCallbacks;
 export declare function completeOperationForEngine<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>, workflowId: string, value: unknown): void;
+/**
+ * Fail the pending operation for a workflow, feeding the error to the generator
+ * so it re-throws at the `yield*` site (catchable by the workflow body). Used by
+ * the wait-condition processor when the user predicate throws, so a throwing
+ * predicate surfaces as a workflow failure instead of parking the run forever.
+ */
+export declare function failOperationForEngine<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>, workflowId: string, operation: OperationWithCallerStack, error: unknown): void;
 export declare function runOperationWithResultForEngine<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>, workflowId: string, operation: OperationWithCallerStack, execute: () => Promise<unknown>): Promise<void>;
 export declare function runOperationWithoutResultForEngine<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>, workflowId: string, operation: OperationWithCallerStack, execute: () => Promise<void>): Promise<void>;
 export declare function executeSubOperationForEngine<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>, workflowId: string, operation: ContextOperationRequest, signal?: AbortSignal, speculativeState?: import('./speculative-execution-state.ts').SpeculativeExecutionState): Promise<unknown>;

package/dist/core/engine/callback-creators-bundles.js

@@ -15,6 +15,7 @@ import {
   createReviewOperationCallbacks,
   handleScheduleTimerForEngine
 } from "./callback-creators-schedule.js";
+import { commitFencedEngineWrite } from "./fenced-write.js";
 import { guardTerminalWorkflow, guardTerminalWorkflowAfterCoordinatedRequest } from "./guards.js";
 import { getInternals } from "./internals.js";
 import {
@@ -27,6 +28,7 @@ import {
 } from "./lifecycle.js";
 import {
   completeOperation,
+  failOperation,
   runOperationWithResult,
   runOperationWithoutResult
 } from "./operations-router.js";
@@ -34,6 +36,7 @@ import { schedulePendingInlineUpdateDrain } from "./pending-updates.js";
 import { processReviewOperation } from "./reviews.js";
 import { loadWorkflowState, runSerializedWorkflowStateWrite } from "./storage-io.js";
 import {
+  feedOperationResult,
   getComposedActivityInterceptor,
   getComposedWorkflowInterceptor
 } from "./strategy-helpers.js";
@@ -41,7 +44,9 @@ import { executeSubOperation } from "./sub-operation.js";
 import {
   ensureTerminalCleanupTracked,
   failWorkflow,
-  runDeferredTerminalCleanup
+  finalizePendingTimelineEntry,
+  runDeferredTerminalCleanup,
+  runWorkflowFinalizer
 } from "./termination.js";
 import {
   createCoordinatedUpdateResponder,
@@ -60,6 +65,8 @@ export function createChildWorkflowOperationCallbacks(engine) {
 export function createActivityOperationCallbacks(engine) {
   return {
     runOperationWithResult: (workflowId, operation, execute) => runOperationWithResultForEngine(engine, workflowId, operation, execute),
+    finalizePendingTimelineEntry: (workflowId, status, value) => finalizePendingTimelineEntry(getInternals(engine), workflowId, status, value),
+    feedOperationResult: (workflowId, result, error) => feedOperationResult(getInternals(engine), workflowId, result, error),
     getComposedActivityInterceptor: () => getComposedActivityInterceptor(getInternals(engine)),
     getComposedWorkflowInterceptor: () => getComposedWorkflowInterceptor(getInternals(engine))
   };
@@ -72,6 +79,22 @@ export function createCoordinationOperationCallbacks(engine) {
     getActivityOperationCallbacks: () => createActivityOperationCallbacks(engine)
   };
 }
+export function createConditionOperationCallbacks(engine) {
+  return {
+    completeOperation: (workflowId, value) => completeOperationForEngine(engine, workflowId, value),
+    failOperation: (workflowId, operation, error) => failOperationForEngine(engine, workflowId, operation, error),
+    isWorkflowRunning: async (workflowId) => {
+      return (await loadWorkflowState(getInternals(engine), workflowId))?.status === "running";
+    },
+    scheduleConditionDeadline: (workflowId, step, fireAt) => getInternals(engine).scheduler.schedule({
+      id: `cond:${workflowId}:${step}`,
+      workflowId,
+      fireAt,
+      kind: "wait-condition"
+    }),
+    cancelConditionDeadline: (workflowId, step) => getInternals(engine).scheduler.cancel(`cond:${workflowId}:${step}`, workflowId)
+  };
+}
 export function createDataOperationCallbacks(engine) {
   return {
     runOperationWithResult: (workflowId, operation, execute) => runOperationWithResultForEngine(engine, workflowId, operation, execute)
@@ -92,6 +115,7 @@ export function createStreamOperationCallbacks(engine) {
 export function createTimeOperationCallbacks(engine) {
   return {
     completeOperation: (workflowId, value) => completeOperationForEngine(engine, workflowId, value),
+    dispatchEvent: (event) => engine.dispatchEvent(event),
     loadWorkflowState: (workflowId) => loadWorkflowState(getInternals(engine), workflowId),
     failWorkflow: (workflowId, error) => failWorkflow(getInternals(engine), workflowId, error, createTerminationCallbacks(engine)),
     runSerializedWorkflowStateWrite: (workflowId, writeOperation) => runSerializedWorkflowStateWrite(getInternals(engine), workflowId, writeOperation),
@@ -101,6 +125,7 @@ export function createTimeOperationCallbacks(engine) {
     loadWorkflowStartHeaders: (workflowId) => loadWorkflowStartHeaders(getInternals(engine), workflowId, createLifecycleCallbacks(engine)),
     parseStartOptionDuration: (value, fieldName) => parseStartOptionDuration(getInternals(engine), value, fieldName, createLifecycleCallbacks(engine)),
     runDeferredTerminalCleanup: (workflowId, timerId) => runDeferredTerminalCleanup(getInternals(engine), workflowId, timerId, createTerminationCallbacks(engine)),
+    runWorkflowFinalizer: (workflowId, timerId) => runWorkflowFinalizer(getInternals(engine), workflowId, timerId, createTerminationCallbacks(engine)),
     handleScheduleTimer: (entry) => handleScheduleTimerForEngine(engine, entry),
     timeout: (workflowId) => engine.timeout(workflowId),
     handleCleanupError: (source, error, workflowId) => createTerminationCallbacks(engine).handleCleanupError(source, error, workflowId)
@@ -126,7 +151,7 @@ export function createUpdateCallbacks(engine) {
 async function persistCoordinatedUpdateResponse(engine, workflowId, updateName, updateId, idempotencyKey, value) {
   const internals = getInternals(engine), responseOperations = internals.updateCoordinator.buildResponseOperations(updateId, workflowId, value, void 0, idempotencyKey);
   try {
-    await internals.storage.batch(responseOperations);
+    await commitFencedEngineWrite(internals, responseOperations, [], () => Error(`Coordinated update response for workflow "${workflowId}" lost its CAS race.`));
     engine.dispatchEvent(new UpdateCompletedEvent(updateId, workflowId, updateName, value));
     broadcastFromInternals(internals, { type: "update:completed", workflowId, updateId }, createBroadcastCallbacks(engine));
   } catch (error) {
@@ -136,6 +161,9 @@ async function persistCoordinatedUpdateResponse(engine, workflowId, updateName,
 export function completeOperationForEngine(engine, workflowId, value) {
   return completeOperation(getInternals(engine), workflowId, value, callRouterCallbacks(engine));
 }
+export function failOperationForEngine(engine, workflowId, operation, error) {
+  return failOperation(getInternals(engine), workflowId, operation, error, callRouterCallbacks(engine));
+}
 export async function runOperationWithResultForEngine(engine, workflowId, operation, execute) {
   return runOperationWithResult(getInternals(engine), workflowId, operation, execute, callRouterCallbacks(engine));
 }

package/dist/core/engine/callback-creators-core.js

@@ -18,7 +18,7 @@ import {
 import { resolveWorkflowTypeTarget } from "./registration.js";
 import { cleanupReviews } from "./reviews.js";
 import {
-  commitWorkflowStateOperations,
+  commitFencedWorkflowStateOperations,
   loadWorkflowState,
   runSerializedWorkflowStateWrite
 } from "./storage-io.js";
@@ -75,7 +75,8 @@ export function createLifecycleCallbacks(engine) {
     handleCleanupError: (source, error, workflowId) => createTerminationCallbacks(engine).handleCleanupError(source, error, workflowId),
     swallowPromiseRejection: (promise) => swallowPromiseRejection(promise),
     enforceHistoryCircuitBreaker: (workflowId) => terminateWorkflow(getInternals(engine), workflowId, "timed-out", createTerminationCallbacks(engine), HISTORY_CIRCUIT_BREAKER_REASON),
-    failWorkflowForUnavailableServices: (workflowId, error) => failWorkflow(getInternals(engine), workflowId, error, createTerminationCallbacks(engine), "system")
+    failWorkflowForUnavailableServices: (workflowId, error) => failWorkflow(getInternals(engine), workflowId, error, createTerminationCallbacks(engine), "system"),
+    failWorkflowForCheckpointDecodeError: (workflowId, error) => failWorkflow(getInternals(engine), workflowId, error, createTerminationCallbacks(engine), "system")
   };
 }
 export function createTerminationCallbacksWith(engine, handleScheduledWorkflowTerminal) {
@@ -91,7 +92,7 @@ export function createTerminationCallbacksWith(engine, handleScheduledWorkflowTe
     handleScheduledWorkflowTerminal,
     loadWorkflowState: (workflowId) => loadWorkflowState(getInternals(engine), workflowId),
     runSerializedWorkflowStateWrite: (workflowId, writeOperation) => runSerializedWorkflowStateWrite(getInternals(engine), workflowId, writeOperation),
-    commitWorkflowStateOperations: (state, operations) => commitWorkflowStateOperations(getInternals(engine), state, operations),
+    commitWorkflowStateOperations: (state, operations, options) => commitFencedWorkflowStateOperations(getInternals(engine), state, operations, options),
     cleanupReviews: (workflowId) => cleanupReviews(getInternals(engine), workflowId)
   };
 }
@@ -106,7 +107,8 @@ export function createTerminationCallbacks(engine) {
 }
 export function createRegistrationCallbacks(engine) {
   return {
-    ensureRetentionSweepInterval: () => ensureRetentionSweepIntervalCallable(engine)
+    ensureRetentionSweepInterval: () => ensureRetentionSweepIntervalCallable(engine),
+    dispatchEvent: (event) => engine.dispatchEvent(event)
   };
 }
 let retentionSweepIntervalCallable = () => {

package/dist/core/engine/callback-creators-router.js

@@ -1,6 +1,7 @@
 import {
   createActivityOperationCallbacks,
   createChildWorkflowOperationCallbacks,
+  createConditionOperationCallbacks,
   createCoordinationOperationCallbacks,
   createDataOperationCallbacks,
   createStateOperationCallbacks,
@@ -28,10 +29,12 @@ import {
   processMemoOperation,
   processOffloadOperation
 } from "./operations-data.js";
+import { completeOperation } from "./operations-router.js";
 import { processSpeculateOperation } from "./operations-speculate.js";
 import { processStateCommitOperation, processStateReadOperation } from "./operations-state.js";
 import { processStreamOperation } from "./operations-stream.js";
 import { processSleepOperation } from "./operations-time.js";
+import { processWaitConditionOperation } from "./operations-wait-condition.js";
 import { feedOperationResult } from "./strategy-helpers.js";
 import { processWaitReviewOperation } from "./sub-operation.js";
 import { finalizePendingTimelineEntry } from "./termination.js";
@@ -42,6 +45,13 @@ export function createOperationRouterCallbacks(engine) {
     processSleepOperation: (workflowId, operation) => processSleepOperation(getInternals(engine), workflowId, operation, createTimeOperationCallbacks(engine)),
     processWaitSignalOperation: (workflowId, operation) => processWaitSignalOperation(getInternals(engine), workflowId, operation, createCoordinationOperationCallbacks(engine)),
     processWaitUpdateOperation: (workflowId, operation) => processWaitUpdateOperation(getInternals(engine), workflowId, operation, createUpdateCallbacks(engine)),
+    processWaitConditionOperation: (workflowId, operation) => processWaitConditionOperation(getInternals(engine), workflowId, operation, createConditionOperationCallbacks(engine)),
+    processGetVersionOperation: async (workflowId, operation) => {
+      completeOperation(getInternals(engine), workflowId, operation.version, {
+        finalizePendingTimelineEntry: (id, status, value) => finalizePendingTimelineEntry(getInternals(engine), id, status, value),
+        feedOperationResult: (id, result, error) => feedOperationResult(getInternals(engine), id, result, error)
+      });
+    },
     processParallelOperation: (workflowId, operation) => processParallelOperation(getInternals(engine), workflowId, operation, createCoordinationOperationCallbacks(engine)),
     processRaceOperation: (workflowId, operation) => processRaceOperation(getInternals(engine), workflowId, operation, createCoordinationOperationCallbacks(engine)),
     processMemoOperation: (workflowId, operation) => processMemoOperation(getInternals(engine), workflowId, operation, createDataOperationCallbacks(engine)),

package/dist/core/engine/callback-creators-schedule.d.ts

@@ -6,8 +6,8 @@ export declare function createScheduleCallbacks<TWorkflows extends object, TActi
 export declare function createReviewOperationCallbacks<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>): ReviewOperationCallbacks;
 export declare function createSubmitReviewCallbacks<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>): SubmitReviewCallbacks;
 export declare function refreshScheduledWorkflowStateForEngine<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>, state: ScheduleState): Promise<RefreshedScheduleState>;
-export declare function startScheduledRunForEngine<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>, state: ScheduleState): Promise<string>;
-export declare function applyScheduleOccurrenceForEngine<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>, state: ScheduleState): Promise<ScheduleState>;
+export declare function startScheduledRunForEngine<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>, state: ScheduleState, occurrence?: number): Promise<string>;
+export declare function applyScheduleOccurrenceForEngine<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>, state: ScheduleState, occurrence?: number): Promise<ScheduleState>;
 export declare function settleBackfillScheduleStateForEngine<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>, state: ScheduleState): Promise<ScheduleState>;
 export declare function handleScheduleTimerForEngine<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>, entry: TimerEntry): Promise<void>;
 export declare function handleScheduledWorkflowTerminalForEngine<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>, workflowId: string): Promise<void>;

package/dist/core/engine/callback-creators-schedule.js

@@ -17,7 +17,7 @@ import {
 } from "./schedules.js";
 import { loadWorkflowState } from "./storage-io.js";
 import { feedOperationResult } from "./strategy-helpers.js";
-import { ensureTerminalCleanupTracked, failWorkflow } from "./termination.js";
+import { ensureTerminalCleanupTracked, failWorkflow, handleCleanupError } from "./termination.js";
 export function createScheduleCallbacks(engine) {
   return {
     startWorkflow: async (type, input, options, additionalStartOperations) => {
@@ -27,10 +27,14 @@ export function createScheduleCallbacks(engine) {
     cancelWorkflow: (workflowId) => engine.cancel(workflowId),
     getWorkflowResult: (workflowId) => engine.getHandle(workflowId).result(),
     refreshScheduledWorkflowState: (state) => refreshScheduledWorkflowStateForEngine(engine, state),
-    startScheduledRun: (state) => startScheduledRunForEngine(engine, state),
-    applyScheduleOccurrence: (state) => applyScheduleOccurrenceForEngine(engine, state),
+    startScheduledRun: (state, occurrence) => startScheduledRunForEngine(engine, state, occurrence),
+    applyScheduleOccurrence: (state, occurrence) => applyScheduleOccurrenceForEngine(engine, state, occurrence),
     settleBackfillScheduleState: (state) => settleBackfillScheduleStateForEngine(engine, state),
-    flushQueuedInlineWorkflowStartsDirectly: () => flushQueuedInlineWorkflowStartsDirectly(getInternals(engine), createInlineLaunchQueueCallbacks(engine))
+    flushQueuedInlineWorkflowStartsDirectly: () => flushQueuedInlineWorkflowStartsDirectly(getInternals(engine), createInlineLaunchQueueCallbacks(engine)),
+    failWorkflow: (workflowId, error) => failWorkflow(getInternals(engine), workflowId, error, createTerminationCallbacks(engine)),
+    handleCleanupError: (source, error, workflowId) => handleCleanupError(getInternals(engine), source, error, workflowId, {
+      dispatchEvent: (event) => engine.dispatchEvent(event)
+    })
   };
 }
 export function createReviewOperationCallbacks(engine) {
@@ -47,11 +51,11 @@ export function createSubmitReviewCallbacks(engine) {
 export async function refreshScheduledWorkflowStateForEngine(engine, state) {
   return refreshScheduledWorkflowState(getInternals(engine), state, createScheduleCallbacks(engine));
 }
-export async function startScheduledRunForEngine(engine, state) {
-  return startScheduledRun(getInternals(engine), state, createScheduleCallbacks(engine));
+export async function startScheduledRunForEngine(engine, state, occurrence) {
+  return startScheduledRun(getInternals(engine), state, createScheduleCallbacks(engine), occurrence);
 }
-export async function applyScheduleOccurrenceForEngine(engine, state) {
-  return applyScheduleOccurrence(getInternals(engine), state, createScheduleCallbacks(engine));
+export async function applyScheduleOccurrenceForEngine(engine, state, occurrence) {
+  return applyScheduleOccurrence(getInternals(engine), state, createScheduleCallbacks(engine), occurrence);
 }
 export async function settleBackfillScheduleStateForEngine(engine, state) {
   return settleBackfillScheduleState(getInternals(engine), state, createScheduleCallbacks(engine));

package/dist/core/engine/candidate-read-batching.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Shared bounded-concurrency size for materializing candidate workflow states.
+ * Keep this internal to engine query paths; it limits storage fan-out without
+ * changing result ordering.
+ */
+export declare const CONSTRAINED_ID_CHUNK_SIZE = 64;

package/dist/core/engine/candidate-read-batching.js

@@ -0,0 +1 @@
+export const CONSTRAINED_ID_CHUNK_SIZE = 64;