@lostgradient/weft 0.2.1 → 0.4.0

package/dist/core/engine/bulk-operations-purge.d.ts

@@ -1,3 +1,4 @@
+import type { BatchOperation } from '../../storage/interface.ts';
 import type { ListFilter, PurgeResult, WorkflowState } from '../types.ts';
 import type { EngineInternals } from './internals.ts';
 export declare const TERMINAL_CLEANUP_DELAY_MS = 60000;
@@ -9,3 +10,28 @@ export type PurgeParameters = {
 export type CleanupWaiters = (workflowId: string) => void;
 export declare function purgeInternal(internals: EngineInternals, filter: ListFilter | undefined, parameters: PurgeParameters, cleanupWaiters: CleanupWaiters): Promise<PurgeResult>;
 export declare function purgeWorkflow(internals: EngineInternals, state: WorkflowState, cleanupWaiters: CleanupWaiters): Promise<void>;
+/**
+ * Collect every storage delete operation that removing a workflow id entails —
+ * index deletes (search attributes + tags), the explicit key set (state,
+ * checkpoint, headers, services marker, update requests/responses, and the
+ * `wf:`/`ev:`/`sig:`/... prefix sweeps), and the visibility-index transition to
+ * `null`. Pure: it reads storage to discover keys but writes nothing, so a caller
+ * can fold the returned ops into a larger atomic batch instead of committing them
+ * standalone. {@link purgeWorkflow} commits them on their own; the
+ * `onTerminalConflict: 'start-new'` restart path prepends them to the create batch
+ * so purge-and-recreate land as one atomic unit (no window where the prior run is
+ * gone but the new one has not committed). Keep this the single source of truth
+ * for "what a purge deletes" — do not fork the delete-set.
+ */
+export declare function collectWorkflowPurgeDeleteOperations(internals: EngineInternals, state: WorkflowState): Promise<BatchOperation[]>;
+/**
+ * Drop every in-memory cache entry a purged workflow id owns (checkpoints,
+ * heartbeat details, event-log heads, version tuples, handle cache, result
+ * resolvers, headers, nesting depths, type map, pending async activities) and run
+ * `cleanupWaiters` to settle any pending signal/update/sleep waiters. Split from
+ * the storage batch in {@link collectWorkflowPurgeDeleteOperations} so the restart
+ * path can clear the OLD run's in-memory state up front — before the new run's
+ * caches are written under the reused id — while the durable delete still commits
+ * atomically with the create.
+ */
+export declare function clearPurgedWorkflowInMemoryState(internals: EngineInternals, workflowId: string, cleanupWaiters: CleanupWaiters): void;

package/dist/core/engine/bulk-operations-purge.js

@@ -1,6 +1,7 @@
 import {
   KEYS,
   encodeStorageKeyComponent,
+  storageHas,
   storageKeys,
   tryDecodeStorageKeyComponent
 } from "../../storage/interface.js";
@@ -25,7 +26,7 @@ export async function purgeInternal(internals, filter, parameters, cleanupWaiter
   let remainingOffset = manualOffset, deleted = 0;
   const workflowStateStream = parameters.expiredOnly && filter === void 0 ? streamExpiredRetentionWorkflowStates(internals, parameters.now) : streamWorkflowStates(internals, filter);
   for await (const state of workflowStateStream) {
-    if (!shouldPurgeWorkflowState(internals, state, parameters.expiredOnly, parameters.now))
+    if (!await shouldPurgeWorkflowState(internals, state, parameters.expiredOnly, parameters.now))
       continue;
     if (remainingOffset > 0) {
       remainingOffset -= 1;
@@ -110,9 +111,11 @@ function resolvePurgeLimit(manualLimit, fallbackLimit) {
     return manualLimit;
   return Math.min(manualLimit, fallbackLimit);
 }
-function shouldPurgeWorkflowState(internals, state, expiredOnly, now) {
+async function shouldPurgeWorkflowState(internals, state, expiredOnly, now) {
   if (!isTerminalWorkflowStatus(state.status))
     return !1;
+  if (await storageHas(internals.storage, KEYS.teardownOwed(state.id)))
+    return !1;
   if (!expiredOnly)
     return !0;
   const deadline = getWorkflowRetentionDeadline(internals, state);
@@ -127,16 +130,25 @@ function getWorkflowRetentionDeadline(internals, state) {
   return state.updatedAt + retentionMs;
 }
 export async function purgeWorkflow(internals, state, cleanupWaiters) {
+  const deleteOperations = await collectWorkflowPurgeDeleteOperations(internals, state);
+  await internals.storage.batch(deleteOperations);
+  clearPurgedWorkflowInMemoryState(internals, state.id, cleanupWaiters);
+}
+export async function collectWorkflowPurgeDeleteOperations(internals, state) {
   const workflowId = state.id, attributeBytes = await internals.storage.get(KEYS.attribute(workflowId)), deleteOperations = buildWorkflowIndexDeleteOperations(state, attributeBytes), deleteKeys = await collectWorkflowPurgeDeleteKeys(internals, state);
   appendKeyDeleteOperations(deleteOperations, deleteKeys);
   deleteOperations.push(...buildWorkflowVisibilityIndexTransition(workflowId, state, null).batchOps);
-  await internals.storage.batch(deleteOperations);
+  return deleteOperations;
+}
+export function clearPurgedWorkflowInMemoryState(internals, workflowId, cleanupWaiters) {
   forgetCommittedCheckpointBytes(internals, workflowId);
   internals.checkpoints.delete(workflowId);
   internals.heartbeatDetails.delete(workflowId);
+  internals.lastHeartbeatDetailsByStep.delete(workflowId);
   for (const [token, pending] of internals.pendingAsyncActivities)
     if (pending.workflowId === workflowId)
       internals.pendingAsyncActivities.delete(token);
+  internals.pendingAsyncActivityResolutions.delete(workflowId);
   internals.eventLogHeads.delete(workflowId);
   internals.workflowVersionTuples.delete(workflowId);
   internals.handleCache.delete(workflowId);
@@ -178,7 +190,10 @@ function buildBaseWorkflowDeleteKeys(state) {
     KEYS.checkpoint(state.id),
     KEYS.workflowHeaders(state.id),
     KEYS.terminalCleanupNeeded(state.id),
+    KEYS.workflowConcurrencyHolder(state.id),
     KEYS.workflowHasServices(state.id),
+    KEYS.finalizerState(state.id),
+    KEYS.teardownOwed(state.id),
     KEYS.attribute(state.id),
     KEYS.terminalWorkflow(state.updatedAt, state.id)
   ]);
@@ -216,11 +231,31 @@ async function addWorkflowPrefixDeleteKeys(storage, deleteKeys, workflowId) {
     for (const key of keys)
       deleteKeys.add(key);
   }
+  await addWorkflowLinkedFleetEventDeleteKeys(storage, deleteKeys, workflowId);
+}
+async function addWorkflowLinkedFleetEventDeleteKeys(storage, deleteKeys, workflowId) {
+  const prefix = KEYS.fleetEventByWorkflowPrefix(workflowId);
+  for await (const [key] of storage.scan(prefix)) {
+    deleteKeys.add(key);
+    const sequence = parseFleetEventSequenceFromWorkflowIndexKey(prefix, key);
+    if (sequence !== null)
+      deleteKeys.add(KEYS.fleetEvent(sequence));
+  }
+}
+function parseFleetEventSequenceFromWorkflowIndexKey(prefix, key) {
+  if (!key.startsWith(prefix))
+    return null;
+  const rawSequence = key.slice(prefix.length);
+  if (!/^\d+$/.test(rawSequence))
+    return null;
+  const sequence = Number(rawSequence);
+  return Number.isSafeInteger(sequence) ? sequence : null;
 }
 function workflowPurgePrefixes(workflowId) {
   const encodedWorkflowId = encodeStorageKeyComponent(workflowId);
   return [
     `wf:${encodedWorkflowId}:ckpt:`,
+    `wf:${encodedWorkflowId}:timeline:`,
     `ev:${encodedWorkflowId}:`,
     `sig:${encodedWorkflowId}:`,
     `review:${encodedWorkflowId}:`,

package/dist/core/engine/bulk-operations-shared.d.ts

@@ -1,6 +1,7 @@
 import type { BulkOperationAction, BulkOperationAuditEvent, BulkOperationCommitOptions, BulkOperationDryRunResult, BulkOperationError, BulkOperationOptions, BulkOperationScopeSummary, ListFilter, WorkflowState, WorkflowStatus } from '../types.ts';
 import type { EngineInternals } from './internals.ts';
 export declare const BULK_OPERATION_SAMPLE_LIMIT = 20;
+export declare const DEFAULT_BULK_OPERATION_CONCURRENCY = 1;
 export type BulkWorkflowSnapshot = {
     id: string;
     type: string;
@@ -22,6 +23,7 @@ export declare function withBulkAuditEvent<TResult extends object>(internals: En
     auditEvent: BulkOperationAuditEvent;
 }>;
 export declare function normalizeBulkOperationOptions<TOptions extends BulkOperationOptions>(options: TOptions): TOptions;
+export declare function resolveBulkOperationConcurrency(options: BulkOperationOptions): number;
 export declare function toBulkOperationError(_internals: EngineInternals, workflowId: string, error: unknown): BulkOperationError;
 export declare function snapshotMatchingWorkflowSnapshots(internals: EngineInternals, filter?: ListFilter): Promise<BulkWorkflowSnapshot[]>;
 export declare function workflowStateToBulkSnapshot(state: WorkflowState): BulkWorkflowSnapshot;

package/dist/core/engine/bulk-operations-shared.js

@@ -13,7 +13,7 @@ import {
 } from "./errors.js";
 import { streamWorkflowStateBatches } from "./listing.js";
 import { isTerminalWorkflowStatus } from "./validation.js";
-export const BULK_OPERATION_SAMPLE_LIMIT = 20;
+export const BULK_OPERATION_SAMPLE_LIMIT = 20, DEFAULT_BULK_OPERATION_CONCURRENCY = 1;
 const DEFAULT_BULK_OPERATION_PRINCIPAL = { method: "in-process" };
 export function buildActionableBulkWorkflowFilter(_internals, filter, actionableStatuses) {
   const effectiveStatuses = (filter.status === void 0 ? actionableStatuses : Array.isArray(filter.status) ? filter.status : [filter.status]).filter((status) => actionableStatuses.includes(status));
@@ -69,7 +69,7 @@ function summarizeBulkFilter(filter) {
   if (filter.attributes !== void 0)
     summary.attributes = filter.attributes.map((attribute) => ({
       key: searchAttributeName(attribute.key),
-      ...attribute.value === void 0 ? {} : { value: attribute.value },
+      ...attribute.value === void 0 ? {} : { value: Array.isArray(attribute.value) ? [...attribute.value] : attribute.value },
       ...attribute.gt === void 0 ? {} : { gt: attribute.gt },
       ...attribute.lt === void 0 ? {} : { lt: attribute.lt },
       ...attribute.gte === void 0 ? {} : { gte: attribute.gte },
@@ -136,6 +136,9 @@ async function persistBulkOperationAuditEvent(internals, preparation, options, a
   return auditEvent;
 }
 export function normalizeBulkOperationOptions(options) {
+  const bulkConcurrency = options.bulkConcurrency;
+  if (bulkConcurrency !== void 0 && (!Number.isSafeInteger(bulkConcurrency) || bulkConcurrency < 1))
+    throw Error('Field "bulkConcurrency" must be a positive integer');
   const confirmationToken = "confirmationToken" in options ? options.confirmationToken : void 0;
   if (confirmationToken !== void 0 && confirmationToken.length > MAX_BULK_CONFIRMATION_TOKEN_LENGTH)
     throw Error(`Field "confirmationToken" must be at most ${String(MAX_BULK_CONFIRMATION_TOKEN_LENGTH)} characters`);
@@ -143,6 +146,9 @@ export function normalizeBulkOperationOptions(options) {
     throw Error(`Field "requestId" must be at most ${String(MAX_BULK_OPERATION_REQUEST_ID_LENGTH)} characters`);
   return options;
 }
+export function resolveBulkOperationConcurrency(options) {
+  return options.bulkConcurrency ?? DEFAULT_BULK_OPERATION_CONCURRENCY;
+}
 export function toBulkOperationError(_internals, workflowId, error) {
   return {
     id: workflowId,

package/dist/core/engine/bulk-operations.d.ts

@@ -1,4 +1,4 @@
-import type { BulkCancelResult, BulkDeleteResult, BulkOperationCommitOptions, BulkOperationDryRunOptions, BulkOperationDryRunResult, BulkOperationOptions, BulkSignalAllCommitOptions, BulkSignalAllDryRunOptions, BulkSignalResult, BulkTagResult, ListFilter, PurgeResult } from '../types.ts';
+import type { BulkCancelResult, BulkDeleteResult, BulkOperationCommitOptions, BulkOperationDryRunOptions, BulkOperationDryRunResult, BulkOperationOptions, BulkRetryFailedResult, BulkSignalAllCommitOptions, BulkSignalAllDryRunOptions, BulkSignalResult, BulkTagResult, ListFilter, PurgeResult } from '../types.ts';
 import { type CleanupWaiters } from './bulk-operations-purge.ts';
 import type { EngineInternals } from './internals.ts';
 export { purgeInternal, TERMINAL_CLEANUP_DELAY_MS } from './bulk-operations-purge.ts';
@@ -6,6 +6,9 @@ export declare function purge(internals: EngineInternals, filter: ListFilter | u
 export declare function cancelAll(internals: EngineInternals, filter: ListFilter, options: BulkOperationDryRunOptions): Promise<BulkOperationDryRunResult>;
 export declare function cancelAll(internals: EngineInternals, filter: ListFilter, options?: BulkOperationCommitOptions): Promise<BulkCancelResult>;
 export declare function cancelAll(internals: EngineInternals, filter: ListFilter, options?: BulkOperationDryRunOptions | BulkOperationCommitOptions): Promise<BulkCancelResult | BulkOperationDryRunResult>;
+export declare function retryFailedAll(internals: EngineInternals, filter: ListFilter, options: BulkOperationDryRunOptions): Promise<BulkOperationDryRunResult>;
+export declare function retryFailedAll(internals: EngineInternals, filter: ListFilter, options?: BulkOperationCommitOptions): Promise<BulkRetryFailedResult>;
+export declare function retryFailedAll(internals: EngineInternals, filter: ListFilter, options?: BulkOperationDryRunOptions | BulkOperationCommitOptions): Promise<BulkRetryFailedResult | BulkOperationDryRunResult>;
 export declare function signalAll(internals: EngineInternals, filter: ListFilter, name: string, payload: unknown, options: BulkSignalAllDryRunOptions): Promise<BulkOperationDryRunResult>;
 export declare function signalAll(internals: EngineInternals, filter: ListFilter, name: string, payload: unknown, options: BulkSignalAllCommitOptions): Promise<BulkSignalResult>;
 export declare function signalAll(internals: EngineInternals, filter: ListFilter, name: string, payload?: unknown, options?: BulkOperationCommitOptions): Promise<BulkSignalResult>;

package/dist/core/engine/bulk-operations.js

@@ -1,5 +1,15 @@
+import {
+  KEYS,
+  requireStorageCapability,
+  storageConditionalBatch,
+  storageHas
+} from "../../storage/interface.js";
 import { assertScopedBulkWorkflowFilter } from "../bulk-workflow-filter.js";
-import { bulkMutateWorkflowTags } from "./attributes-tags.js";
+import { deserializeCheckpoint } from "../checkpoint.js";
+import { decode, encode } from "../codec.js";
+import { buildTimerBatchOperations } from "../scheduler.js";
+import { buildIndexOperations } from "../search-attributes.js";
+import { buildTerminalWorkflowIndexOperations, bulkMutateWorkflowTags } from "./attributes-tags.js";
 import { purgeInternal, purgeWorkflow } from "./bulk-operations-purge.js";
 import {
   buildActionableBulkWorkflowFilter,
@@ -7,6 +17,7 @@ import {
   collectTerminalWorkflowSnapshots,
   normalizeBulkOperationOptions,
   prepareBulkOperation,
+  resolveBulkOperationConcurrency,
   shouldPersistBulkAudit,
   toBulkOperationError,
   validateBulkConfirmation,
@@ -14,10 +25,27 @@ import {
 } from "./bulk-operations-shared.js";
 import { BulkDeleteRequiresTerminalWorkflowsError } from "./errors.js";
 import { BULK_OPERATION_BATCH_SIZE } from "./listing.js";
-import { loadWorkflowState } from "./storage-io.js";
-import { isTerminalWorkflowStatus } from "./validation.js";
+import { createTerminalCleanupTimerId } from "./state-utilities.js";
+import { loadWorkflowState, runSerializedWorkflowStateWrite } from "./storage-io.js";
+import { decodeWorkflowState, isTerminalWorkflowStatus } from "./validation.js";
+import { buildWorkflowConcurrencyStartOperations } from "./workflow-concurrency.js";
+import { buildWorkflowVisibilityIndexTransition } from "./workflow-indexes.js";
 export { purgeInternal, TERMINAL_CLEANUP_DELAY_MS } from "./bulk-operations-purge.js";
-const ACTIVE_WORKFLOW_STATUSES = ["pending", "running"];
+const ACTIVE_WORKFLOW_STATUSES = ["pending", "running", "suspended"], FAILED_WORKFLOW_STATUSES = ["failed"], CHECKPOINT_RETRY_CONCURRENCY_ADMISSION_MAX_ATTEMPTS = 5;
+async function runBulkWorkflowPool(items, concurrencyLimit, operation) {
+  const results = [];
+  for (let batchStart = 0;batchStart < items.length; batchStart += concurrencyLimit) {
+    const batchItems = items.slice(batchStart, batchStart + concurrencyLimit);
+    results.push(...await Promise.all(batchItems.map(async (item) => {
+      try {
+        return { item, status: "fulfilled", value: await operation(item) };
+      } catch (reason) {
+        return { item, status: "rejected", reason };
+      }
+    })));
+  }
+  return results;
+}
 export async function purge(internals, filter, cleanupWaiters) {
   return purgeInternal(internals, filter, { expiredOnly: !1, now: internals.options.getNow() }, cleanupWaiters);
 }
@@ -28,20 +56,36 @@ async function runBulkCancellation(internals, filter, options = {}) {
   if (options.dryRun === !0)
     return preparation.preview;
   validateBulkConfirmation(options, preparation);
-  const workflowIdsToCancel = preparation.workflowIds;
+  const workflowIdsToCancel = preparation.workflowIds, bulkConcurrency = resolveBulkOperationConcurrency(options);
   let cancelled = 0;
-  const errors = [];
-  for (const workflowId of workflowIdsToCancel)
+  const errors = [], cancellationResults = await runBulkWorkflowPool(workflowIdsToCancel, bulkConcurrency, async (workflowId) => {
     try {
       await internals.engine.cancel(workflowId);
-      if ((await loadWorkflowState(internals, workflowId))?.status === "cancelled") {
-        cancelled += 1;
-        continue;
-      }
-      errors.push({ id: workflowId, error: "Workflow no longer cancellable" });
+      if ((await loadWorkflowState(internals, workflowId))?.status === "cancelled")
+        return { status: "cancelled" };
+      return {
+        status: "failed",
+        error: { id: workflowId, error: "Workflow no longer cancellable" }
+      };
     } catch (error) {
-      errors.push(toBulkOperationError(internals, workflowId, error));
+      return {
+        status: "failed",
+        error: toBulkOperationError(internals, workflowId, error)
+      };
+    }
+  });
+  for (const cancellationResult of cancellationResults) {
+    if (cancellationResult.status === "rejected") {
+      errors.push(toBulkOperationError(internals, cancellationResult.item, cancellationResult.reason));
+      continue;
+    }
+    const { value } = cancellationResult;
+    if (value.status === "cancelled") {
+      cancelled += 1;
+      continue;
     }
+    errors.push(value.error);
+  }
   const result = { cancelled, failed: errors.length, errors };
   if (!shouldPersistBulkAudit(options))
     return result;
@@ -50,6 +94,145 @@ async function runBulkCancellation(internals, filter, options = {}) {
 export async function cancelAll(internals, filter, options = {}) {
   return runBulkCancellation(internals, filter, options);
 }
+async function runBulkFailedWorkflowRetry(internals, filter, options = {}) {
+  options = normalizeBulkOperationOptions(options);
+  assertScopedBulkWorkflowFilter(filter);
+  const actionableFilter = buildActionableBulkWorkflowFilter(internals, filter, FAILED_WORKFLOW_STATUSES), preparation = await prepareBulkOperation(internals, "retry-failed", actionableFilter, filter, {}, options);
+  if (options.dryRun === !0)
+    return preparation.preview;
+  validateBulkConfirmation(options, preparation);
+  const bulkConcurrency = resolveBulkOperationConcurrency(options);
+  let retried = 0;
+  const errors = [], retryResults = await runBulkWorkflowPool(preparation.workflowIds, bulkConcurrency, async (workflowId) => {
+    try {
+      await retryFailedWorkflow(internals, workflowId);
+      return { status: "retried" };
+    } catch (error) {
+      return {
+        status: "failed",
+        error: toBulkOperationError(internals, workflowId, error)
+      };
+    }
+  });
+  for (const retryResult of retryResults) {
+    if (retryResult.status === "rejected") {
+      errors.push(toBulkOperationError(internals, retryResult.item, retryResult.reason));
+      continue;
+    }
+    const { value } = retryResult;
+    if (value.status === "retried") {
+      retried += 1;
+      continue;
+    }
+    errors.push(value.error);
+  }
+  const result = { retried, failed: errors.length, errors };
+  if (!shouldPersistBulkAudit(options))
+    return result;
+  return withBulkAuditEvent(internals, preparation, options, result, retried);
+}
+export async function retryFailedAll(internals, filter, options = {}) {
+  return runBulkFailedWorkflowRetry(internals, filter, options);
+}
+async function retryFailedWorkflow(internals, workflowId) {
+  const state = await loadWorkflowState(internals, workflowId);
+  if (state === null)
+    throw Error("Workflow no longer exists");
+  if (state.status !== "failed")
+    throw Error(`Workflow is ${state.status}, not failed`);
+  if (await internals.storage.get(KEYS.checkpoint(workflowId)) !== null) {
+    await reactivateFailedWorkflowFromCheckpoint(internals, state);
+    await internals.engine.resume(workflowId);
+    return;
+  }
+  await internals.engine.start(state.type, state.input, {
+    id: workflowId,
+    onTerminalConflict: "start-new",
+    ...state.tags !== void 0 ? { tags: state.tags } : {}
+  });
+}
+async function reactivateFailedWorkflowFromCheckpoint(internals, state) {
+  const reactivated = await runSerializedWorkflowStateWrite(internals, state.id, async () => reactivateFailedWorkflowFromCheckpointSerialized(internals, state.id));
+  if (reactivated.terminalCleanupTimerId !== void 0)
+    await internals.scheduler.cancel(reactivated.terminalCleanupTimerId, state.id);
+}
+async function reactivateFailedWorkflowFromCheckpointSerialized(internals, workflowId) {
+  let lastConcurrencyStateKey;
+  for (let attempt = 0;attempt < CHECKPOINT_RETRY_CONCURRENCY_ADMISSION_MAX_ATTEMPTS; attempt += 1) {
+    const currentStateBytes = await internals.storage.get(KEYS.workflow(workflowId));
+    if (currentStateBytes === null)
+      throw Error("Workflow no longer exists");
+    const currentState = decodeWorkflowState(currentStateBytes);
+    if (currentState.status !== "failed")
+      throw Error(`Workflow is ${currentState.status}, not failed`);
+    const currentCheckpointBytes = await internals.storage.get(KEYS.checkpoint(workflowId));
+    if (currentCheckpointBytes === null)
+      throw Error("Checkpoint no longer exists");
+    const checkpoint = deserializeCheckpoint(currentCheckpointBytes), registration = internals.registrations.get(currentState.type);
+    if (registration === void 0)
+      throw Error(`No workflow registered with name "${currentState.type}" (needed to retry "${workflowId}")`);
+    const concurrencyStartOperations = registration.concurrency === void 0 ? void 0 : await buildWorkflowConcurrencyStartOperations(internals, currentState.type, workflowId, currentState.input, registration.concurrency);
+    lastConcurrencyStateKey = concurrencyStartOperations?.stateKey ?? lastConcurrencyStateKey;
+    const reactivatedState = buildReactivatedWorkflowState(internals, currentState), currentAttributes = await loadSearchAttributes(internals, workflowId), operations = [
+      ...buildTerminalWorkflowIndexOperations(currentState, reactivatedState),
+      { type: "put", key: KEYS.workflow(workflowId), value: encode(reactivatedState) },
+      ...buildWorkflowVisibilityIndexTransition(workflowId, currentState, reactivatedState).batchOps,
+      ...buildRetrySearchAttributeOperations(workflowId, currentAttributes, checkpoint.searchAttributes),
+      ...reactivatedState.executionDeadline === void 0 ? [] : buildTimerBatchOperations({
+        id: `deadline:${workflowId}`,
+        workflowId,
+        fireAt: reactivatedState.executionDeadline,
+        kind: "execution-deadline"
+      }),
+      ...concurrencyStartOperations?.operations ?? []
+    ], conditions = concurrencyStartOperations?.conditions ?? [];
+    if (await commitFailedWorkflowReactivation(internals, operations, conditions))
+      return {
+        terminalCleanupTimerId: currentState.terminalCleanupToken === void 0 ? void 0 : createTerminalCleanupTimerId(!1, currentState.terminalCleanupToken)
+      };
+  }
+  throw Error(`Workflow concurrency admission for "${lastConcurrencyStateKey ?? workflowId}" changed too many times while retrying failed workflow "${workflowId}"`);
+}
+async function commitFailedWorkflowReactivation(internals, operations, conditions) {
+  if (conditions.length === 0) {
+    await internals.storage.batch(operations);
+    return !0;
+  }
+  requireStorageCapability(internals.storage, "conditionalBatch", "retry failed workflow");
+  return storageConditionalBatch(internals.storage, conditions, operations);
+}
+function buildReactivatedWorkflowState(internals, state) {
+  const reactivatedState = {
+    ...state,
+    status: "running",
+    updatedAt: internals.options.getNow()
+  };
+  delete reactivatedState.error;
+  delete reactivatedState.errorStack;
+  delete reactivatedState.failureCategory;
+  delete reactivatedState.result;
+  delete reactivatedState.terminationReason;
+  delete reactivatedState.terminalCleanupToken;
+  return reactivatedState;
+}
+async function loadSearchAttributes(internals, workflowId) {
+  const attributeBytes = await internals.storage.get(KEYS.attribute(workflowId));
+  if (attributeBytes === null)
+    return {};
+  return decode(attributeBytes);
+}
+function buildRetrySearchAttributeOperations(workflowId, currentAttributes, checkpointAttributes) {
+  const operations = buildIndexOperations(workflowId, currentAttributes, checkpointAttributes);
+  if (Object.keys(checkpointAttributes).length === 0)
+    operations.push({ type: "delete", key: KEYS.attribute(workflowId) });
+  else
+    operations.push({
+      type: "put",
+      key: KEYS.attribute(workflowId),
+      value: encode(checkpointAttributes)
+    });
+  return operations;
+}
 export async function signalAll(internals, filter, name, payload, maybeOptions) {
   const options = normalizeBulkOperationOptions(maybeOptions ?? {});
   assertScopedBulkWorkflowFilter(filter);
@@ -59,15 +242,21 @@ export async function signalAll(internals, filter, name, payload, maybeOptions)
   if (options.dryRun === !0)
     return preparation.preview;
   validateBulkConfirmation(options, preparation);
-  const workflowIdsToSignal = preparation.workflowIds;
+  const workflowIdsToSignal = preparation.workflowIds, bulkConcurrency = resolveBulkOperationConcurrency(options);
   let signalled = 0, failed = 0;
-  for (const workflowId of workflowIdsToSignal)
+  const signalResults = await runBulkWorkflowPool(workflowIdsToSignal, bulkConcurrency, async (workflowId) => {
     try {
       await internals.engine.signal(workflowId, name, payload);
-      signalled += 1;
+      return { status: "signalled" };
     } catch {
-      failed += 1;
+      return { status: "failed" };
     }
+  });
+  for (const signalResult of signalResults)
+    if (signalResult.status === "fulfilled" && signalResult.value.status === "signalled")
+      signalled += 1;
+    else
+      failed += 1;
   const result = { signalled, failed };
   if (!shouldPersistBulkAudit(options))
     return result;
@@ -77,18 +266,40 @@ async function runBulkDeletion(internals, filter, cleanupWaiters, options = {})
   options = normalizeBulkOperationOptions(options);
   assertScopedBulkWorkflowFilter(filter);
   const candidateWorkflowSnapshots = await collectTerminalWorkflowSnapshots(internals, filter), preparation = buildBulkOperationPreparation("delete", filter, {}, candidateWorkflowSnapshots, options);
-  if (options.dryRun === !0)
-    return preparation.preview;
+  if (options.dryRun === !0) {
+    const skippedTeardownPending = [];
+    for (const workflowId of preparation.workflowIds)
+      if (await storageHas(internals.storage, KEYS.teardownOwed(workflowId)))
+        skippedTeardownPending.push(workflowId);
+    return skippedTeardownPending.length > 0 ? { ...preparation.preview, skippedTeardownPending } : preparation.preview;
+  }
   validateBulkConfirmation(options, preparation);
+  const bulkConcurrency = resolveBulkOperationConcurrency(options);
   let deleted = 0;
+  const skippedTeardownPending = [];
   for (let batchStart = 0;batchStart < preparation.workflowIds.length; batchStart += BULK_OPERATION_BATCH_SIZE) {
-    const batchWorkflowIds = preparation.workflowIds.slice(batchStart, batchStart + BULK_OPERATION_BATCH_SIZE), workflowStatesToDelete = await loadTerminalWorkflowStatesForBatch(internals, batchWorkflowIds);
-    for (const workflowState of workflowStatesToDelete) {
+    const batchWorkflowIds = preparation.workflowIds.slice(batchStart, batchStart + BULK_OPERATION_BATCH_SIZE), workflowStatesToDelete = await loadTerminalWorkflowStatesForBatch(internals, batchWorkflowIds), deletableStates = [];
+    for (const workflowState of workflowStatesToDelete)
+      if (await storageHas(internals.storage, KEYS.teardownOwed(workflowState.id)))
+        skippedTeardownPending.push(workflowState.id);
+      else
+        deletableStates.push(workflowState);
+    const deletionResults = await runBulkWorkflowPool(deletableStates, bulkConcurrency, async (workflowState) => {
       await purgeWorkflow(internals, workflowState, cleanupWaiters);
-      deleted += 1;
-    }
+      return workflowState.id;
+    }), deletionErrors = [];
+    for (const deletionResult of deletionResults)
+      if (deletionResult.status === "fulfilled")
+        deleted += 1;
+      else
+        deletionErrors.push(deletionResult.reason);
+    if (deletionErrors.length > 0)
+      throw AggregateError(deletionErrors, `Bulk delete failed for ${deletionErrors.length} workflow(s) after deleting ${deleted} workflow(s)`);
   }
-  const result = { deleted };
+  const result = {
+    deleted,
+    ...skippedTeardownPending.length > 0 ? { skippedTeardownPending } : {}
+  };
   if (!shouldPersistBulkAudit(options))
     return result;
   return withBulkAuditEvent(internals, preparation, options, result, deleted);

package/dist/core/engine/callback-creators-bundles.d.ts

@@ -9,16 +9,25 @@ import { type OperationWithCallerStack } from './operations-router.ts';
 import type { StateOperationCallbacks } from './operations-state.ts';
 import type { StreamOperationCallbacks } from './operations-stream.ts';
 import type { TimeOperationCallbacks } from './operations-time.ts';
+import type { ConditionOperationCallbacks } from './operations-wait-condition.ts';
 import { type UpdateCallbacks } from './updates.ts';
 export declare function createChildWorkflowOperationCallbacks<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>): ChildWorkflowOperationCallbacks;
 export declare function createActivityOperationCallbacks<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>): ActivityOperationCallbacks;
 export declare function createCoordinationOperationCallbacks<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>): CoordinationOperationCallbacks;
+export declare function createConditionOperationCallbacks<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>): ConditionOperationCallbacks;
 export declare function createDataOperationCallbacks<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>): DataOperationCallbacks;
 export declare function createStateOperationCallbacks<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>): StateOperationCallbacks;
 export declare function createStreamOperationCallbacks<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>): StreamOperationCallbacks;
 export declare function createTimeOperationCallbacks<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>): TimeOperationCallbacks;
 export declare function createUpdateCallbacks<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>): UpdateCallbacks;
 export declare function completeOperationForEngine<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>, workflowId: string, value: unknown): void;
+/**
+ * Fail the pending operation for a workflow, feeding the error to the generator
+ * so it re-throws at the `yield*` site (catchable by the workflow body). Used by
+ * the wait-condition processor when the user predicate throws, so a throwing
+ * predicate surfaces as a workflow failure instead of parking the run forever.
+ */
+export declare function failOperationForEngine<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>, workflowId: string, operation: OperationWithCallerStack, error: unknown): void;
 export declare function runOperationWithResultForEngine<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>, workflowId: string, operation: OperationWithCallerStack, execute: () => Promise<unknown>): Promise<void>;
 export declare function runOperationWithoutResultForEngine<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>, workflowId: string, operation: OperationWithCallerStack, execute: () => Promise<void>): Promise<void>;
 export declare function executeSubOperationForEngine<TWorkflows extends object, TActivities extends object>(engine: Engine<TWorkflows, TActivities>, workflowId: string, operation: ContextOperationRequest, signal?: AbortSignal, speculativeState?: import('./speculative-execution-state.ts').SpeculativeExecutionState): Promise<unknown>;

package/dist/core/engine/callback-creators-bundles.js

@@ -15,6 +15,7 @@ import {
   createReviewOperationCallbacks,
   handleScheduleTimerForEngine
 } from "./callback-creators-schedule.js";
+import { commitFencedEngineWrite } from "./fenced-write.js";
 import { guardTerminalWorkflow, guardTerminalWorkflowAfterCoordinatedRequest } from "./guards.js";
 import { getInternals } from "./internals.js";
 import {
@@ -27,6 +28,7 @@ import {
 } from "./lifecycle.js";
 import {
   completeOperation,
+  failOperation,
   runOperationWithResult,
   runOperationWithoutResult
 } from "./operations-router.js";
@@ -34,6 +36,7 @@ import { schedulePendingInlineUpdateDrain } from "./pending-updates.js";
 import { processReviewOperation } from "./reviews.js";
 import { loadWorkflowState, runSerializedWorkflowStateWrite } from "./storage-io.js";
 import {
+  feedOperationResult,
   getComposedActivityInterceptor,
   getComposedWorkflowInterceptor
 } from "./strategy-helpers.js";
@@ -41,7 +44,9 @@ import { executeSubOperation } from "./sub-operation.js";
 import {
   ensureTerminalCleanupTracked,
   failWorkflow,
-  runDeferredTerminalCleanup
+  finalizePendingTimelineEntry,
+  runDeferredTerminalCleanup,
+  runWorkflowFinalizer
 } from "./termination.js";
 import {
   createCoordinatedUpdateResponder,
@@ -60,6 +65,8 @@ export function createChildWorkflowOperationCallbacks(engine) {
 export function createActivityOperationCallbacks(engine) {
   return {
     runOperationWithResult: (workflowId, operation, execute) => runOperationWithResultForEngine(engine, workflowId, operation, execute),
+    finalizePendingTimelineEntry: (workflowId, status, value) => finalizePendingTimelineEntry(getInternals(engine), workflowId, status, value),
+    feedOperationResult: (workflowId, result, error) => feedOperationResult(getInternals(engine), workflowId, result, error),
     getComposedActivityInterceptor: () => getComposedActivityInterceptor(getInternals(engine)),
     getComposedWorkflowInterceptor: () => getComposedWorkflowInterceptor(getInternals(engine))
   };
@@ -72,6 +79,22 @@ export function createCoordinationOperationCallbacks(engine) {
     getActivityOperationCallbacks: () => createActivityOperationCallbacks(engine)
   };
 }
+export function createConditionOperationCallbacks(engine) {
+  return {
+    completeOperation: (workflowId, value) => completeOperationForEngine(engine, workflowId, value),
+    failOperation: (workflowId, operation, error) => failOperationForEngine(engine, workflowId, operation, error),
+    isWorkflowRunning: async (workflowId) => {
+      return (await loadWorkflowState(getInternals(engine), workflowId))?.status === "running";
+    },
+    scheduleConditionDeadline: (workflowId, step, fireAt) => getInternals(engine).scheduler.schedule({
+      id: `cond:${workflowId}:${step}`,
+      workflowId,
+      fireAt,
+      kind: "wait-condition"
+    }),
+    cancelConditionDeadline: (workflowId, step) => getInternals(engine).scheduler.cancel(`cond:${workflowId}:${step}`, workflowId)
+  };
+}
 export function createDataOperationCallbacks(engine) {
   return {
     runOperationWithResult: (workflowId, operation, execute) => runOperationWithResultForEngine(engine, workflowId, operation, execute)
@@ -92,6 +115,7 @@ export function createStreamOperationCallbacks(engine) {
 export function createTimeOperationCallbacks(engine) {
   return {
     completeOperation: (workflowId, value) => completeOperationForEngine(engine, workflowId, value),
+    dispatchEvent: (event) => engine.dispatchEvent(event),
     loadWorkflowState: (workflowId) => loadWorkflowState(getInternals(engine), workflowId),
     failWorkflow: (workflowId, error) => failWorkflow(getInternals(engine), workflowId, error, createTerminationCallbacks(engine)),
     runSerializedWorkflowStateWrite: (workflowId, writeOperation) => runSerializedWorkflowStateWrite(getInternals(engine), workflowId, writeOperation),
@@ -101,6 +125,7 @@ export function createTimeOperationCallbacks(engine) {
     loadWorkflowStartHeaders: (workflowId) => loadWorkflowStartHeaders(getInternals(engine), workflowId, createLifecycleCallbacks(engine)),
     parseStartOptionDuration: (value, fieldName) => parseStartOptionDuration(getInternals(engine), value, fieldName, createLifecycleCallbacks(engine)),
     runDeferredTerminalCleanup: (workflowId, timerId) => runDeferredTerminalCleanup(getInternals(engine), workflowId, timerId, createTerminationCallbacks(engine)),
+    runWorkflowFinalizer: (workflowId, timerId) => runWorkflowFinalizer(getInternals(engine), workflowId, timerId, createTerminationCallbacks(engine)),
     handleScheduleTimer: (entry) => handleScheduleTimerForEngine(engine, entry),
     timeout: (workflowId) => engine.timeout(workflowId),
     handleCleanupError: (source, error, workflowId) => createTerminationCallbacks(engine).handleCleanupError(source, error, workflowId)
@@ -126,7 +151,7 @@ export function createUpdateCallbacks(engine) {
 async function persistCoordinatedUpdateResponse(engine, workflowId, updateName, updateId, idempotencyKey, value) {
   const internals = getInternals(engine), responseOperations = internals.updateCoordinator.buildResponseOperations(updateId, workflowId, value, void 0, idempotencyKey);
   try {
-    await internals.storage.batch(responseOperations);
+    await commitFencedEngineWrite(internals, responseOperations, [], () => Error(`Coordinated update response for workflow "${workflowId}" lost its CAS race.`));
     engine.dispatchEvent(new UpdateCompletedEvent(updateId, workflowId, updateName, value));
     broadcastFromInternals(internals, { type: "update:completed", workflowId, updateId }, createBroadcastCallbacks(engine));
   } catch (error) {
@@ -136,6 +161,9 @@ async function persistCoordinatedUpdateResponse(engine, workflowId, updateName,
 export function completeOperationForEngine(engine, workflowId, value) {
   return completeOperation(getInternals(engine), workflowId, value, callRouterCallbacks(engine));
 }
+export function failOperationForEngine(engine, workflowId, operation, error) {
+  return failOperation(getInternals(engine), workflowId, operation, error, callRouterCallbacks(engine));
+}
 export async function runOperationWithResultForEngine(engine, workflowId, operation, execute) {
   return runOperationWithResult(getInternals(engine), workflowId, operation, execute, callRouterCallbacks(engine));
 }