@lostgradient/weft 0.24.0 → 0.24.2

package/dist/core/catalog/removal.d.ts

@@ -1,9 +1,10 @@
 /**
  * The mechanical CAS-delete primitive `WorkflowCatalog.remove()` delegates
- * to. Split into its own module to protect `workflow-catalog.ts`'s size
- * against the repository's 500-line implementation-file ceiling — mirrors
- * how `storage-io.ts`/`codec.ts` already sit alongside `workflow-catalog.ts`
- * rather than inline in it.
+ * to, plus the tombstone primitives that make its own resolution
+ * (restore-or-finalize) atomic too. Split into its own module to protect
+ * `workflow-catalog.ts`'s size against the repository's 500-line
+ * implementation-file ceiling — mirrors how `storage-io.ts`/`codec.ts`
+ * already sit alongside `workflow-catalog.ts` rather than inline in it.
  *
  * Fences the delete on BOTH the exact entry bytes read AND the exact active
  * pointer bytes read, in one `conditionalBatch` — not entry bytes alone.
@@ -20,12 +21,37 @@
  * counts computed by the caller and should not blindly re-attempt against
  * possibly-stale counts.
  *
+ * WFT-17/18 (Codex review, PR #958): a bare delete of the entry key, with
+ * `removeWorkflowRevision`'s own reference re-check and possible restore as
+ * a SEPARATE, later commit, left a crash window between the two commits
+ * where the entry was durably gone but a run still referenced it, with NO
+ * durable record of what was deleted for ANY process — including the one
+ * that crashed — to recover from. {@link removeCatalogEntry} now writes a
+ * `catalog-tombstone:<name>:<revision>` record (the exact deleted entry
+ * bytes) in the SAME `conditionalBatch` as the delete, so "the entry is
+ * gone" and "a durable record of what it was" land atomically together.
+ * {@link finalizeCatalogTombstone} and {@link restoreCatalogEntryFromTombstone}
+ * are the two possible resolutions, each its own single-CAS atomic
+ * operation conditioned on the tombstone's exact bytes — so a concurrent
+ * second attempt at resolving the SAME tombstone (this process retrying, or
+ * `catalog-readiness.ts`'s boot-time sweep racing a live caller) loses its
+ * CAS harmlessly rather than double-processing. Orphan resolution (a
+ * tombstone left behind by a process that crashed between the delete and
+ * its own resolution) is `resolveOrphanedCatalogTombstones` in
+ * `core/engine/catalog-tombstone-recovery.ts` — deliberately OUTSIDE
+ * `core/catalog/**`, not alongside these primitives, because it needs
+ * `core/engine/nonterminal-revision-count.ts`'s durable non-terminal-run
+ * scan and `core/catalog/**` never imports from `core/engine/**` (a
+ * directional boundary `check-import-cycles.ts` enforces); see that
+ * module's doc for why it lives there.
+ *
  * @module core/catalog/removal
  */
 import { type Storage } from '../../storage/interface.ts';
 /** Outcome of {@link removeCatalogEntry}. */
 export type WorkflowCatalogRemovalOutcome = Readonly<{
     outcome: 'removed';
+    tombstoneBytes: Uint8Array;
 }> | Readonly<{
     outcome: 'not-found';
 }> | Readonly<{
@@ -35,7 +61,9 @@ export type WorkflowCatalogRemovalOutcome = Readonly<{
     outcome: 'conflict';
 }>;
 /**
- * Durably delete the installed-revision record for `(name, revision)`.
+ * Durably delete the installed-revision record for `(name, revision)`,
+ * atomically replacing it with a tombstone carrying the deleted bytes (see
+ * this module's doc for why).
  *
  * Reads the current `catalog-entry:<name>:<revision>` bytes; `null` is
  * `'not-found'` (a no-op — the caller may already believe the revision is
@@ -43,10 +71,35 @@ export type WorkflowCatalogRemovalOutcome = Readonly<{
  * `'active'` when they decode to exactly this revision — a structural
  * invariant independent of reference counts, since every future or
  * resuming run resolves the active pointer, not a specific installed
- * entry. Otherwise deletes the entry key via a `conditionalBatch` CAS'd on
- * BOTH the exact entry bytes AND the exact active-pointer bytes read above
- * (present as a no-op `put`-free precondition when active-pointer bytes
- * are `null` — never activated). A CAS loss (either key changed
- * concurrently) surfaces as `'conflict'`.
+ * entry. Otherwise deletes the entry key and puts the tombstone key (CAS'd
+ * on the tombstone being absent — a stale leftover from a previous,
+ * unresolved removal of this exact `(name, revision)` is a durable-store
+ * inconsistency this fails closed on rather than silently overwriting) via
+ * ONE `conditionalBatch`, also CAS'd on the exact entry bytes AND the exact
+ * active-pointer bytes read above (present as a no-op `put`-free
+ * precondition when active-pointer bytes are `null` — never activated). A
+ * CAS loss (any of the three keys changed concurrently) surfaces as
+ * `'conflict'`.
  */
 export declare function removeCatalogEntry(storage: Storage, name: string, revision: string): Promise<WorkflowCatalogRemovalOutcome>;
+/**
+ * Resolve an outstanding tombstone by completing the removal: CAS-delete
+ * the `catalog-tombstone:<name>:<revision>` key, conditioned on its exact
+ * bytes still being `tombstoneBytes`. Returns `false` (a harmless no-op for
+ * the caller, not an error) when the tombstone was already resolved by
+ * someone else — this same process re-driving after a transient failure,
+ * or a concurrent boot-time sweep on another process.
+ */
+export declare function finalizeCatalogTombstone(storage: Storage, name: string, revision: string, tombstoneBytes: Uint8Array): Promise<boolean>;
+/**
+ * Resolve an outstanding tombstone by restoring the entry: ONE
+ * `conditionalBatch` that both re-`put`s the `catalog-entry:<name>:<revision>`
+ * key (byte-identical to what {@link removeCatalogEntry} deleted — the
+ * tombstone bytes ARE the entry bytes) and deletes the tombstone key,
+ * conditioned on the tombstone's exact bytes still being `tombstoneBytes`.
+ * Atomic by construction — there is no intermediate durable state where the
+ * entry is back but the tombstone still claims a removal is in flight, or
+ * vice versa. Returns `false` (harmless no-op) when the tombstone was
+ * already resolved by someone else.
+ */
+export declare function restoreCatalogEntryFromTombstone(storage: Storage, name: string, revision: string, tombstoneBytes: Uint8Array): Promise<boolean>;

package/dist/core/catalog/removal.js

@@ -7,8 +7,24 @@ export async function removeCatalogEntry(storage, name, revision) {
   const activeKey = KEYS.catalogActive(name), activeBytes = await storage.get(activeKey), activePointer = activeBytes === null ? null : decodeActivePointer(activeBytes);
   if (activePointer !== null && activePointer.revision === revision)
     return { outcome: "active", activeRevision: activePointer.revision };
+  const tombstoneKey = KEYS.catalogTombstone(name, revision);
   return await storageConditionalBatch(storage, [
     { key: entryKey, expectedValue: entryBytes },
-    { key: activeKey, expectedValue: activeBytes }
-  ], [{ type: "delete", key: entryKey }]) ? { outcome: "removed" } : { outcome: "conflict" };
+    { key: activeKey, expectedValue: activeBytes },
+    { key: tombstoneKey, expectedValue: null }
+  ], [
+    { type: "delete", key: entryKey },
+    { type: "put", key: tombstoneKey, value: entryBytes }
+  ]) ? { outcome: "removed", tombstoneBytes: entryBytes } : { outcome: "conflict" };
+}
+export async function finalizeCatalogTombstone(storage, name, revision, tombstoneBytes) {
+  const tombstoneKey = KEYS.catalogTombstone(name, revision);
+  return storageConditionalBatch(storage, [{ key: tombstoneKey, expectedValue: tombstoneBytes }], [{ type: "delete", key: tombstoneKey }]);
+}
+export async function restoreCatalogEntryFromTombstone(storage, name, revision, tombstoneBytes) {
+  const entryKey = KEYS.catalogEntry(name, revision), tombstoneKey = KEYS.catalogTombstone(name, revision);
+  return storageConditionalBatch(storage, [{ key: tombstoneKey, expectedValue: tombstoneBytes }], [
+    { type: "put", key: entryKey, value: tombstoneBytes },
+    { type: "delete", key: tombstoneKey }
+  ]);
 }

package/dist/core/catalog/storage-io.d.ts

@@ -28,6 +28,29 @@ export type RestoredWorkflowCatalogState = {
  * returns empty maps.
  */
 export declare function restoreWorkflowCatalog(storage: Storage): Promise<RestoredWorkflowCatalogState>;
+/**
+ * Decode and validate one durable catalog-entry record: parse as JSON,
+ * validate as a manifest via {@link parseWorkflowRevisionManifest}, and check
+ * that the decoded `(name, revision)` agrees with the caller-supplied
+ * expectation (normally read from the storage key itself). Fails closed on
+ * any disagreement — shared by {@link restoreWorkflowCatalog}'s per-entry
+ * restore, {@link readCatalogEntry}'s single-key read, and the by-name scan
+ * `WorkflowCatalog.listInstalledRevisions` uses, so the three consumers of
+ * one durable record shape can never validate it three different ways.
+ */
+/**
+ * Decode and validate a raw catalog-entry-record byte string against an
+ * expected `(name, revision)` — exported for {@link import('./removal.ts').restoreCatalogEntryFromTombstone}'s
+ * caller (`WorkflowCatalog.restoreFromTombstone`) and the boot-time orphan
+ * sweep (`orphaned-tombstones.ts`) to re-populate the in-memory cache from
+ * a tombstone's bytes, which are byte-identical to the entry bytes this
+ * function already validates for {@link restoreWorkflowCatalog} and
+ * {@link readCatalogEntry}.
+ */
+export declare function decodeCatalogEntryRecord(key: string, bytes: Uint8Array, expectedName: string, expectedRevision: string): Promise<{
+    manifest: WorkflowRevisionManifest;
+    installedAt: number;
+}>;
 /**
  * Read one durable installed-revision record for `(name, revision)`, or
  * `null` when absent. Fails closed on corruption, exactly matching

package/dist/core/catalog/storage-io.js

@@ -42,7 +42,7 @@ export async function restoreWorkflowCatalog(storage) {
     restoreOneActivePointer(active, key, bytes);
   return { entries, active };
 }
-async function decodeCatalogEntryRecord(key, bytes, expectedName, expectedRevision) {
+export async function decodeCatalogEntryRecord(key, bytes, expectedName, expectedRevision) {
   const parsedRecord = parseCatalogEntryRecord(bytes);
   if (parsedRecord === null)
     failClosed("workflow catalog entry", key, "could not be parsed as JSON");

package/dist/core/engine/activity-resolution.js

@@ -1,22 +1,22 @@
 import { ActivityResolutionError } from "./errors.js";
-function resolveActivityViaRegistries(internals, workflowId, activityName) {
-  const workflowType = internals.workflowTypeByWorkflowId.get(workflowId);
-  if (workflowType !== void 0) {
-    const perWorkflow = internals.activityRegistriesByWorkflow.get(workflowType);
-    if (perWorkflow !== void 0) {
-      const perWorkflowFn = perWorkflow.resolve(activityName);
-      if (perWorkflowFn)
-        return { fn: perWorkflowFn, workflowType };
-    }
-    const globalFn = internals.activityRegistry.resolve(activityName);
-    if (globalFn)
-      return { fn: globalFn, workflowType };
-    return;
+function resolveViaKnownIdentity(internals, identity, activityName) {
+  const eagerFn = internals.activityRegistriesByWorkflow.get(identity.type)?.resolve(activityName);
+  if (eagerFn)
+    return { fn: eagerFn, workflowType: identity.type };
+  if (identity.revision !== void 0) {
+    const revisionFn = internals.sources.resolved.get(identity.type)?.get(identity.revision)?.activityRegistry.resolve(activityName);
+    if (revisionFn)
+      return { fn: revisionFn, workflowType: identity.type };
   }
   const globalFn = internals.activityRegistry.resolve(activityName);
-  if (globalFn)
-    return { fn: globalFn, workflowType: "<unknown>" };
-  return;
+  return globalFn ? { fn: globalFn, workflowType: identity.type } : void 0;
+}
+function resolveActivityViaRegistries(internals, workflowId, activityName) {
+  const identity = internals.workflowTypeByWorkflowId.get(workflowId);
+  if (identity !== void 0)
+    return resolveViaKnownIdentity(internals, identity, activityName);
+  const globalFn = internals.activityRegistry.resolve(activityName);
+  return globalFn ? { fn: globalFn, workflowType: "<unknown>" } : void 0;
 }
 export function getActivityFunctionWithMetadata(internals, workflowId, operation) {
   const resolved = resolveActivityViaRegistries(internals, workflowId, operation.activityName);
@@ -32,6 +32,6 @@ export function resolveActivityFunction(internals, workflowId, operation) {
     return resolved.fn;
   if (operation.fn)
     return operation.fn;
-  const workflowType = internals.workflowTypeByWorkflowId.get(workflowId) ?? "<unknown>";
+  const workflowType = internals.workflowTypeByWorkflowId.get(workflowId)?.type ?? "<unknown>";
   throw new ActivityResolutionError(workflowType, operation.activityName);
 }

package/dist/core/engine/bulk-operations-purge.d.ts

@@ -21,7 +21,7 @@ export declare function purgeWorkflow(internals: EngineInternals, state: Workflo
  * `onTerminalConflict: 'start-new'` restart path prepends them to the create batch
  * so purge-and-recreate land as one atomic unit (no window where the prior run is
  * gone but the new one has not committed). Keep this the single source of truth
- * for "what a purge deletes" — do not fork the delete-set.
+ * for "what a purge deletes" — do not fork the delete-set. (`wf-gen:<id>` is deliberately excluded; WFT-153, see {@link buildBaseWorkflowDeleteKeys}.)
  */
 export declare function collectWorkflowPurgeDeleteOperations(internals: EngineInternals, state: WorkflowState): Promise<BatchOperation[]>;
 /**

package/dist/core/engine/bulk-operations-purge.js

@@ -12,15 +12,17 @@ import { asyncActivityWorkflowPrefix } from "./async-activity-records.js";
 import { forgetCommittedCheckpointBytes } from "./checkpoint-commit-snapshots.js";
 import { commitFencedEngineWrite } from "./fenced-write.js";
 import { streamWorkflowStates } from "./listing.js";
+import {
+  forEachResolvedDynamicRetentionPolicy,
+  hasUnresolvedDynamicSourceCandidate
+} from "./registration.js";
 import { decodeScheduleRunMetadata } from "./schedule-run-metadata.js";
 import { createTerminalCleanupTimerId } from "./state-utilities.js";
 import { buildExternalTerminalRotationFragment } from "./storage-io.js";
-import {
-  decodeWorkflowState,
-  isTerminalWorkflowStatus,
-  resolveRetentionForStatus
-} from "./validation.js";
+import { decodeWorkflowState, isTerminalWorkflowStatus } from "./validation.js";
+import { foldWorkflowGenerationBumpForPurge } from "./workflow-generation-fence.js";
 import { buildWorkflowVisibilityIndexTransition } from "./workflow-indexes.js";
+import { getWorkflowRetentionDeadline } from "./workflow-retention-deadline.js";
 export const TERMINAL_CLEANUP_DELAY_MS = 60000;
 export async function purgeInternal(internals, filter, parameters, cleanupWaiters) {
   const { effectiveLimit, manualOffset } = resolvePurgeWindow(internals, filter, parameters.limit);
@@ -59,16 +61,17 @@ function getMinimumRetentionMs(internals) {
   considerRetentionPolicy(internals.options.retention);
   for (const registration of internals.registrations.values())
     considerRetentionPolicy(registration.retention);
+  forEachResolvedDynamicRetentionPolicy(internals, considerRetentionPolicy);
   return minimumRetentionMs;
 }
 async function* streamExpiredRetentionWorkflowStates(internals, now) {
-  const minimumRetentionMs = getMinimumRetentionMs(internals);
-  if (minimumRetentionMs === null)
+  const untrustworthyBound = hasUnresolvedDynamicSourceCandidate(internals), minimumRetentionMs = untrustworthyBound ? null : getMinimumRetentionMs(internals);
+  if (!untrustworthyBound && minimumRetentionMs === null)
     return;
-  const terminalWorkflowPrefix = KEYS.terminalWorkflowPrefix(), newestPossibleExpiredUpdatedAt = now - minimumRetentionMs, upperBound = `${terminalWorkflowPrefix}${String(newestPossibleExpiredUpdatedAt).padStart(16, "0")}:\xFF`;
-  for await (const [key] of internals.storage.scan(terminalWorkflowPrefix, {
-    lte: upperBound
-  })) {
+  const terminalWorkflowPrefix = KEYS.terminalWorkflowPrefix(), scanOptions = minimumRetentionMs === null ? {} : {
+    lte: `${terminalWorkflowPrefix}${String(now - minimumRetentionMs).padStart(16, "0")}:\xFF`
+  };
+  for await (const [key] of internals.storage.scan(terminalWorkflowPrefix, scanOptions)) {
     const encodedWorkflowId = key.slice(key.lastIndexOf(":") + 1), workflowId = tryDecodeStorageKeyComponent(encodedWorkflowId);
     if (workflowId === null)
       continue;
@@ -121,20 +124,12 @@ async function shouldPurgeWorkflowState(internals, state, expiredOnly, now) {
     return !1;
   if (!expiredOnly)
     return !0;
-  const deadline = getWorkflowRetentionDeadline(internals, state);
+  const deadline = await getWorkflowRetentionDeadline(internals, state);
   return deadline !== null && deadline <= now;
 }
-function getWorkflowRetentionDeadline(internals, state) {
-  if (!isTerminalWorkflowStatus(state.status))
-    return null;
-  const policy = internals.registrations.get(state.type)?.retention ?? internals.options.retention, retentionMs = resolveRetentionForStatus(policy, state.status);
-  if (retentionMs === void 0)
-    return null;
-  return state.updatedAt + retentionMs;
-}
 export async function purgeWorkflow(internals, state, cleanupWaiters) {
-  const deleteOperations = await collectWorkflowPurgeDeleteOperations(internals, state), rotation = await buildExternalTerminalRotationFragment(internals, state.id), operations = [...deleteOperations, ...rotation.operations];
-  await commitFencedEngineWrite(internals, null, operations, rotation.conditions, () => {
+  const deleteOperations = await collectWorkflowPurgeDeleteOperations(internals, state), rotation = await buildExternalTerminalRotationFragment(internals, state.id), fenced = await foldWorkflowGenerationBumpForPurge(internals, state.id, rotation), operations = [...deleteOperations, ...fenced.operations];
+  await commitFencedEngineWrite(internals, null, operations, fenced.conditions, () => {
     return Error(`Purge commit for workflow "${state.id}" lost its precondition.`);
   });
   clearPurgedWorkflowInMemoryState(internals, state.id, cleanupWaiters);

package/dist/core/engine/bulk-operations-retry.d.ts

@@ -0,0 +1,24 @@
+import type { BulkOperationDryRunResult, BulkOperationOptions, BulkRetryFailedResult, ListFilter } from '../types.ts';
+import type { EngineInternals } from './internals.ts';
+import { type LifecycleCallbacks } from './lifecycle.ts';
+/**
+ * `bulkOperations.ts`'s `retryFailedAll()` implementation, split into its
+ * own module (with the rest of the retry-failed cluster below) to keep
+ * `bulk-operations.ts` under the 800-line implementation-file ceiling —
+ * the same reason `bulk-operations-purge.ts` was split out earlier.
+ */
+export declare function runBulkFailedWorkflowRetry(internals: EngineInternals, filter: ListFilter, options?: BulkOperationOptions, 
+/**
+ * Threaded down from `Engine#retryFailedAll()`'s own `#createLifecycleCallbacks()`
+ * (WFT-95 issue 2 follow-up). Needed only by `retryFailedWorkflow()`'s
+ * checkpoint-absent fallback, which must call the internal `startWorkflow()`
+ * directly (see its doc comment) rather than the public `engine.start()`.
+ * Sourced this way — as a parameter, rather than this module importing
+ * `createLifecycleCallbacks` from `callback-creators-core.ts` itself — to avoid
+ * introducing a `bulk-operations-retry.ts` -> `callback-creators-core.ts` ->
+ * `termination.ts` -> `termination/complete.ts` -> `bulk-operations.ts` ->
+ * `bulk-operations-retry.ts` import cycle. Optional so direct-internals test
+ * harnesses that never reach the checkpoint-absent fallback (e.g.
+ * `bulk-operations-retry-direct.test.ts`) do not need to construct one.
+ */
+callbacks?: LifecycleCallbacks): Promise<BulkRetryFailedResult | BulkOperationDryRunResult>;

package/dist/core/engine/bulk-operations-retry.js

@@ -0,0 +1,190 @@
+import {
+  KEYS,
+  requireStorageCapability
+} from "../../storage/interface.js";
+import { assertScopedBulkWorkflowFilter } from "../bulk-workflow-filter.js";
+import { deserializeCheckpoint } from "../checkpoint.js";
+import { decode, encode } from "../codec.js";
+import { buildTimerBatchOperations } from "../scheduler.js";
+import { buildIndexOperations } from "../search-attributes.js";
+import { buildTerminalWorkflowIndexOperations } from "./attributes-tags.js";
+import {
+  buildActionableBulkWorkflowFilter,
+  normalizeBulkOperationOptions,
+  prepareBulkOperation,
+  resolveBulkOperationConcurrency,
+  runBulkWorkflowPool,
+  shouldPersistBulkAudit,
+  toBulkOperationError,
+  validateBulkConfirmation,
+  withBulkAuditEvent
+} from "./bulk-operations-shared.js";
+import { ensureWorkflowCatalogReady, isWorkflowCatalogReady } from "./catalog-readiness.js";
+import { resolveExecutableRegistrationForRetry } from "./dynamic-source-execution.js";
+import {
+  assertLeaseHeldForEngineWork,
+  commitFencedEngineWriteAllowingPreconditionFailure
+} from "./fenced-write.js";
+import { startWorkflow } from "./lifecycle.js";
+import { buildCatalogEntryRevisionCondition } from "./lifecycle/start-commit.js";
+import { createTerminalCleanupTimerId } from "./state-utilities.js";
+import { loadWorkflowState, runSerializedWorkflowStateWrite } from "./storage-io.js";
+import { decodeWorkflowState } from "./validation.js";
+import {
+  commitWithWorkflowClaimFold,
+  prepareWorkflowClaimFold,
+  throwWorkflowClaimUnavailable
+} from "./workflow-claim-fold.js";
+import { buildWorkflowConcurrencyStartOperations } from "./workflow-concurrency.js";
+import { buildWorkflowVisibilityIndexTransition } from "./workflow-indexes.js";
+const FAILED_WORKFLOW_STATUSES = ["failed"], CHECKPOINT_RETRY_CONCURRENCY_ADMISSION_MAX_ATTEMPTS = 5;
+export async function runBulkFailedWorkflowRetry(internals, filter, options = {}, callbacks) {
+  options = normalizeBulkOperationOptions(options);
+  assertScopedBulkWorkflowFilter(filter);
+  const actionableFilter = buildActionableBulkWorkflowFilter(internals, filter, FAILED_WORKFLOW_STATUSES), preparation = await prepareBulkOperation(internals, "retry-failed", actionableFilter, filter, {}, options);
+  if (options.dryRun === !0)
+    return preparation.preview;
+  validateBulkConfirmation(options, preparation);
+  const bulkConcurrency = resolveBulkOperationConcurrency(options);
+  let retried = 0;
+  const errors = [], retryResults = await runBulkWorkflowPool(preparation.workflowIds, bulkConcurrency, async (workflowId) => {
+    await retryFailedWorkflow(internals, workflowId, callbacks);
+    return { status: "retried" };
+  });
+  for (const retryResult of retryResults) {
+    if (retryResult.status === "rejected") {
+      errors.push(toBulkOperationError(internals, retryResult.item, retryResult.reason));
+      continue;
+    }
+    retried += 1;
+  }
+  const result = { retried, failed: errors.length, errors };
+  if (!shouldPersistBulkAudit(options))
+    return result;
+  return withBulkAuditEvent(internals, preparation, options, result, retried);
+}
+async function retryFailedWorkflow(internals, workflowId, callbacks) {
+  const state = await loadWorkflowState(internals, workflowId);
+  if (state === null)
+    throw Error("Workflow no longer exists");
+  if (state.status !== "failed")
+    throw Error(`Workflow is ${state.status}, not failed`);
+  if (await internals.storage.get(KEYS.checkpoint(workflowId)) !== null) {
+    await reactivateFailedWorkflowFromCheckpoint(internals, state);
+    await internals.engine.resume(workflowId);
+    return;
+  }
+  const restartOptions = {
+    id: workflowId,
+    onTerminalConflict: "start-new",
+    ...state.tags !== void 0 ? { tags: state.tags } : {}
+  };
+  if (callbacks === void 0) {
+    await internals.engine.start(state.type, state.input, restartOptions);
+    return;
+  }
+  assertLeaseHeldForEngineWork(internals);
+  if (!isWorkflowCatalogReady(internals.engine))
+    await ensureWorkflowCatalogReady(internals.engine);
+  await startWorkflow(internals, state.type, state.input, restartOptions, void 0, callbacks, void 0, void 0, "bulk-retry-only");
+}
+async function reactivateFailedWorkflowFromCheckpoint(internals, state) {
+  const reactivated = await runSerializedWorkflowStateWrite(internals, state.id, async () => reactivateFailedWorkflowFromCheckpointSerialized(internals, state.id));
+  if (reactivated.terminalCleanupTimerId !== void 0)
+    await internals.scheduler.cancel(reactivated.terminalCleanupTimerId, state.id);
+}
+async function reactivateFailedWorkflowFromCheckpointSerialized(internals, workflowId) {
+  let lastConcurrencyStateKey;
+  for (let attempt = 0;attempt < CHECKPOINT_RETRY_CONCURRENCY_ADMISSION_MAX_ATTEMPTS; attempt += 1) {
+    const { currentState, checkpoint } = await loadFailedWorkflowForReactivation(internals, workflowId), commit = await buildReactivationCommit(internals, workflowId, currentState, checkpoint);
+    lastConcurrencyStateKey = commit.concurrencyStateKey ?? lastConcurrencyStateKey;
+    if (await commitFailedWorkflowReactivation(internals, workflowId, commit.operations, commit.conditions))
+      return {
+        terminalCleanupTimerId: currentState.terminalCleanupToken === void 0 ? void 0 : createTerminalCleanupTimerId(!1, currentState.terminalCleanupToken)
+      };
+  }
+  throw Error(`Workflow concurrency admission for "${lastConcurrencyStateKey ?? workflowId}" changed too many times while retrying failed workflow "${workflowId}"`);
+}
+async function loadFailedWorkflowForReactivation(internals, workflowId) {
+  const currentStateBytes = await internals.storage.get(KEYS.workflow(workflowId));
+  if (currentStateBytes === null)
+    throw Error("Workflow no longer exists");
+  const currentState = decodeWorkflowState(currentStateBytes);
+  if (currentState.status !== "failed")
+    throw Error(`Workflow is ${currentState.status}, not failed`);
+  const currentCheckpointBytes = await internals.storage.get(KEYS.checkpoint(workflowId));
+  if (currentCheckpointBytes === null)
+    throw Error("Checkpoint no longer exists");
+  return { currentState, checkpoint: deserializeCheckpoint(currentCheckpointBytes) };
+}
+async function buildReactivationCommit(internals, workflowId, currentState, checkpoint) {
+  const { entry: registration } = await resolveExecutableRegistrationForRetry(internals, currentState.type, currentState.revision, workflowId), catalogEntryCondition = currentState.revision === void 0 ? void 0 : await buildCatalogEntryRevisionCondition(internals, currentState.type, currentState.revision), concurrencyStartOperations = registration.concurrency === void 0 ? void 0 : await buildWorkflowConcurrencyStartOperations(internals, currentState.type, workflowId, currentState.input, registration.concurrency), reactivatedState = buildReactivatedWorkflowState(internals, currentState), currentAttributes = await loadSearchAttributes(internals, workflowId);
+  return {
+    operations: buildReactivationBatchOperations(workflowId, currentState, reactivatedState, checkpoint, currentAttributes, concurrencyStartOperations?.operations),
+    conditions: [
+      ...concurrencyStartOperations?.conditions ?? [],
+      ...catalogEntryCondition === void 0 ? [] : [catalogEntryCondition]
+    ],
+    concurrencyStateKey: concurrencyStartOperations?.stateKey
+  };
+}
+function buildReactivationBatchOperations(workflowId, currentState, reactivatedState, checkpoint, currentAttributes, concurrencyOperations) {
+  const deadlineOperations = reactivatedState.executionDeadline === void 0 ? [] : buildTimerBatchOperations({
+    id: `deadline:${workflowId}`,
+    workflowId,
+    fireAt: reactivatedState.executionDeadline,
+    kind: "execution-deadline"
+  });
+  return [
+    ...buildTerminalWorkflowIndexOperations(currentState, reactivatedState),
+    { type: "put", key: KEYS.workflow(workflowId), value: encode(reactivatedState) },
+    ...buildWorkflowVisibilityIndexTransition(workflowId, currentState, reactivatedState).batchOps,
+    ...buildRetrySearchAttributeOperations(workflowId, currentAttributes, checkpoint.searchAttributes),
+    ...deadlineOperations,
+    ...concurrencyOperations ?? []
+  ];
+}
+async function commitFailedWorkflowReactivation(internals, workflowId, operations, conditions) {
+  const claimFold = await prepareWorkflowClaimFold(internals, workflowId);
+  if (claimFold === void 0) {
+    if (conditions.length > 0)
+      requireStorageCapability(internals.storage, "conditionalBatch", "retry failed workflow");
+    return commitFencedEngineWriteAllowingPreconditionFailure(internals, workflowId, operations, conditions);
+  }
+  const result = await commitWithWorkflowClaimFold(internals, claimFold, operations, conditions, "retry failed workflow claim acquisition");
+  if (result.status === "committed")
+    return !0;
+  return result.claimConflict ? throwWorkflowClaimUnavailable(internals, workflowId) : !1;
+}
+function buildReactivatedWorkflowState(internals, state) {
+  const reactivatedState = {
+    ...state,
+    status: "running",
+    updatedAt: internals.options.getNow()
+  };
+  delete reactivatedState.error;
+  delete reactivatedState.errorStack;
+  delete reactivatedState.failureCategory;
+  delete reactivatedState.result;
+  delete reactivatedState.terminationReason;
+  delete reactivatedState.terminalCleanupToken;
+  return reactivatedState;
+}
+async function loadSearchAttributes(internals, workflowId) {
+  const attributeBytes = await internals.storage.get(KEYS.attribute(workflowId));
+  if (attributeBytes === null)
+    return {};
+  return decode(attributeBytes);
+}
+function buildRetrySearchAttributeOperations(workflowId, currentAttributes, checkpointAttributes) {
+  const operations = buildIndexOperations(workflowId, currentAttributes, checkpointAttributes);
+  if (Object.keys(checkpointAttributes).length === 0)
+    operations.push({ type: "delete", key: KEYS.attribute(workflowId) });
+  else
+    operations.push({
+      type: "put",
+      key: KEYS.attribute(workflowId),
+      value: encode(checkpointAttributes)
+    });
+  return operations;
+}

package/dist/core/engine/bulk-operations-shared.d.ts

@@ -2,6 +2,24 @@ import type { BulkOperationAction, BulkOperationAuditEvent, BulkOperationCommitO
 import type { EngineInternals } from './internals.ts';
 export declare const BULK_OPERATION_SAMPLE_LIMIT = 20;
 export declare const DEFAULT_BULK_OPERATION_CONCURRENCY = 1;
+export type BulkWorkflowPoolResult<TItem, TResult> = {
+    item: TItem;
+    status: 'fulfilled';
+    value: TResult;
+} | {
+    item: TItem;
+    status: 'rejected';
+    reason: unknown;
+};
+/**
+ * Shared bounded-concurrency runner every bulk operation (cancel, retry,
+ * signal, delete) uses to drive its per-workflow work: runs `items` in
+ * fixed-size batches of `concurrencyLimit`, capturing each item's outcome
+ * rather than letting one rejection abort the batch (mirrors
+ * `Promise.allSettled`, but keyed back to the originating `item` so a
+ * caller can map a failure to the workflow it belongs to).
+ */
+export declare function runBulkWorkflowPool<TItem, TResult>(items: readonly TItem[], concurrencyLimit: number, operation: (item: TItem) => Promise<TResult>): Promise<BulkWorkflowPoolResult<TItem, TResult>[]>;
 export type BulkWorkflowSnapshot = {
     id: string;
     type: string;

package/dist/core/engine/bulk-operations-shared.js

@@ -15,6 +15,20 @@ import { streamWorkflowStateBatches } from "./listing.js";
 import { isTerminalWorkflowStatus } from "./validation.js";
 export const BULK_OPERATION_SAMPLE_LIMIT = 20, DEFAULT_BULK_OPERATION_CONCURRENCY = 1;
 const DEFAULT_BULK_OPERATION_PRINCIPAL = { method: "in-process" };
+export async function runBulkWorkflowPool(items, concurrencyLimit, operation) {
+  const results = [];
+  for (let batchStart = 0;batchStart < items.length; batchStart += concurrencyLimit) {
+    const batchItems = items.slice(batchStart, batchStart + concurrencyLimit);
+    results.push(...await Promise.all(batchItems.map(async (item) => {
+      try {
+        return { item, status: "fulfilled", value: await operation(item) };
+      } catch (reason) {
+        return { item, status: "rejected", reason };
+      }
+    })));
+  }
+  return results;
+}
 export function buildActionableBulkWorkflowFilter(_internals, filter, actionableStatuses) {
   const effectiveStatuses = (filter.status === void 0 ? actionableStatuses : Array.isArray(filter.status) ? filter.status : [filter.status]).filter((status) => actionableStatuses.includes(status));
   if (effectiveStatuses.length === 0)

package/dist/core/engine/bulk-operations.d.ts

@@ -1,14 +1,15 @@
 import type { BulkCancelResult, BulkDeleteResult, BulkOperationCommitOptions, BulkOperationDryRunOptions, BulkOperationDryRunResult, BulkOperationOptions, BulkRetryFailedResult, BulkSignalAllCommitOptions, BulkSignalAllDryRunOptions, BulkSignalResult, BulkTagResult, ListFilter, PurgeResult } from '../types.ts';
 import { type CleanupWaiters } from './bulk-operations-purge.ts';
 import type { EngineInternals } from './internals.ts';
+import type { LifecycleCallbacks } from './lifecycle.ts';
 export { purgeInternal, TERMINAL_CLEANUP_DELAY_MS } from './bulk-operations-purge.ts';
 export declare function purge(internals: EngineInternals, filter: ListFilter | undefined, cleanupWaiters: CleanupWaiters): Promise<PurgeResult>;
 export declare function cancelAll(internals: EngineInternals, filter: ListFilter, options: BulkOperationDryRunOptions): Promise<BulkOperationDryRunResult>;
 export declare function cancelAll(internals: EngineInternals, filter: ListFilter, options?: BulkOperationCommitOptions): Promise<BulkCancelResult>;
 export declare function cancelAll(internals: EngineInternals, filter: ListFilter, options?: BulkOperationDryRunOptions | BulkOperationCommitOptions): Promise<BulkCancelResult | BulkOperationDryRunResult>;
 export declare function retryFailedAll(internals: EngineInternals, filter: ListFilter, options: BulkOperationDryRunOptions): Promise<BulkOperationDryRunResult>;
-export declare function retryFailedAll(internals: EngineInternals, filter: ListFilter, options?: BulkOperationCommitOptions): Promise<BulkRetryFailedResult>;
-export declare function retryFailedAll(internals: EngineInternals, filter: ListFilter, options?: BulkOperationDryRunOptions | BulkOperationCommitOptions): Promise<BulkRetryFailedResult | BulkOperationDryRunResult>;
+export declare function retryFailedAll(internals: EngineInternals, filter: ListFilter, options?: BulkOperationCommitOptions, callbacks?: LifecycleCallbacks): Promise<BulkRetryFailedResult>;
+export declare function retryFailedAll(internals: EngineInternals, filter: ListFilter, options?: BulkOperationDryRunOptions | BulkOperationCommitOptions, callbacks?: LifecycleCallbacks): Promise<BulkRetryFailedResult | BulkOperationDryRunResult>;
 export declare function signalAll(internals: EngineInternals, filter: ListFilter, name: string, payload: unknown, options: BulkSignalAllDryRunOptions): Promise<BulkOperationDryRunResult>;
 export declare function signalAll(internals: EngineInternals, filter: ListFilter, name: string, payload: unknown, options: BulkSignalAllCommitOptions): Promise<BulkSignalResult>;
 export declare function signalAll(internals: EngineInternals, filter: ListFilter, name: string, payload?: unknown, options?: BulkOperationCommitOptions): Promise<BulkSignalResult>;