@lostgradient/weft 0.24.0 → 0.24.1

package/dist/core/engine/bulk-operations.js

@@ -1,15 +1,8 @@
-import {
-  KEYS,
-  requireStorageCapability,
-  storageHas
-} from "../../storage/interface.js";
+import { KEYS, storageHas } from "../../storage/interface.js";
 import { assertScopedBulkWorkflowFilter } from "../bulk-workflow-filter.js";
-import { deserializeCheckpoint } from "../checkpoint.js";
-import { decode, encode } from "../codec.js";
-import { buildTimerBatchOperations } from "../scheduler.js";
-import { buildIndexOperations } from "../search-attributes.js";
-import { buildTerminalWorkflowIndexOperations, bulkMutateWorkflowTags } from "./attributes-tags.js";
+import { bulkMutateWorkflowTags } from "./attributes-tags.js";
 import { purgeInternal, purgeWorkflow } from "./bulk-operations-purge.js";
+import { runBulkFailedWorkflowRetry } from "./bulk-operations-retry.js";
 import {
   buildActionableBulkWorkflowFilter,
   buildBulkOperationPreparation,
@@ -17,43 +10,19 @@ import {
   normalizeBulkOperationOptions,
   prepareBulkOperation,
   resolveBulkOperationConcurrency,
+  runBulkWorkflowPool,
   shouldPersistBulkAudit,
   toBulkOperationError,
   validateBulkConfirmation,
   withBulkAuditEvent
 } from "./bulk-operations-shared.js";
 import { BulkDeleteRequiresTerminalWorkflowsError } from "./errors.js";
-import {
-  assertLeaseHeldForEngineWork,
-  commitFencedEngineWriteAllowingPreconditionFailure
-} from "./fenced-write.js";
+import { assertLeaseHeldForEngineWork } from "./fenced-write.js";
 import { BULK_OPERATION_BATCH_SIZE } from "./listing.js";
-import { createTerminalCleanupTimerId } from "./state-utilities.js";
-import { loadWorkflowState, runSerializedWorkflowStateWrite } from "./storage-io.js";
-import { decodeWorkflowState, isTerminalWorkflowStatus } from "./validation.js";
-import {
-  commitWithWorkflowClaimFold,
-  prepareWorkflowClaimFold,
-  throwWorkflowClaimUnavailable
-} from "./workflow-claim-fold.js";
-import { buildWorkflowConcurrencyStartOperations } from "./workflow-concurrency.js";
-import { buildWorkflowVisibilityIndexTransition } from "./workflow-indexes.js";
+import { loadWorkflowState } from "./storage-io.js";
+import { isTerminalWorkflowStatus } from "./validation.js";
 export { purgeInternal, TERMINAL_CLEANUP_DELAY_MS } from "./bulk-operations-purge.js";
-const ACTIVE_WORKFLOW_STATUSES = ["pending", "running", "suspended"], FAILED_WORKFLOW_STATUSES = ["failed"], CHECKPOINT_RETRY_CONCURRENCY_ADMISSION_MAX_ATTEMPTS = 5;
-async function runBulkWorkflowPool(items, concurrencyLimit, operation) {
-  const results = [];
-  for (let batchStart = 0;batchStart < items.length; batchStart += concurrencyLimit) {
-    const batchItems = items.slice(batchStart, batchStart + concurrencyLimit);
-    results.push(...await Promise.all(batchItems.map(async (item) => {
-      try {
-        return { item, status: "fulfilled", value: await operation(item) };
-      } catch (reason) {
-        return { item, status: "rejected", reason };
-      }
-    })));
-  }
-  return results;
-}
+const ACTIVE_WORKFLOW_STATUSES = ["pending", "running", "suspended"];
 export async function purge(internals, filter, cleanupWaiters) {
   return purgeInternal(internals, filter, { expiredOnly: !1, now: internals.options.getNow() }, cleanupWaiters);
 }
@@ -95,138 +64,10 @@ async function runBulkCancellation(internals, filter, options = {}) {
 export async function cancelAll(internals, filter, options = {}) {
   return runBulkCancellation(internals, filter, options);
 }
-async function runBulkFailedWorkflowRetry(internals, filter, options = {}) {
-  options = normalizeBulkOperationOptions(options);
-  assertScopedBulkWorkflowFilter(filter);
-  const actionableFilter = buildActionableBulkWorkflowFilter(internals, filter, FAILED_WORKFLOW_STATUSES), preparation = await prepareBulkOperation(internals, "retry-failed", actionableFilter, filter, {}, options);
-  if (options.dryRun === !0)
-    return preparation.preview;
-  validateBulkConfirmation(options, preparation);
-  const bulkConcurrency = resolveBulkOperationConcurrency(options);
-  let retried = 0;
-  const errors = [], retryResults = await runBulkWorkflowPool(preparation.workflowIds, bulkConcurrency, async (workflowId) => {
-    await retryFailedWorkflow(internals, workflowId);
-    return { status: "retried" };
-  });
-  for (const retryResult of retryResults) {
-    if (retryResult.status === "rejected") {
-      errors.push(toBulkOperationError(internals, retryResult.item, retryResult.reason));
-      continue;
-    }
-    retried += 1;
-  }
-  const result = { retried, failed: errors.length, errors };
-  if (!shouldPersistBulkAudit(options))
-    return result;
-  return withBulkAuditEvent(internals, preparation, options, result, retried);
-}
 export async function retryFailedAll(internals, filter, options = {}) {
   assertLeaseHeldForEngineWork(internals);
   return runBulkFailedWorkflowRetry(internals, filter, options);
 }
-async function retryFailedWorkflow(internals, workflowId) {
-  const state = await loadWorkflowState(internals, workflowId);
-  if (state === null)
-    throw Error("Workflow no longer exists");
-  if (state.status !== "failed")
-    throw Error(`Workflow is ${state.status}, not failed`);
-  if (await internals.storage.get(KEYS.checkpoint(workflowId)) !== null) {
-    await reactivateFailedWorkflowFromCheckpoint(internals, state);
-    await internals.engine.resume(workflowId);
-    return;
-  }
-  await internals.engine.start(state.type, state.input, {
-    id: workflowId,
-    onTerminalConflict: "start-new",
-    ...state.tags !== void 0 ? { tags: state.tags } : {}
-  });
-}
-async function reactivateFailedWorkflowFromCheckpoint(internals, state) {
-  const reactivated = await runSerializedWorkflowStateWrite(internals, state.id, async () => reactivateFailedWorkflowFromCheckpointSerialized(internals, state.id));
-  if (reactivated.terminalCleanupTimerId !== void 0)
-    await internals.scheduler.cancel(reactivated.terminalCleanupTimerId, state.id);
-}
-async function reactivateFailedWorkflowFromCheckpointSerialized(internals, workflowId) {
-  let lastConcurrencyStateKey;
-  for (let attempt = 0;attempt < CHECKPOINT_RETRY_CONCURRENCY_ADMISSION_MAX_ATTEMPTS; attempt += 1) {
-    const currentStateBytes = await internals.storage.get(KEYS.workflow(workflowId));
-    if (currentStateBytes === null)
-      throw Error("Workflow no longer exists");
-    const currentState = decodeWorkflowState(currentStateBytes);
-    if (currentState.status !== "failed")
-      throw Error(`Workflow is ${currentState.status}, not failed`);
-    const currentCheckpointBytes = await internals.storage.get(KEYS.checkpoint(workflowId));
-    if (currentCheckpointBytes === null)
-      throw Error("Checkpoint no longer exists");
-    const checkpoint = deserializeCheckpoint(currentCheckpointBytes), registration = internals.registrations.get(currentState.type);
-    if (registration === void 0)
-      throw Error(`No workflow registered with name "${currentState.type}" (needed to retry "${workflowId}")`);
-    const concurrencyStartOperations = registration.concurrency === void 0 ? void 0 : await buildWorkflowConcurrencyStartOperations(internals, currentState.type, workflowId, currentState.input, registration.concurrency);
-    lastConcurrencyStateKey = concurrencyStartOperations?.stateKey ?? lastConcurrencyStateKey;
-    const reactivatedState = buildReactivatedWorkflowState(internals, currentState), currentAttributes = await loadSearchAttributes(internals, workflowId), operations = [
-      ...buildTerminalWorkflowIndexOperations(currentState, reactivatedState),
-      { type: "put", key: KEYS.workflow(workflowId), value: encode(reactivatedState) },
-      ...buildWorkflowVisibilityIndexTransition(workflowId, currentState, reactivatedState).batchOps,
-      ...buildRetrySearchAttributeOperations(workflowId, currentAttributes, checkpoint.searchAttributes),
-      ...reactivatedState.executionDeadline === void 0 ? [] : buildTimerBatchOperations({
-        id: `deadline:${workflowId}`,
-        workflowId,
-        fireAt: reactivatedState.executionDeadline,
-        kind: "execution-deadline"
-      }),
-      ...concurrencyStartOperations?.operations ?? []
-    ], conditions = concurrencyStartOperations?.conditions ?? [];
-    if (await commitFailedWorkflowReactivation(internals, workflowId, operations, conditions))
-      return {
-        terminalCleanupTimerId: currentState.terminalCleanupToken === void 0 ? void 0 : createTerminalCleanupTimerId(!1, currentState.terminalCleanupToken)
-      };
-  }
-  throw Error(`Workflow concurrency admission for "${lastConcurrencyStateKey ?? workflowId}" changed too many times while retrying failed workflow "${workflowId}"`);
-}
-async function commitFailedWorkflowReactivation(internals, workflowId, operations, conditions) {
-  const claimFold = await prepareWorkflowClaimFold(internals, workflowId);
-  if (claimFold === void 0) {
-    if (conditions.length > 0)
-      requireStorageCapability(internals.storage, "conditionalBatch", "retry failed workflow");
-    return commitFencedEngineWriteAllowingPreconditionFailure(internals, workflowId, operations, conditions);
-  }
-  const result = await commitWithWorkflowClaimFold(internals, claimFold, operations, conditions, "retry failed workflow claim acquisition");
-  if (result.status === "committed")
-    return !0;
-  return result.claimConflict ? throwWorkflowClaimUnavailable(internals, workflowId) : !1;
-}
-function buildReactivatedWorkflowState(internals, state) {
-  const reactivatedState = {
-    ...state,
-    status: "running",
-    updatedAt: internals.options.getNow()
-  };
-  delete reactivatedState.error;
-  delete reactivatedState.errorStack;
-  delete reactivatedState.failureCategory;
-  delete reactivatedState.result;
-  delete reactivatedState.terminationReason;
-  delete reactivatedState.terminalCleanupToken;
-  return reactivatedState;
-}
-async function loadSearchAttributes(internals, workflowId) {
-  const attributeBytes = await internals.storage.get(KEYS.attribute(workflowId));
-  if (attributeBytes === null)
-    return {};
-  return decode(attributeBytes);
-}
-function buildRetrySearchAttributeOperations(workflowId, currentAttributes, checkpointAttributes) {
-  const operations = buildIndexOperations(workflowId, currentAttributes, checkpointAttributes);
-  if (Object.keys(checkpointAttributes).length === 0)
-    operations.push({ type: "delete", key: KEYS.attribute(workflowId) });
-  else
-    operations.push({
-      type: "put",
-      key: KEYS.attribute(workflowId),
-      value: encode(checkpointAttributes)
-    });
-  return operations;
-}
 export async function signalAll(internals, filter, name, payload, maybeOptions) {
   const options = normalizeBulkOperationOptions(maybeOptions ?? {});
   assertScopedBulkWorkflowFilter(filter);

package/dist/core/engine/callback-creators-bundles.js

@@ -16,6 +16,7 @@ import {
   createReviewOperationCallbacks,
   handleScheduleTimerForEngine
 } from "./callback-creators-schedule.js";
+import { resolveExecutableRegistrationForRevision } from "./dynamic-source-execution.js";
 import { commitFencedEngineWrite } from "./fenced-write.js";
 import { guardTerminalWorkflow, guardTerminalWorkflowAfterCoordinatedRequest } from "./guards.js";
 import { getInternals } from "./internals.js";
@@ -122,7 +123,7 @@ export function createTimeOperationCallbacks(engine) {
     loadWorkflowState: (workflowId) => loadWorkflowState(getInternals(engine), workflowId),
     failWorkflow: (workflowId, error) => failWorkflow(getInternals(engine), workflowId, error, createTerminationCallbacks(engine)),
     runSerializedWorkflowStateWrite: (workflowId, writeOperation) => runSerializedWorkflowStateWrite(getInternals(engine), workflowId, writeOperation),
-    beginWorkflowExecution: (workflowId, workflowExecutionToken, workflowType, input, checkpoint, executionDeadline, executionStateOwnerId, registration) => beginWorkflowExecution(getInternals(engine), workflowId, workflowExecutionToken, workflowType, input, checkpoint, executionDeadline, executionStateOwnerId, registration, createLifecycleCallbacks(engine)),
+    beginWorkflowExecution: (workflowId, workflowExecutionToken, workflowType, revision, input, checkpoint, executionDeadline, executionStateOwnerId, registration) => beginWorkflowExecution(getInternals(engine), workflowId, workflowExecutionToken, workflowType, revision, input, checkpoint, executionDeadline, executionStateOwnerId, registration, createLifecycleCallbacks(engine)),
     workflowVersionTupleFromState: (state) => workflowVersionTupleFromState(getInternals(engine), state, createLifecycleCallbacks(engine)),
     setWorkflowStartHeaders: (workflowId, headers) => setWorkflowStartHeaders(getInternals(engine), workflowId, headers, createLifecycleCallbacks(engine)),
     loadWorkflowStartHeaders: (workflowId) => loadWorkflowStartHeaders(getInternals(engine), workflowId, createLifecycleCallbacks(engine)),
@@ -131,7 +132,8 @@ export function createTimeOperationCallbacks(engine) {
     runWorkflowFinalizer: (workflowId, timerId) => runWorkflowFinalizer(getInternals(engine), workflowId, timerId, createTerminationCallbacks(engine)),
     handleScheduleTimer: (entry) => handleScheduleTimerForEngine(engine, entry),
     timeout: (workflowId) => engine.timeout(workflowId),
-    handleCleanupError: (source, error, workflowId) => createTerminationCallbacks(engine).handleCleanupError(source, error, workflowId)
+    handleCleanupError: (source, error, workflowId) => createTerminationCallbacks(engine).handleCleanupError(source, error, workflowId),
+    resolveExecutableRegistrationForRevision: (type, revision) => resolveExecutableRegistrationForRevision(engine, getInternals(engine), type, revision)
   };
 }
 export function createUpdateCallbacks(engine) {

package/dist/core/engine/callback-creators-core.js

@@ -3,6 +3,10 @@ import {
   broadcast as broadcastFromInternals,
   forwardEventToHandle as forwardEventToHandleFromBroadcast
 } from "./broadcast.js";
+import {
+  resolveExecutableRegistration,
+  resolveExecutableRegistrationForRevision
+} from "./dynamic-source-execution.js";
 import { createWorkflowHandleWithResultPromise } from "./handle-result.js";
 import {
   hasLocalCheckpointOwnership,
@@ -79,7 +83,11 @@ export function createLifecycleCallbacks(engine) {
     failWorkflowForUnavailableServices: (workflowId, error) => failWorkflow(getInternals(engine), workflowId, error, createTerminationCallbacks(engine), "system"),
     failWorkflowForRecoveryHook: (workflowId, error) => failWorkflow(getInternals(engine), workflowId, error, createTerminationCallbacks(engine), "system"),
     failWorkflowForCheckpointDecodeError: (workflowId, error) => failWorkflow(getInternals(engine), workflowId, error, createTerminationCallbacks(engine), "system"),
-    failWorkflowForVersionMismatch: (workflowId, error) => failWorkflow(getInternals(engine), workflowId, error, createTerminationCallbacks(engine), "system")
+    failWorkflowForVersionMismatch: (workflowId, error) => failWorkflow(getInternals(engine), workflowId, error, createTerminationCallbacks(engine), "system"),
+    resolveExecutableRegistration: (type, onRevisionChosen) => resolveExecutableRegistration(engine, getInternals(engine), type, onRevisionChosen),
+    failWorkflowForUnavailableDynamicSource: (workflowId, error) => failWorkflow(getInternals(engine), workflowId, error, createTerminationCallbacks(engine), "system"),
+    resolveExecutableRegistrationForRevision: (type, revision) => resolveExecutableRegistrationForRevision(engine, getInternals(engine), type, revision),
+    failWorkflowForRevisionUnavailable: (workflowId, error) => failWorkflow(getInternals(engine), workflowId, error, createTerminationCallbacks(engine), "system")
   };
 }
 export function createTerminationCallbacksWith(engine, handleScheduledWorkflowTerminal) {

package/dist/core/engine/callback-creators-schedule.js

@@ -20,8 +20,8 @@ import { feedOperationResult } from "./strategy-helpers.js";
 import { ensureTerminalCleanupTracked, failWorkflow, handleCleanupError } from "./termination.js";
 export function createScheduleCallbacks(engine) {
   return {
-    startWorkflow: async (type, input, options, additionalStartOperations) => {
-      await startWorkflow(getInternals(engine), type, input, options, additionalStartOperations, createLifecycleCallbacks(engine));
+    startWorkflow: async (type, input, options, additionalStartOperations, revisionOverride) => {
+      await startWorkflow(getInternals(engine), type, input, options, additionalStartOperations, createLifecycleCallbacks(engine), void 0, revisionOverride);
     },
     loadWorkflowState: (workflowId) => loadWorkflowState(getInternals(engine), workflowId),
     cancelWorkflow: (workflowId) => engine.cancel(workflowId),

package/dist/core/engine/catalog-readiness.js

@@ -1,6 +1,7 @@
 import { restoreWorkflowCatalog, WorkflowCatalog } from "../catalog/index.js";
 import { buildWorkflowManifestForType } from "../registry-workflow-manifest.js";
 import { dispatchCatalogInstallAndActivatedEvents } from "./catalog-events.js";
+import { resolveOrphanedCatalogTombstones } from "./catalog-tombstone-recovery.js";
 import { EngineDisposedError } from "./errors.js";
 import { getInternals } from "./internals.js";
 export { getWorkflowCatalog } from "./internals.js";
@@ -44,6 +45,7 @@ export async function ensureWorkflowCatalogReady(engine) {
     if (internals.disposed)
       throw new EngineDisposedError;
     if (!internals.catalogRestored) {
+      await resolveOrphanedCatalogTombstones(internals.storage);
       internals.workflowCatalog = new WorkflowCatalog(internals.storage, await restoreWorkflowCatalog(internals.storage));
       internals.catalogRestored = !0;
     }

package/dist/core/engine/catalog-removal.d.ts

@@ -15,6 +15,7 @@
 import { type WorkflowRevisionReferenceCounts } from '../catalog/index.ts';
 import type { Engine } from './index.ts';
 import { type EngineInternals } from './internals.ts';
+type RegistrationEntry = EngineInternals['registrations'] extends Map<string, infer Entry> ? Entry : never;
 /**
  * Reserve one in-flight-start slot against `type`'s currently active
  * revision (if any), returning the revision reserved (or `undefined` when
@@ -25,15 +26,70 @@ import { type EngineInternals } from './internals.ts';
  * caller and passed back to {@link releaseInFlightStart} in its `finally` —
  * never re-resolved, so a concurrent activation moving the pointer
  * mid-start cannot decrement a different revision than was incremented.
+ *
+ * `revisionOverride` (WFT-15/16) is the revision `resolveExecutableRegistration()`
+ * already resolved for a `registerSource()`-registered `type` — passed
+ * through instead of re-deriving via `catalog.resolveActive(type)`, which
+ * would read `undefined` for a dynamic source with no catalog active
+ * pointer set (a source can resolve and install without ever being
+ * `engine.workflows.activate()`-d), making `inFlightStarts` diagnostics
+ * silently miss a lazy run in flight. `undefined` (the default) preserves
+ * the eager-path behavior byte-for-byte.
+ */
+export declare function reserveInFlightStart(internals: EngineInternals, type: string, revisionOverride?: string): string | undefined;
+/**
+ * `startWorkflow`'s combined "resolve `type`, reserve an `inFlightStarts`
+ * slot for the resolved revision" step. Reserves EARLY — before the loader
+ * is awaited, via `resolve()`'s `onRevisionChosen` hook — for a lazy type,
+ * closing the window where a concurrent `removeWorkflowRevision()` could
+ * observe zero references against a revision this call's own source load
+ * is about to durably (re)install; reserves afterward for an eager type,
+ * exactly as before WFT-15/16 (that path never invokes the hook). If the
+ * loader then fails — `resolve()` rejects AFTER the hook already fired — the
+ * early reservation is released here before rethrowing: `startWorkflow`
+ * only assigns its own `inFlightRevision` on a successful return, so its
+ * `finally` releases nothing on this path and the increment would otherwise
+ * leak forever, permanently reporting the revision non-`removable`.
+ *
+ * `resolvedRevision` is the raw value `resolve()` itself returned —
+ * `undefined` for an eager registration, the resolved candidate for a
+ * dynamic source — distinct from `inFlightRevision`, which for an eager
+ * type falls back to the catalog's cached ACTIVE pointer (see
+ * {@link reserveInFlightStart}). A caller that needs "the exact revision of
+ * the code this process is about to run" (WFT-17's `WorkflowState.revision`)
+ * must use `resolvedRevision`, not `inFlightRevision` — the two intentionally
+ * diverge for an eager type under a multi-engine deployment where this
+ * process's own registration lags the durable active pointer.
  */
-export declare function reserveInFlightStart(internals: EngineInternals, type: string): string | undefined;
+export declare function resolveAndReserveExecutableRegistration(internals: EngineInternals, type: string, resolve: (type: string, onRevisionChosen?: (revision: string) => void) => Promise<{
+    entry: RegistrationEntry;
+    revision: string | undefined;
+}>): Promise<{
+    registration: RegistrationEntry;
+    inFlightRevision: string | undefined;
+    resolvedRevision: string | undefined;
+}>;
 /** Release the slot {@link reserveInFlightStart} reserved; a no-op when `revision` is `undefined`. */
 export declare function releaseInFlightStart(internals: EngineInternals, type: string, revision: string | undefined): void;
 /**
  * Count every in-process reference this batch wires to a real signal
  * against `(name, revision)`. `registeredDefinitions` and `inFlightStarts`
- * are real; the other five fields of {@link WorkflowRevisionReferenceCounts}
- * stay `0` until WFT-17 (see that type's own field-level docs).
+ * are in-memory, constant-time lookups; `nonTerminalRuns` (WFT-17 — see
+ * {@link countNonTerminalRunsForRevision}) is an UNBOUNDED, full `wf:`-prefix
+ * durable scan — the same shape `groupActiveWorkflowsByType` already uses
+ * elsewhere for this class of diagnostic, so it's consistent with how this
+ * codebase already pays for this, but it is O(n) in the total number of
+ * workflow records, not bounded by `(name, revision)`. `removeWorkflowRevision()`
+ * calls this TWICE per removal attempt (its own pre-check, and
+ * {@link finalizeRevisionRemoval}'s post-check) — worth knowing before
+ * wiring `removeWorkflowRevision()` into an automated cleanup path on a
+ * large durable store, rather than the operator-triggered use this batch
+ * assumes. `pinnedSchedules` is ALSO an unbounded, full `schedule:`-prefix
+ * durable scan (WFT-20 — see {@link countPinnedSchedulesForRevision}), so
+ * this function now pays for two bounded-but-unbounded-in-store-size scans
+ * per call. The remaining two fields of {@link WorkflowRevisionReferenceCounts}
+ * stay `0` — the dispatch ledger and execution realms are out of this
+ * batch's scope.
  */
 export declare function countWorkflowRevisionReferences(engine: Engine, name: string, revision: string): Promise<WorkflowRevisionReferenceCounts>;
 /**
@@ -76,6 +132,39 @@ export type WorkflowCatalogRemovalResult = Readonly<{
  * compare-and-swap loses to a concurrent writer — the caller may re-read
  * and retry. Dispatches `catalog:revision-removed` on success.
  *
+ * Checks references BOTH before and after the durable delete (WFT-17,
+ * Codex review on PR #958). The pre-check above is an OPTIMIZATION — it
+ * refuses upfront in the common case, avoiding a needless delete-then-restore
+ * round trip — but is not, on its own, the guarantee: a concurrent
+ * `engine.start()` on a DIFFERENT process could read the entry as still
+ * installed, and commit a NEW run pinned to `revision`, in the narrow
+ * window between this function's pre-check scan and its own delete CAS
+ * landing; the pre-check's zero-references snapshot would then be stale
+ * by the time the delete actually commits. The POST-check closes this: it
+ * is safe to trust because `start-commit.ts`'s own start batch fences on
+ * this exact catalog entry's bytes (`buildCatalogEntryStartPrecondition`)
+ * — `conditionalBatch`'s precondition is evaluated against LIVE storage
+ * state at commit time, so ANY start whose commit lands after this
+ * function's delete necessarily loses its own CAS and never creates the
+ * reference at all. A nonzero post-delete count can therefore only be a
+ * run that committed BEFORE the delete — this function restores the entry
+ * and reports `'referenced'` rather than leaving a real, still-referenced
+ * run pinned to a revision the catalog no longer carries.
+ *
+ * `catalog.remove()`'s delete and `finalizeRevisionRemoval`'s restore-or-
+ * finalize used to be two SEPARATE commits (WFT-17/18, Codex review on PR
+ * #958): a process crash between them left the revision durably uninstalled
+ * with NO durable record of what was deleted, recoverable by no one. Both
+ * halves of `removeCatalogEntry` (the delete) now land in ONE
+ * `conditionalBatch` with a `catalog-tombstone:<name>:<revision>` record
+ * (the exact deleted bytes) — see `core/catalog/removal.ts`'s module doc —
+ * so this function's own restore-or-finalize below is itself a SECOND
+ * atomic operation (not a bare `install()`/no-op), and a crash between the
+ * two commits leaves a durable tombstone ANY process can resolve, not just
+ * this one. `catalog-tombstone-recovery.ts`'s boot-time sweep and this
+ * function's own targeted pre-check (below) both close that residual
+ * window; see that module's doc for the full mechanism.
+ *
  * @example
  * ```ts
  * import { Engine, removeWorkflowRevision, workflow } from '@lostgradient/weft';
@@ -111,6 +200,20 @@ export type WorkflowRevisionDiagnostics = Readonly<{
     activeRevision?: string;
     references: WorkflowRevisionReferenceCounts;
     removable: boolean;
+    /**
+     * Dynamic-source load diagnostics (WFT-15/16), present only when `name`
+     * has an entry in `internals.sources.byName` — i.e. `name` has ever been
+     * `registerSource()`-registered on this engine. `undefined` for a purely
+     * eager name, or a dynamic source this process has never heard of.
+     */
+    source?: {
+        kind: import('../source/index.ts').WorkflowSourceKind;
+        requestedRevision: string;
+        state: import('./source-runtime-state.ts').SourceLoadState;
+        loadDurationMs?: number;
+        lastFailureCategory?: import('../types/identity.ts').FailureCategory;
+        waiterCount: number;
+    };
 }>;
 /**
  * Bounded diagnostics for one `(name, revision)`: whether it is installed,
@@ -130,3 +233,4 @@ export type WorkflowRevisionDiagnostics = Readonly<{
  * ```
  */
 export declare function getWorkflowRevisionDiagnostics(engine: Engine, name: string, revision: string): Promise<WorkflowRevisionDiagnostics>;
+export {};

package/dist/core/engine/catalog-removal.js

@@ -1,29 +1,47 @@
 import {
   decrementNestedRevisionCount,
+  finalizeCatalogTombstone,
   incrementNestedRevisionCount,
   readNestedRevisionCount,
+  restoreCatalogEntryFromTombstone,
   totalWorkflowRevisionReferences
 } from "../catalog/index.js";
 import { WorkflowRevisionRemovedEvent } from "../events/catalog-events.js";
 import { ensureWorkflowCatalogReady, getWorkflowCatalog } from "./catalog-readiness.js";
+import { resolveCatalogTombstoneIfPresent } from "./catalog-tombstone-recovery.js";
 import { getInternals } from "./internals.js";
-export function reserveInFlightStart(internals, type) {
-  const revision = internals.workflowCatalog?.resolveActive(type)?.revision;
+import { countNonTerminalRunsForRevision } from "./nonterminal-revision-count.js";
+import { countPinnedSchedulesForRevision } from "./pinned-schedule-revision-count.js";
+import { readSourceLoadDiagnostics, readSourceWaiterCount } from "./source-diagnostics.js";
+export function reserveInFlightStart(internals, type, revisionOverride) {
+  const revision = revisionOverride ?? internals.workflowCatalog?.resolveActive(type)?.revision;
   if (revision !== void 0)
     incrementNestedRevisionCount(internals.inFlightStartsByRevision, type, revision);
   return revision;
 }
+export async function resolveAndReserveExecutableRegistration(internals, type, resolve) {
+  let earlyReservation;
+  try {
+    const { entry: registration, revision } = await resolve(type, (chosen) => {
+      earlyReservation = reserveInFlightStart(internals, type, chosen);
+    }), inFlightRevision = earlyReservation !== void 0 ? earlyReservation : reserveInFlightStart(internals, type, revision);
+    return { registration, inFlightRevision, resolvedRevision: revision };
+  } catch (error) {
+    releaseInFlightStart(internals, type, earlyReservation);
+    throw error;
+  }
+}
 export function releaseInFlightStart(internals, type, revision) {
   if (revision !== void 0)
     decrementNestedRevisionCount(internals.inFlightStartsByRevision, type, revision);
 }
 export async function countWorkflowRevisionReferences(engine, name, revision) {
-  const internals = getInternals(engine);
+  const internals = getInternals(engine), nonTerminalRuns = await countNonTerminalRunsForRevision(internals.storage, name, revision), pinnedSchedules = await countPinnedSchedulesForRevision(internals.storage, name, revision);
   return {
     registeredDefinitions: internals.registeredCatalogRevisions.get(name) === revision ? 1 : 0,
     inFlightStarts: readNestedRevisionCount(internals.inFlightStartsByRevision, name, revision),
-    nonTerminalRuns: 0,
-    pinnedSchedules: 0,
+    nonTerminalRuns,
+    pinnedSchedules,
     pendingDispatches: 0,
     activeExecutionRealms: 0,
     retainedRecoveryRecords: 0
@@ -31,20 +49,20 @@ export async function countWorkflowRevisionReferences(engine, name, revision) {
 }
 export async function removeWorkflowRevision(engine, name, revision) {
   await ensureWorkflowCatalogReady(engine);
-  const catalog = getWorkflowCatalog(engine);
+  const catalog = getWorkflowCatalog(engine), internals = getInternals(engine);
+  await resolveCatalogTombstoneIfPresent(internals.storage, name, revision);
   if (!await catalog.hasInstalled(name, revision))
     return { removed: !1, reason: "not-found" };
   const active = await catalog.resolveActiveDurable(name);
   if (active !== void 0 && active.revision === revision)
     return { removed: !1, reason: "active", activeRevision: active.revision };
-  const references = await countWorkflowRevisionReferences(engine, name, revision);
-  if (totalWorkflowRevisionReferences(references) > 0)
-    return { removed: !1, reason: "referenced", references };
+  const preReferences = await countWorkflowRevisionReferences(engine, name, revision);
+  if (totalWorkflowRevisionReferences(preReferences) > 0)
+    return { removed: !1, reason: "referenced", references: preReferences };
   const result = await catalog.remove(name, revision);
   switch (result.outcome) {
     case "removed":
-      engine.dispatchEvent(new WorkflowRevisionRemovedEvent(name, revision));
-      return { removed: !0 };
+      return finalizeRevisionRemoval(engine, name, revision, result.tombstoneBytes);
     case "not-found":
       return { removed: !1, reason: "not-found" };
     case "active":
@@ -55,9 +73,19 @@ export async function removeWorkflowRevision(engine, name, revision) {
       throw Error(`Unknown workflow catalog removal outcome: ${String(result)}`);
   }
 }
+async function finalizeRevisionRemoval(engine, name, revision, tombstoneBytes) {
+  const internals = getInternals(engine), postReferences = await countWorkflowRevisionReferences(engine, name, revision);
+  if (totalWorkflowRevisionReferences(postReferences) > 0) {
+    await restoreCatalogEntryFromTombstone(internals.storage, name, revision, tombstoneBytes);
+    return { removed: !1, reason: "referenced", references: postReferences };
+  }
+  await finalizeCatalogTombstone(internals.storage, name, revision, tombstoneBytes);
+  engine.dispatchEvent(new WorkflowRevisionRemovedEvent(name, revision));
+  return { removed: !0 };
+}
 export async function getWorkflowRevisionDiagnostics(engine, name, revision) {
   await ensureWorkflowCatalogReady(engine);
-  const catalog = getWorkflowCatalog(engine), installed = await catalog.hasInstalled(name, revision), activePointer = await catalog.resolveActiveDurable(name), active = activePointer !== void 0 && activePointer.revision === revision, references = await countWorkflowRevisionReferences(engine, name, revision), removable = installed && !active && totalWorkflowRevisionReferences(references) === 0;
+  const catalog = getWorkflowCatalog(engine), installed = await catalog.hasInstalled(name, revision), activePointer = await catalog.resolveActiveDurable(name), active = activePointer !== void 0 && activePointer.revision === revision, references = await countWorkflowRevisionReferences(engine, name, revision), removable = installed && !active && totalWorkflowRevisionReferences(references) === 0, source = buildSourceDiagnostics(getInternals(engine), name, revision);
   return {
     name,
     revision,
@@ -65,6 +93,31 @@ export async function getWorkflowRevisionDiagnostics(engine, name, revision) {
     active,
     ...activePointer !== void 0 ? { activeRevision: activePointer.revision } : {},
     references,
-    removable
+    removable,
+    ...source !== void 0 ? { source } : {}
+  };
+}
+function resolveDiagnosticsSourceKind(diagnostics, registeredRevisions, revision) {
+  if (diagnostics !== void 0)
+    return diagnostics.kind;
+  const registered = registeredRevisions.get(revision);
+  return registered === void 0 ? "module" : registered.descriptor.kind;
+}
+function buildSourceDiagnostics(internals, name, revision) {
+  const registeredRevisions = internals.sources.byName.get(name);
+  if (registeredRevisions === void 0)
+    return;
+  const diagnostics = readSourceLoadDiagnostics(internals, name, revision);
+  return {
+    kind: resolveDiagnosticsSourceKind(diagnostics, registeredRevisions, revision),
+    requestedRevision: revision,
+    state: diagnostics === void 0 ? "idle" : diagnostics.state,
+    ...diagnostics?.loadDurationMs !== void 0 && {
+      loadDurationMs: diagnostics.loadDurationMs
+    },
+    ...diagnostics?.lastFailureCategory !== void 0 && {
+      lastFailureCategory: diagnostics.lastFailureCategory
+    },
+    waiterCount: readSourceWaiterCount(internals, name, revision)
   };
 }

package/dist/core/engine/catalog-tombstone-recovery.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Resolves durable `catalog-tombstone:<name>:<revision>` records left
+ * behind by {@link import('../catalog/removal.ts').removeCatalogEntry}
+ * (WFT-17/18, Codex review on PR #958) — a tombstone exists only in the
+ * brief window between its delete-and-tombstone commit and the SAME call's
+ * own resolution (restore or finalize), normally sub-millisecond. A
+ * tombstone that outlives that window is an ORPHAN: the process that
+ * created it crashed (or otherwise failed) before resolving it. This
+ * module makes that recoverable by ANY process, not just the one that
+ * crashed — the exact gap the tombstone design closes (see
+ * `core/catalog/removal.ts`'s module doc for the full mechanism).
+ *
+ * Deliberately lives in `core/engine/`, not alongside the tombstone
+ * primitives in `core/catalog/`: resolving an orphan needs a FRESH
+ * reference count, and the only reference signal meaningful for a
+ * crashed peer's tombstone is the durable one —
+ * `nonterminal-revision-count.ts`'s `wf:`-prefix scan (in-process signals
+ * like `registeredDefinitions`/`inFlightStarts` are per-engine-instance and
+ * say nothing about what a DIFFERENT, crashed process was doing).
+ * `core/catalog/**` never imports from `core/engine/**` — a directional
+ * boundary `check-import-cycles.ts` enforces — so this orchestration has to
+ * live on the engine side of that line.
+ *
+ * Two call sites: `catalog-readiness.ts`'s boot-time sweep (every
+ * orphaned tombstone in the whole store, run once per engine instance
+ * before recovery or any new start can observe stale state) and
+ * `catalog-removal.ts`'s `removeWorkflowRevision` (a single, targeted
+ * check for the EXACT `(name, revision)` key it is about to act on, since
+ * a long-lived engine could observe a peer crash mid-lifetime, after its
+ * own boot sweep already ran).
+ *
+ * @module core/engine/catalog-tombstone-recovery
+ */
+import { type Storage } from '../../storage/interface.ts';
+/**
+ * Sweep every `catalog-tombstone:` record in `storage` and resolve each
+ * one. Bounded by the number of orphaned tombstones actually present —
+ * normally zero, so this is a cheap prefix scan that yields nothing to
+ * iterate; only non-trivial after a real crash left orphans behind. Called
+ * once per engine instance, at catalog-boot time
+ * (`catalog-readiness.ts`), before recovery's own preflight or any fresh
+ * start can observe a revision this sweep would otherwise still be
+ * resolving.
+ */
+export declare function resolveOrphanedCatalogTombstones(storage: Storage): Promise<void>;
+/**
+ * Targeted check for ONE `(name, revision)` tombstone — used by
+ * `removeWorkflowRevision` immediately before it acts, so a peer crash
+ * that happened after this engine's own boot-time sweep already ran (a
+ * long-lived engine) does not leave a stale tombstone permanently blocking
+ * every future removal attempt on that exact key (`removeCatalogEntry`'s
+ * own CAS requires the tombstone key absent). A no-op when no tombstone is
+ * present for this exact key — the overwhelmingly common case, costing one
+ * `storage.get`.
+ */
+export declare function resolveCatalogTombstoneIfPresent(storage: Storage, name: string, revision: string): Promise<void>;