@dudousxd/nestjs-durable-core 0.81.0 → 0.83.0

package/dist/index.cjs

@@ -30,6 +30,7 @@ __export(index_exports, {
   ContinueAsNew: () => ContinueAsNew,
   DEFAULT_CONCURRENCY_STATUSES: () => DEFAULT_CONCURRENCY_STATUSES,
   DEFAULT_PARTITION: () => DEFAULT_PARTITION,
+  DEFAULT_STALLED_AFTER_MS: () => DEFAULT_STALLED_AFTER_MS,
   DURABLE_OPTIONS: () => DURABLE_OPTIONS,
   DURABLE_OPTIONS_CANONICAL: () => DURABLE_OPTIONS_CANONICAL,
   DURABLE_STEP_CONFIG: () => DURABLE_STEP_CONFIG,
@@ -53,6 +54,7 @@ __export(index_exports, {
   NestedWorkflowCallError: () => NestedWorkflowCallError,
   NonDeterminismError: () => NonDeterminismError,
   QueueController: () => QueueController,
+  RUN_HISTORY_FIXTURE_VERSION: () => RUN_HISTORY_FIXTURE_VERSION,
   RUN_VALUE_FACET_LIMIT: () => RUN_VALUE_FACET_LIMIT,
   RUN_VALUE_FACET_SCAN: () => RUN_VALUE_FACET_SCAN,
   RemoteStepError: () => RemoteStepError,
@@ -70,6 +72,7 @@ __export(index_exports, {
   ScheduleAlreadyExistsError: () => ScheduleAlreadyExistsError,
   ScheduleClient: () => ScheduleClient,
   ScheduleNotFoundError: () => ScheduleNotFoundError,
+  SchemaValidationError: () => SchemaValidationError,
   SignalTimeoutError: () => SignalTimeoutError,
   SingletonQueueFullError: () => SingletonQueueFullError,
   TERMINAL_RUN_STATUSES: () => TERMINAL_RUN_STATUSES,
@@ -126,6 +129,7 @@ __export(index_exports, {
   facetOrigin: () => facetOrigin,
   fanPageBounds: () => fanPageBounds,
   fanPageEtag: () => fanPageEtag,
+  findStalledRuns: () => findStalledRuns,
   foldedCount: () => foldedCount,
   handoffHandlerId: () => handoffHandlerId,
   handoffName: () => handoffName,
@@ -180,6 +184,7 @@ __export(index_exports, {
   runStepHandler: () => runStepHandler,
   runValueFacetsFromRuns: () => runValueFacetsFromRuns,
   sanitizeQueueToken: () => sanitizeQueueToken,
+  scheduleRunIdAt: () => scheduleRunIdAt,
   scheduledRunId: () => scheduledRunId,
   singletonAdmissionDenialPatch: () => singletonAdmissionDenialPatch,
   singletonAdmissionTags: () => singletonAdmissionTags,
@@ -478,6 +483,31 @@ var FatalError = class extends Error {
     this.details = details;
   }
 };
+var SchemaValidationError = class extends FatalError {
+  static {
+    __name(this, "SchemaValidationError");
+  }
+  static code = "schema_validation";
+  issues;
+  /**
+  * @param label What was validated, e.g. `workflow "checkout" input`; the message is
+  *   `invalid <label>: <path>: <message>; …`.
+  * @param summary Overrides the `<path>: <message>; …` part, for issues whose messages already
+  *   name their field (class-validator's `orderId must be a string`).
+  */
+  constructor(label, issues, code = "schema_validation", summary) {
+    const normalized = issues.map((issue) => ({
+      message: issue.message,
+      path: (issue.path ?? []).map((segment) => String(typeof segment === "object" && segment !== null ? segment.key : segment))
+    }));
+    const text = summary ?? normalized.map((issue) => issue.path.length > 0 ? `${issue.path.join(".")}: ${issue.message}` : issue.message).join("; ");
+    super(`invalid ${label}: ${text}`, code, {
+      issues: normalized
+    });
+    this.name = "SchemaValidationError";
+    this.issues = normalized;
+  }
+};
 var ConcurrencyLimitError = class extends Error {
   static {
     __name(this, "ConcurrencyLimitError");
@@ -2209,9 +2239,14 @@ function liveHandoff(runId, checkpoints) {
   return live;
 }
 __name(liveHandoff, "liveHandoff");
+var SCHEMA_VALIDATION_CODES = /* @__PURE__ */ new Set([
+  "invalid_input",
+  "invalid_output"
+]);
 function routeFor(routes, error2) {
   if (!routes?.length || error2.code && UNROUTABLE_CODES.has(error2.code)) return void 0;
-  return (error2.code ? routes.find((r) => r.code === error2.code) : void 0) ?? routes.find((r) => r.code === void 0);
+  const { code } = error2;
+  return (code ? routes.find((r) => r.code === code) : void 0) ?? (code && SCHEMA_VALIDATION_CODES.has(code) ? routes.find((r) => r.code === "schema_validation") : void 0) ?? routes.find((r) => r.code === void 0);
 }
 __name(routeFor, "routeFor");
 function assertRoutes(workflow, routes) {
@@ -2483,9 +2518,13 @@ __name(scheduleRunIdsAt, "scheduleRunIdsAt");
 async function runSchedules(engine, schedules, nowMs, opts) {
   const random = opts?.random ?? Math.random;
   const sleep = opts?.sleep ?? defaultSleep;
+  const settled = settledIdsFor(engine);
   const ids = [];
   for (const s of schedules) {
+    if (engine.isDraining?.()) break;
     if (s.paused) continue;
+    const candidates = scheduleRunIdsAt(s, nowMs).filter((id) => !settled.get(s.key)?.has(id));
+    if (candidates.length === 0) continue;
     if (s.overlap === "skip" && s.everyMs) {
       const prevBucket = Math.floor(nowMs / s.everyMs) - 1;
       if (prevBucket >= 0) {
@@ -2494,14 +2533,44 @@ async function runSchedules(engine, schedules, nowMs, opts) {
       }
     }
     if (s.jitter && s.jitter > 0) await sleep(Math.floor(random() * s.jitter));
-    for (const runId of scheduleRunIdsAt(s, nowMs)) {
-      await engine.start(s.workflow, s.input, runId);
+    for (const runId of candidates) {
+      if (engine.isDraining?.()) break;
+      const existing = await engine.getRun(runId);
+      remember(settled, s.key, runId);
+      if (existing) continue;
+      await engine.start(workflowName(s.workflow), s.input, runId, {
+        namespace: s.namespace
+      });
       ids.push(runId);
     }
   }
   return ids;
 }
 __name(runSchedules, "runSchedules");
+var settledRunIds = /* @__PURE__ */ new WeakMap();
+function settledIdsFor(engine) {
+  let memo = settledRunIds.get(engine);
+  if (memo === void 0) {
+    memo = /* @__PURE__ */ new Map();
+    settledRunIds.set(engine, memo);
+  }
+  return memo;
+}
+__name(settledIdsFor, "settledIdsFor");
+function remember(memo, key, runId) {
+  const seen = memo.get(key);
+  if (seen === void 0) {
+    memo.set(key, /* @__PURE__ */ new Set([
+      runId
+    ]));
+    return;
+  }
+  seen.add(runId);
+  if (seen.size > 64) memo.set(key, /* @__PURE__ */ new Set([
+    runId
+  ]));
+}
+__name(remember, "remember");
 
 // src/schedules.ts
 var ScheduleAlreadyExistsError = class extends Error {
@@ -3017,6 +3086,66 @@ var SingletonGate = class {
   }
 };
 
+// src/stalled-runs.ts
+var DEFAULT_STALLED_AFTER_MS = 15 * 6e4;
+async function findStalledRuns(store, options = {}) {
+  const now = options.now ?? Date.now();
+  const thresholdMs = options.thresholdMs ?? DEFAULT_STALLED_AFTER_MS;
+  const scan = Math.max(1, options.scan ?? 1e3);
+  const pageSize = Math.min(200, scan);
+  const stalled = [];
+  for (let offset = 0; offset < scan; offset += pageSize) {
+    const limit = Math.min(pageSize, scan - offset);
+    const page = await store.listRuns({
+      statuses: [
+        "running",
+        "suspended"
+      ],
+      ...options.namespace !== void 0 ? {
+        namespace: options.namespace
+      } : {},
+      limit,
+      offset
+    });
+    for (const run of page) {
+      const ageMs = now - run.updatedAt.getTime();
+      if (ageMs < thresholdMs) continue;
+      const info2 = await stalledInfo(store, run, now, thresholdMs, ageMs);
+      if (info2) stalled.push(info2);
+    }
+    if (page.length < limit) break;
+  }
+  return stalled;
+}
+__name(findStalledRuns, "findStalledRuns");
+async function stalledInfo(store, run, now, thresholdMs, ageMs) {
+  let stalePending = null;
+  for (const cp of await store.listCheckpoints(run.id)) {
+    if (cp.kind !== "remote" || cp.status !== "pending") continue;
+    const stepAgeMs = now - cp.enqueuedAt.getTime();
+    if (stepAgeMs < thresholdMs) continue;
+    if (cp.wakeAt != null && cp.wakeAt > now) continue;
+    if (!stalePending || stepAgeMs > stalePending.ageMs) {
+      stalePending = {
+        seq: cp.seq,
+        name: cp.name,
+        attempts: cp.attempts,
+        ageMs: stepAgeMs,
+        leaseUntil: cp.wakeAt ?? null
+      };
+    }
+  }
+  const wakeForever = run.status === "suspended" && run.wakeAt == null;
+  if (!wakeForever && !stalePending) return null;
+  return {
+    run,
+    ageMs,
+    wakeForever,
+    stalePending
+  };
+}
+__name(stalledInfo, "stalledInfo");
+
 // src/transport-pool.ts
 var TransportPool = class {
   static {
@@ -3240,11 +3369,6 @@ function normalizeChildOptions(options) {
   } : options ?? {};
 }
 __name(normalizeChildOptions, "normalizeChildOptions");
-function formatIssuePath(path) {
-  if (!path || path.length === 0) return "(root)";
-  return path.map((segment) => String(typeof segment === "object" ? segment.key : segment)).join(".");
-}
-__name(formatIssuePath, "formatIssuePath");
 function resolveStepPolicy(config, opts) {
   const policy = {};
   const retries = opts?.retries ?? config?.retries;
@@ -4076,11 +4200,7 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
         throw new Error(`workflow "${workflow}": ctx.upsertSearchAttributes schema validation must be synchronous (the searchAttributes schema's \`~standard.validate\` returned a Promise) \u2014 replay-determinism requires a sync validator; drop any async refinement from the schema passed to @Workflow({ searchAttributes }).`);
       }
       if (result.issues) {
-        const offendingKeys = [
-          ...new Set(result.issues.map((issue) => formatIssuePath(issue.path)))
-        ];
-        const details = result.issues.map((issue) => `${formatIssuePath(issue.path)}: ${issue.message}`).join("; ");
-        throw new Error(`workflow "${workflow}": ctx.upsertSearchAttributes rejected by its searchAttributes schema \u2014 offending key(s) ${offendingKeys.join(", ")} \u2014 ${details}`);
+        throw new SchemaValidationError(`workflow "${workflow}" ctx.upsertSearchAttributes`, result.issues);
       }
     }
     await host.upsertSearchAttributes(runId, attrs);
@@ -4374,6 +4494,18 @@ var WorkflowEngine = class {
   stepQueue = /* @__PURE__ */ new Map();
   /** Executions currently in flight, so a graceful shutdown can wait for them to settle. */
   inflight = /* @__PURE__ */ new Set();
+  /**
+  * The run lease of each execution in flight on THIS instance, by run id — renewed while it runs
+  * and checked by every settle (the zombie fence, see {@link ExecutionLease}). Also what `drain()`
+  * releases when its budget runs out.
+  */
+  executionLeases = /* @__PURE__ */ new Map();
+  /** Pager hooks for stranded runs (see {@link onStalled}). */
+  stalledListeners = /* @__PURE__ */ new Set();
+  /** Runs already paged about, by id → the `updatedAt` of the paged episode (see {@link sweepStalled}). */
+  notifiedStalled = /* @__PURE__ */ new Map();
+  lastStalledSweepAt = Number.NEGATIVE_INFINITY;
+  stalledAfterMs;
   draining = false;
   /** Wait already spent on this shutdown before `drain()` (see `beginDrain`). */
   drainSpentMs = 0;
@@ -4398,6 +4530,8 @@ var WorkflowEngine = class {
     this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
     this.remoteAdvanceSilenceMs = deps.remoteAdvanceSilenceMs;
     this.reconcileMs = deps.reconcileMs === void 0 ? 3e5 : deps.reconcileMs || void 0;
+    const stalledAfter = deps.stalledAfterMs;
+    this.stalledAfterMs = stalledAfter !== void 0 && Number.isFinite(stalledAfter) ? Math.max(6e4, stalledAfter) : DEFAULT_STALLED_AFTER_MS;
     const batch = deps.timerBatchSize;
     this.timerBatchSize = batch !== void 0 && Number.isFinite(batch) && batch >= 1 ? Math.floor(batch) : 500;
     this.eventBufferTtlMs = deps.eventBufferTtlMs || void 0;
@@ -4741,6 +4875,64 @@ var WorkflowEngine = class {
     this.deadListeners.add(listener);
     return () => this.deadListeners.delete(listener);
   }
+  /**
+  * Register a pager hook for STRANDED runs: an in-flight run untouched past `stalledAfterMs` that is
+  * either suspended with NO wake timer (wake-forever: nothing will ever re-drive it) or parked on an
+  * old pending remote step with no live lease (dispatched and, most likely, lost). The timer poller
+  * sweeps for these ({@link sweepStalled}, throttled to one pass a minute) and notifies each episode
+  * ONCE per instance: a run is paged again only after it makes progress and strands again. Wire your
+  * alerting here like {@link onDead}. A listener that throws never breaks the sweep. Returns an
+  * unsubscribe function.
+  */
+  onStalled(listener) {
+    this.stalledListeners.add(listener);
+    return () => this.stalledListeners.delete(listener);
+  }
+  /**
+  * Detect stranded in-flight runs and notify the {@link onStalled} listeners. Self-throttled to one
+  * pass a minute, skipped while draining, and a no-op without listeners — call it every tick (the
+  * NestJS timer poller does). Scoped to this engine's `namespace` (an operator engine with no
+  * namespace sweeps every namespace). Returns the runs notified THIS pass.
+  */
+  async sweepStalled(now = this.clock()) {
+    if (this.stalledListeners.size === 0 || this.draining) return [];
+    if (now - this.lastStalledSweepAt < 6e4) return [];
+    this.lastStalledSweepAt = now;
+    const notified = [];
+    for (const info2 of await this.listStalledRuns({
+      now
+    })) {
+      const episode = info2.run.updatedAt.getTime();
+      if (this.notifiedStalled.get(info2.run.id) === episode) continue;
+      this.notifiedStalled.set(info2.run.id, episode);
+      if (this.notifiedStalled.size > 1e3) this.notifiedStalled.clear();
+      for (const listener of this.stalledListeners) {
+        try {
+          listener(info2);
+        } catch {
+        }
+      }
+      notified.push(info2);
+    }
+    return notified;
+  }
+  /**
+  * The stranded runs right now, without notifying anyone: the query behind {@link sweepStalled} and
+  * the CLI's `durable inspect --stale`. A `running`/`suspended` run qualifies when it has been
+  * untouched for `thresholdMs` (default `stalledAfterMs`) AND either it is suspended with no wake
+  * timer, or its oldest pending remote step is older than the threshold with no lease in the future
+  * (a step with a `timeoutMs`, a heartbeat or `remoteRedispatchMs` keeps a lease; a long
+  * `ctx.sleep` has a future `wakeAt`; neither is stranded). Scans up to `scan` candidates (default
+  * 1000), newest first, in this engine's namespace.
+  */
+  async listStalledRuns(options = {}) {
+    return findStalledRuns(this.store, {
+      now: options.now ?? this.clock(),
+      thresholdMs: options.thresholdMs ?? this.stalledAfterMs,
+      scan: options.scan,
+      namespace: this.namespace
+    });
+  }
   notifyDead(run) {
     for (const fn of this.deadListeners) {
       try {
@@ -4899,6 +5091,11 @@ var WorkflowEngine = class {
   getRun(runId) {
     return this.store.getRun(runId);
   }
+  /** A run's full checkpoint timeline, ordered by seq. A thin pass-through to the store (it is what
+  *  the testing kit's `captureHistory` reads, next to {@link getRun}). */
+  listCheckpoints(runId) {
+    return this.store.listCheckpoints(runId);
+  }
   /**
   * Re-execute a run's body (replaying its checkpoints). This is the EXPLICIT entry point: besides
   * driving a live run, it re-drives a `failed` run — the manual retry (prefer {@link requeue}, which
@@ -4984,6 +5181,8 @@ var WorkflowEngine = class {
   * diagnosis.
   */
   async releaseLeaseBestEffort(runId) {
+    if (this.isFenced(runId)) return;
+    this.handOffLease(runId);
     try {
       await this.store.releaseRunLock(runId);
     } catch (err) {
@@ -5207,6 +5406,10 @@ var WorkflowEngine = class {
       for (const rearm of this.drainRearms) rearm();
     }
   }
+  /** True once {@link beginDrain} (or {@link drain}) has been called. */
+  isDraining() {
+    return this.draining;
+  }
   /**
   * Graceful shutdown: stop picking up new runs (recovery/timer become no-ops) and wait for
   * in-flight executions to settle, up to `timeoutMs`. Call from your app's shutdown hook so a
@@ -5233,17 +5436,47 @@ var WorkflowEngine = class {
       rearm();
     });
     this.drainRearms.add(rearm);
+    let settled = false;
     try {
       await Promise.race([
         Promise.allSettled([
           ...this.inflight
-        ]),
+        ]).then(() => {
+          settled = true;
+        }),
         timer
       ]);
     } finally {
       this.drainRearms.delete(rearm);
       clearTimeout(timeout);
     }
+    if (!settled) await this.releaseInflightLeases();
+  }
+  /**
+  * The drain budget ran out with executions still in flight: release their recovery leases so
+  * another instance re-drives those runs on its next recovery pass, instead of waiting out
+  * `leaseMs` after this process exits. Each still-running turn is FENCED first, so whatever it
+  * computes after this point is never written over the new owner's work (see settleRun). Its
+  * completed steps are checkpointed, so the new owner replays them; a step still executing here
+  * may run again there, exactly as after a crash. The release is owner-scoped where the store
+  * supports it (only a lease this instance still holds is cleared). Best effort: never throws.
+  */
+  async releaseInflightLeases() {
+    const released = [];
+    for (const [runId, lease] of [
+      ...this.executionLeases
+    ]) {
+      if (lease.lost || lease.handedOff || lease.done) continue;
+      lease.lost = true;
+      try {
+        await this.store.releaseRunLock(runId, this.instanceId);
+        released.push(runId);
+      } catch {
+      }
+    }
+    if (released.length > 0) {
+      console.warn(`[nestjs-durable] drain timed out with ${released.length} run(s) still executing; released their leases so another instance recovers them now: ${released.join(", ")}`);
+    }
   }
   /**
   * Cancel in-flight runs that have outlived their workflow's `executionTimeout`. Call it from the
@@ -6733,7 +6966,10 @@ var WorkflowEngine = class {
   }
   async singletonWaitResult(run) {
     const current = await this.store.getRun(run.id);
-    await this.store.releaseRunLock(run.id);
+    if (!this.isFenced(run.id)) {
+      this.handOffLease(run.id);
+      await this.store.releaseRunLock(run.id);
+    }
     if (!current) throw new Error(`run ${run.id} was deleted during singleton admission`);
     if (current.status === "suspended") {
       this.emit({
@@ -6761,10 +6997,7 @@ var WorkflowEngine = class {
         };
       }
     }
-    const renew = setInterval(() => {
-      void this.store.renewRunLock(run.id, this.instanceId, this.clock() + this.leaseMs).catch(() => void 0);
-    }, Math.max(50, Math.floor(this.leaseMs / 2)));
-    renew.unref?.();
+    const held = this.holdExecutionLease(run.id);
     try {
       const result = registered?.remote ? await this.runRemoteExecution(run, registered) : await this.runExecution(run, fn, registered);
       if (registered?.singleton && (result.status === "completed" || result.status === "failed" || result.status === "cancelled" || result.status === "dead")) {
@@ -6772,10 +7005,59 @@ var WorkflowEngine = class {
       }
       return result;
     } finally {
-      clearInterval(renew);
+      held.stop();
     }
   }
   /**
+  * Renew `runId`'s lease for as long as this execution holds it, and register it in
+  * {@link executionLeases} so settles can check the zombie fence and `drain()` can release it.
+  *
+  * A renewal that returns `false` means the lease is gone: this turn stalled past `leaseMs` and
+  * another instance reclaimed the run, so this executor is now a zombie. It is FENCED (`lost`): its
+  * settles degrade to an echo of the store's state instead of clobbering the new owner's writes, and
+  * its own lease release is skipped (it would free the new owner's lease). A renewal that THROWS (a
+  * transient store error) is not evidence of a takeover, so it stays swallowed and the next
+  * interval retries. A lease this execution released on purpose (`handedOff`) is not renewed, and a
+  * renewal resolving after the execution ended (`done`) fences nothing.
+  */
+  holdExecutionLease(runId) {
+    const lease = {
+      lost: false,
+      handedOff: false,
+      done: false
+    };
+    this.executionLeases.set(runId, lease);
+    const renew = setInterval(() => {
+      if (lease.handedOff || lease.lost || lease.done) return;
+      void this.store.renewRunLock(runId, this.instanceId, this.clock() + this.leaseMs).then((renewed) => {
+        if (renewed || lease.handedOff || lease.lost || lease.done) return;
+        lease.lost = true;
+        console.warn(`[nestjs-durable] lost the lease of run ${runId} while executing it (another instance took it over after it expired); this instance will not write the run's outcome`);
+      }).catch(() => void 0);
+    }, Math.max(50, Math.floor(this.leaseMs / 2)));
+    renew.unref?.();
+    return {
+      lease,
+      stop: /* @__PURE__ */ __name(() => {
+        clearInterval(renew);
+        lease.done = true;
+        if (this.executionLeases.get(runId) === lease) this.executionLeases.delete(runId);
+      }, "stop")
+    };
+  }
+  /** Whether `runId`'s execution on this instance lost its lease (see {@link holdExecutionLease}). */
+  isFenced(runId) {
+    return this.executionLeases.get(runId)?.lost === true;
+  }
+  /**
+  * Mark `runId`'s lease as given away on purpose by its own execution (released mid-turn), so a
+  * renewal that now fails is not read as a takeover.
+  */
+  handOffLease(runId) {
+    const lease = this.executionLeases.get(runId);
+    if (lease) lease.handedOff = true;
+  }
+  /**
   * Advance a remote (cross-SDK) workflow one turn: hand its history to the executor (which dispatches
   * a workflow task to the worker and awaits its replay) and apply the decision. Mirrors
   * {@link runExecution}'s settle/suspend; the lease is held by {@link execute}. The result that lands
@@ -6992,6 +7274,7 @@ ${run.workflowVersion}`;
   * keeps its own lease handling (the TS executor's `finally`, the remote path's caller).
   */
   async settleRun(run, outcome) {
+    if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
     const updatedAt = /* @__PURE__ */ new Date();
     if (outcome.kind === "completed") {
       const applied2 = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SETTLE_ELIGIBLE_STATUSES, {
@@ -7072,6 +7355,7 @@ ${run.workflowVersion}`;
   }
   async runRemoteExecution(run, registered) {
     if (run.status === "cancelling") {
+      if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
       const error2 = {
         message: "cancelled"
       };
@@ -7080,6 +7364,7 @@ ${run.workflowVersion}`;
         error: error2,
         updatedAt: /* @__PURE__ */ new Date()
       });
+      if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
       this.emit({
         type: "run.failed",
         runId: run.id,
@@ -7126,6 +7411,7 @@ ${run.workflowVersion}`;
     }
     const history = await this.remoteHistory(run.id, checkpoints);
     if (remote.executor.dispatch) {
+      if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
       const taskId = `${run.id}:wf:${globalThis.crypto.randomUUID().slice(0, 8)}`;
       const silenceMs = this.remoteAdvanceSilenceMs;
       const wakeAt = silenceMs != null ? this.clock() + silenceMs : void 0;
@@ -7141,6 +7427,7 @@ ${run.workflowVersion}`;
         workflow: run.workflow,
         namespace: run.namespace
       });
+      this.handOffLease(run.id);
       await this.store.releaseRunLock(run.id);
       await remote.executor.dispatch(run, history, taskId);
       return {
@@ -7169,6 +7456,7 @@ ${run.workflowVersion}`;
         message: err instanceof Error ? err.message : String(err),
         stack: err instanceof Error ? err.stack : void 0
       };
+      if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
       await this.store.updateRun(run.id, {
         status: "failed",
         error: error2,
@@ -7197,6 +7485,7 @@ ${run.workflowVersion}`;
   * — the caller has already ensured this is the decision for the currently-awaited turn.
   */
   async applyDecision(run, decision) {
+    if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
     if (decision.status === "completed") {
       await this.applyCommands(run, decision.commands);
       return this.settleRun(run, {
@@ -7666,6 +7955,7 @@ ${run.workflowVersion}`;
   * cancel flag stays too, re-derived from the status on any later re-drive regardless.
   */
   async compensateAndCancel(run, compensations, replay) {
+    if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
     const pending = await this.unwindCompensations(run, compensations, replay);
     if (pending) return {
       runId: run.id,
@@ -7757,6 +8047,7 @@ ${run.workflowVersion}`;
       });
     } catch (err) {
       if (err instanceof ContinueAsNew) {
+        if (this.isFenced(run.id)) return await this.echoCurrentStatus(run.id);
         if (this.cancelRequested.has(run.id)) {
           return await this.compensateAndCancel(run, compensations, replay);
         }
@@ -8208,6 +8499,7 @@ ${run.workflowVersion}`;
   * own `finally` also releases (the run-execution path) is safe.
   */
   async parkBlocked(run, blocked) {
+    if (this.isFenced(run.id)) return this.echoCurrentStatus(run.id);
     const error2 = {
       message: blocked.reason,
       code: blocked.code,
@@ -8794,10 +9086,7 @@ ${run.workflowVersion}`;
     if (!await this.store.tryLockRun(decision.runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
       return;
     }
-    const renew = setInterval(() => {
-      void this.store.renewRunLock(decision.runId, this.instanceId, this.clock() + this.leaseMs).catch(() => void 0);
-    }, Math.max(50, Math.floor(this.leaseMs / 2)));
-    renew.unref?.();
+    const held = this.holdExecutionLease(decision.runId);
     try {
       const fresh = await this.store.getRun(decision.runId);
       if (!fresh || fresh.awaitingDecisionTaskId !== decision.taskId) return;
@@ -8812,8 +9101,9 @@ ${run.workflowVersion}`;
       fresh.awaitingDecisionTaskId = void 0;
       await this.applyDecision(fresh, decision);
     } finally {
-      clearInterval(renew);
-      await this.store.releaseRunLock(decision.runId);
+      const fenced = held.lease.lost;
+      held.stop();
+      if (!fenced) await this.store.releaseRunLock(decision.runId);
     }
   }
   /** Earliest clock time a suspended turn's deadline is worth LOOKING at again (`runId` → clock ms) —
@@ -9009,6 +9299,9 @@ function childIdsOf(checkpoints) {
 }
 __name(childIdsOf, "childIdsOf");
 
+// src/run-history.ts
+var RUN_HISTORY_FIXTURE_VERSION = 1;
+
 // src/run-facets.ts
 function facetOrigin(origin) {
   return origin?.trim() ? origin : null;
@@ -9562,13 +9855,14 @@ var CodecStateStore = class {
   }
   inner;
   codec;
+  extended;
   /** Atomic claims touch only searchable metadata and bypass payload encoding. */
   tryAdmitSingleton;
   /**
   * Forwarded verbatim — a facet cell counts `status`/`origin`, neither of which the codec touches
-  * (it encodes payloads: input/output/error). Bound as a PROPERTY, and only when the inner store
-  * actually aggregates, so `store.runFacets?.()` still reports absence through the wrapper instead
-  * of resolving to a silently wrong empty result.
+  * (it encodes payloads: input/output, plus events/error under `extended` coverage). Bound as a
+  * PROPERTY, and only when the inner store actually aggregates, so `store.runFacets?.()` still
+  * reports absence through the wrapper instead of resolving to a silently wrong empty result.
   */
   runFacets;
   /**
@@ -9620,9 +9914,10 @@ var CodecStateStore = class {
   updateSchedule;
   deleteSchedule;
   listSchedules;
-  constructor(inner, codec) {
+  constructor(inner, codec, options = {}) {
     this.inner = inner;
     this.codec = codec;
+    this.extended = options.coverage === "extended";
     if (inner.tryAdmitSingleton) this.tryAdmitSingleton = inner.tryAdmitSingleton.bind(inner);
     const deleteRuns = inner.deleteRuns;
     if (deleteRuns) this.deleteRuns = (ids) => deleteRuns.call(inner, ids);
@@ -9680,32 +9975,52 @@ var CodecStateStore = class {
     return v === void 0 ? void 0 : this.codec.decode(v);
   }
   encRun(run) {
-    return {
+    const next = {
       ...run,
       input: this.enc(run.input),
       output: this.enc(run.output)
     };
+    if (this.extended && run.error !== void 0) {
+      next.error = this.enc(run.error);
+    }
+    return next;
   }
   decRun(run) {
-    return run && {
+    if (!run) return run;
+    const next = {
       ...run,
       input: this.dec(run.input),
       output: this.dec(run.output)
     };
+    if (this.extended && run.error !== void 0) {
+      next.error = this.dec(run.error);
+    }
+    return next;
   }
   encCp(cp) {
-    return {
+    const next = {
       ...cp,
       input: this.enc(cp.input),
       output: this.enc(cp.output)
     };
+    if (this.extended) {
+      if (cp.events !== void 0) next.events = this.enc(cp.events);
+      if (cp.error !== void 0) next.error = this.enc(cp.error);
+    }
+    return next;
   }
   decCp(cp) {
-    return cp && {
+    if (!cp) return cp;
+    const next = {
       ...cp,
       input: this.dec(cp.input),
       output: this.dec(cp.output)
     };
+    if (this.extended) {
+      if (cp.events !== void 0) next.events = this.dec(cp.events);
+      if (cp.error !== void 0) next.error = this.dec(cp.error);
+    }
+    return next;
   }
   encSpec(spec) {
     return "input" in spec ? {
@@ -9744,6 +10059,9 @@ var CodecStateStore = class {
     };
     if ("input" in patch) next.input = this.enc(patch.input);
     if ("output" in patch) next.output = this.enc(patch.output);
+    if (this.extended && patch.error !== void 0) {
+      next.error = this.enc(patch.error);
+    }
     return next;
   }
   async getRun(runId) {
@@ -9779,8 +10097,8 @@ var CodecStateStore = class {
   tryLockRun(runId, owner, leaseUntilMs, nowMs) {
     return this.inner.tryLockRun(runId, owner, leaseUntilMs, nowMs);
   }
-  releaseRunLock(runId) {
-    return this.inner.releaseRunLock(runId);
+  releaseRunLock(runId, owner) {
+    return owner === void 0 ? this.inner.releaseRunLock(runId) : this.inner.releaseRunLock(runId, owner);
   }
   renewRunLock(runId, owner, leaseUntilMs) {
     return this.inner.renewRunLock(runId, owner, leaseUntilMs);
@@ -10288,9 +10606,9 @@ var InMemoryStateStore = class {
     run.lockedUntil = leaseUntilMs;
     return true;
   }
-  async releaseRunLock(runId) {
+  async releaseRunLock(runId, owner) {
     const run = this.runs.get(runId);
-    if (run) {
+    if (run && (owner === void 0 || run.lockedBy === owner)) {
       run.lockedBy = void 0;
       run.lockedUntil = void 0;
     }
@@ -10641,6 +10959,7 @@ var InMemoryTransport = class {
   ContinueAsNew,
   DEFAULT_CONCURRENCY_STATUSES,
   DEFAULT_PARTITION,
+  DEFAULT_STALLED_AFTER_MS,
   DURABLE_OPTIONS,
   DURABLE_OPTIONS_CANONICAL,
   DURABLE_STEP_CONFIG,
@@ -10664,6 +10983,7 @@ var InMemoryTransport = class {
   NestedWorkflowCallError,
   NonDeterminismError,
   QueueController,
+  RUN_HISTORY_FIXTURE_VERSION,
   RUN_VALUE_FACET_LIMIT,
   RUN_VALUE_FACET_SCAN,
   RemoteStepError,
@@ -10681,6 +11001,7 @@ var InMemoryTransport = class {
   ScheduleAlreadyExistsError,
   ScheduleClient,
   ScheduleNotFoundError,
+  SchemaValidationError,
   SignalTimeoutError,
   SingletonQueueFullError,
   TERMINAL_RUN_STATUSES,
@@ -10737,6 +11058,7 @@ var InMemoryTransport = class {
   facetOrigin,
   fanPageBounds,
   fanPageEtag,
+  findStalledRuns,
   foldedCount,
   handoffHandlerId,
   handoffName,
@@ -10791,6 +11113,7 @@ var InMemoryTransport = class {
   runStepHandler,
   runValueFacetsFromRuns,
   sanitizeQueueToken,
+  scheduleRunIdAt,
   scheduledRunId,
   singletonAdmissionDenialPatch,
   singletonAdmissionTags,