@workflow/world-testing 4.1.0-beta.70 → 4.1.0-beta.71

package/dist/.well-known/workflow/v1/flow.js

@@ -49437,6 +49437,19 @@ async function deleteJSON(filePath) {
   }
 }
 __name(deleteJSON, "deleteJSON");
+async function writeExclusive(filePath, data) {
+  await ensureDir(import_node_path2.default.dirname(filePath));
+  try {
+    await import_node_fs.promises.writeFile(filePath, data, { flag: "wx" });
+    return true;
+  } catch (error48) {
+    if (error48.code === "EEXIST") {
+      return false;
+    }
+    throw error48;
+  }
+}
+__name(writeExclusive, "writeExclusive");
 async function listJSONFiles(dirPath) {
   return listFilesByExtension(dirPath, ".json");
 }
@@ -49590,6 +49603,11 @@ function filterHookData(hook, resolveData) {
 __name(filterHookData, "filterHookData");
 
 // ../world-local/dist/storage/helpers.js
+var import_node_crypto2 = require("node:crypto");
+function hashToken(token) {
+  return (0, import_node_crypto2.createHash)("sha256").update(token).digest("hex");
+}
+__name(hashToken, "hashToken");
 var monotonicUlid = monotonicFactory(() => Math.random());
 var getObjectCreatedAt = /* @__PURE__ */ __name((idPrefix) => (filename) => {
   const replaceRegex = new RegExp(`^${idPrefix}_`, "g");
@@ -49678,6 +49696,8 @@ async function deleteAllHooksForRun(basedir, runId) {
     const hookPath = import_node_path3.default.join(hooksDir, `${file2}.json`);
     const hook = await readJSON(hookPath, HookSchema);
     if (hook && hook.runId === runId) {
+      const constraintPath = import_node_path3.default.join(hooksDir, "tokens", `${hashToken(hook.token)}.json`);
+      await deleteJSON(constraintPath);
       await deleteJSON(hookPath);
     }
   }
@@ -50086,18 +50106,13 @@ function createEventsStorage(basedir) {
         data.eventType === "hook_created" && "eventData" in data
       ) {
         const hookData = data.eventData;
-        const hooksDir = import_node_path5.default.join(basedir, "hooks");
-        const hookFiles = await listJSONFiles(hooksDir);
-        let hasConflict = false;
-        for (const file2 of hookFiles) {
-          const existingHookPath = import_node_path5.default.join(hooksDir, `${file2}.json`);
-          const existingHook = await readJSON(existingHookPath, HookSchema);
-          if (existingHook && existingHook.token === hookData.token) {
-            hasConflict = true;
-            break;
-          }
-        }
-        if (hasConflict) {
+        const constraintPath = import_node_path5.default.join(basedir, "hooks", "tokens", `${hashToken(hookData.token)}.json`);
+        const tokenClaimed = await writeExclusive(constraintPath, JSON.stringify({
+          token: hookData.token,
+          hookId: data.correlationId,
+          runId: effectiveRunId
+        }));
+        if (!tokenClaimed) {
           const conflictEvent = {
             eventType: "hook_conflict",
             correlationId: data.correlationId,
@@ -50138,6 +50153,11 @@ function createEventsStorage(basedir) {
         await writeJSON(hookPath, hook);
       } else if (data.eventType === "hook_disposed") {
         const hookPath = import_node_path5.default.join(basedir, "hooks", `${data.correlationId}.json`);
+        const existingHook = await readJSON(hookPath, HookSchema);
+        if (existingHook) {
+          const disposedConstraintPath = import_node_path5.default.join(basedir, "hooks", "tokens", `${hashToken(existingHook.token)}.json`);
+          await deleteJSON(disposedConstraintPath);
+        }
         await deleteJSON(hookPath);
       } else if (data.eventType === "wait_created" && "eventData" in data) {
         const waitData = data.eventData;
@@ -50602,7 +50622,7 @@ function createLocalWorld(args) {
 __name(createLocalWorld, "createLocalWorld");
 
 // ../world-vercel/dist/encryption.js
-var import_node_crypto2 = require("node:crypto");
+var import_node_crypto3 = require("node:crypto");
 var import_oidc2 = __toESM(require_dist2(), 1);
 
 // ../world-vercel/dist/http-client.js
@@ -50636,9 +50656,9 @@ async function deriveRunKey(deploymentKey, projectId, runId) {
   if (!projectId || typeof projectId !== "string") {
     throw new Error("projectId must be a non-empty string");
   }
-  const baseKey = await import_node_crypto2.webcrypto.subtle.importKey("raw", deploymentKey, "HKDF", false, ["deriveBits"]);
+  const baseKey = await import_node_crypto3.webcrypto.subtle.importKey("raw", deploymentKey, "HKDF", false, ["deriveBits"]);
   const info = new TextEncoder().encode(`${projectId}|${runId}`);
-  const derivedBits = await import_node_crypto2.webcrypto.subtle.deriveBits(
+  const derivedBits = await import_node_crypto3.webcrypto.subtle.deriveBits(
     {
       name: "HKDF",
       hash: "SHA-256",
@@ -54381,7 +54401,7 @@ var createWorld = /* @__PURE__ */ __name(() => {
       "WORKFLOW_VERCEL_AUTH_TOKEN",
       "WORKFLOW_VERCEL_ENV"
     ].filter((key) => process.env[key]);
-    if (staleEnvVars.length > 0) {
+    if (staleEnvVars.length > 0 && process.env.VERCEL === "1") {
       console.warn(`[workflow] Warning: ${staleEnvVars.join(", ")} env var(s) are set but have no effect at runtime. These are only used by the Workflow CLI. Remove them from your Vercel project environment variables.`);
     }
     return createVercelWorld();
@@ -54552,69 +54572,6 @@ function getQueueOverhead(message) {
   }
 }
 __name(getQueueOverhead, "getQueueOverhead");
-async function withThrottleRetry(fn) {
-  try {
-    return await fn();
-  } catch (err) {
-    if (WorkflowAPIError.is(err) && err.status === 429) {
-      const retryAfterSeconds = Math.max(
-        // If we don't have a retry-after value, 30s seems a reasonable default
-        // to avoid re-trying during the unknown rate-limiting period.
-        1,
-        typeof err.retryAfter === "number" ? err.retryAfter : 30
-      );
-      if (retryAfterSeconds < 10) {
-        runtimeLogger.warn("Throttled by workflow-server (429), retrying in-process", {
-          retryAfterSeconds,
-          url: err.url
-        });
-        await new Promise((resolve3) => setTimeout(resolve3, retryAfterSeconds * 1e3));
-        try {
-          return await fn();
-        } catch (retryErr) {
-          if (WorkflowAPIError.is(retryErr) && retryErr.status === 429) {
-            const retryRetryAfter = Math.max(1, typeof retryErr.retryAfter === "number" ? retryErr.retryAfter : 1);
-            runtimeLogger.warn("Throttled again on retry, deferring to queue", {
-              retryAfterSeconds: retryRetryAfter
-            });
-            return { timeoutSeconds: retryRetryAfter };
-          }
-          throw retryErr;
-        }
-      }
-      runtimeLogger.warn("Throttled by workflow-server (429), deferring to queue", {
-        retryAfterSeconds,
-        url: err.url
-      });
-      return { timeoutSeconds: retryAfterSeconds };
-    }
-    throw err;
-  }
-}
-__name(withThrottleRetry, "withThrottleRetry");
-async function withServerErrorRetry(fn) {
-  const delays = [500, 1e3, 2e3];
-  for (let attempt = 0; attempt <= delays.length; attempt++) {
-    try {
-      return await fn();
-    } catch (err) {
-      if (WorkflowAPIError.is(err) && err.status !== void 0 && err.status >= 500 && attempt < delays.length) {
-        runtimeLogger.warn("Server error (5xx) from workflow-server, retrying in-process", {
-          status: err.status,
-          attempt: attempt + 1,
-          maxRetries: delays.length,
-          nextDelayMs: delays[attempt],
-          url: err.url
-        });
-        await new Promise((resolve3) => setTimeout(resolve3, delays[attempt]));
-        continue;
-      }
-      throw err;
-    }
-  }
-  throw new Error("withServerErrorRetry: unreachable");
-}
-__name(withServerErrorRetry, "withServerErrorRetry");
 
 // ../core/dist/runtime/suspension-handler.js
 var import_functions = __toESM(require_functions(), 1);
@@ -58257,18 +58214,18 @@ var stepHandler = getWorldHandlers().createQueueHandler("__wkf_step_", async (me
       });
       let step;
       try {
-        const startResult = await withServerErrorRetry(() => world.events.create(workflowRunId, {
+        const startResult = await world.events.create(workflowRunId, {
           eventType: "step_started",
           specVersion: SPEC_VERSION_CURRENT,
           correlationId: stepId
-        }));
+        });
         if (!startResult.step) {
           throw new WorkflowRuntimeError(`step_started event for "${stepId}" did not return step entity`);
         }
         step = startResult.step;
       } catch (err) {
         if (WorkflowAPIError.is(err)) {
-          if (WorkflowAPIError.is(err) && err.status === 429) {
+          if (err.status === 429) {
             const retryRetryAfter = Math.max(1, typeof err.retryAfter === "number" ? err.retryAfter : 1);
             runtimeLogger.warn("Throttled again on retry, deferring to queue", {
               retryAfterSeconds: retryRetryAfter
@@ -58377,28 +58334,57 @@ var stepHandler = getWorldHandlers().createQueueHandler("__wkf_step_", async (me
         });
         return;
       }
-      try {
-        const attempt = step.attempt;
-        if (!step.startedAt) {
-          throw new WorkflowRuntimeError(`Step "${stepId}" has no "startedAt" timestamp`);
-        }
-        const stepStartedAt = step.startedAt;
-        const ops = [];
-        const rawKey = await world.getEncryptionKeyForRun?.(workflowRunId);
-        const encryptionKey = rawKey ? await importKey(rawKey) : void 0;
-        const hydratedInput = await trace2("step.hydrate", {}, async (hydrateSpan) => {
-          const startTime = Date.now();
-          const result2 = await hydrateStepArguments(step.input, workflowRunId, encryptionKey, ops);
-          const durationMs = Date.now() - startTime;
-          hydrateSpan?.setAttributes({
-            ...StepArgumentsCount(result2.args.length),
-            ...QueueDeserializeTimeMs(durationMs)
+      const attempt = step.attempt;
+      if (!step.startedAt) {
+        const errorMessage = `Step "${stepId}" has no "startedAt" timestamp`;
+        runtimeLogger.error("Fatal runtime error during step setup", {
+          workflowRunId,
+          stepId,
+          error: errorMessage
+        });
+        try {
+          await world.events.create(workflowRunId, {
+            eventType: "step_failed",
+            specVersion: SPEC_VERSION_CURRENT,
+            correlationId: stepId,
+            eventData: {
+              error: errorMessage,
+              stack: new Error(errorMessage).stack ?? ""
+            }
           });
-          return result2;
+        } catch (failErr) {
+          if (WorkflowAPIError.is(failErr) && failErr.status === 409) {
+            return;
+          }
+          throw failErr;
+        }
+        await queueMessage(world, getWorkflowQueueName(workflowName), {
+          runId: workflowRunId,
+          traceCarrier: await serializeTraceCarrier(),
+          requestedAt: /* @__PURE__ */ new Date()
+        });
+        return;
+      }
+      const stepStartedAt = step.startedAt;
+      const ops = [];
+      const rawKey = await world.getEncryptionKeyForRun?.(workflowRunId);
+      const encryptionKey = rawKey ? await importKey(rawKey) : void 0;
+      const hydratedInput = await trace2("step.hydrate", {}, async (hydrateSpan) => {
+        const startTime = Date.now();
+        const result2 = await hydrateStepArguments(step.input, workflowRunId, encryptionKey, ops);
+        const durationMs = Date.now() - startTime;
+        hydrateSpan?.setAttributes({
+          ...StepArgumentsCount(result2.args.length),
+          ...QueueDeserializeTimeMs(durationMs)
         });
-        const args = hydratedInput.args;
-        const thisVal = hydratedInput.thisVal ?? null;
-        const executionStartTime = Date.now();
+        return result2;
+      });
+      const args = hydratedInput.args;
+      const thisVal = hydratedInput.thisVal ?? null;
+      let userCodeError;
+      let userCodeFailed = false;
+      const executionStartTime = Date.now();
+      try {
         result = await trace2("step.execute", {}, async () => {
           return await contextStorage.run({
             stepMetadata: {
@@ -58420,63 +58406,29 @@ var stepHandler = getWorldHandlers().createQueueHandler("__wkf_step_", async (me
             encryptionKey
           }, () => stepFn.apply(thisVal, args));
         });
-        const executionTimeMs = Date.now() - executionStartTime;
-        span?.setAttributes({
-          ...QueueExecutionTimeMs(executionTimeMs)
-        });
-        result = await trace2("step.dehydrate", {}, async (dehydrateSpan) => {
-          const startTime = Date.now();
-          const dehydrated = await dehydrateStepReturnValue(result, workflowRunId, encryptionKey, ops);
-          const durationMs = Date.now() - startTime;
-          dehydrateSpan?.setAttributes({
-            ...QueueSerializeTimeMs(durationMs),
-            ...StepResultType(typeof dehydrated)
-          });
-          return dehydrated;
-        });
-        (0, import_functions5.waitUntil)(Promise.all(ops).catch((err) => {
-          const isAbortError = err?.name === "AbortError" || err?.name === "ResponseAborted";
-          if (!isAbortError)
-            throw err;
-        }));
-        let stepCompleted409 = false;
-        const [, traceCarrier] = await Promise.all([
-          withServerErrorRetry(() => world.events.create(workflowRunId, {
-            eventType: "step_completed",
-            specVersion: SPEC_VERSION_CURRENT,
-            correlationId: stepId,
-            eventData: {
-              result
-            }
-          })).catch((err) => {
-            if (WorkflowAPIError.is(err) && err.status === 409) {
-              runtimeLogger.warn("Tried completing step, but step has already finished.", {
-                workflowRunId,
-                stepId,
-                stepName,
-                message: err.message
-              });
-              stepCompleted409 = true;
-              return;
-            }
+      } catch (err) {
+        userCodeError = err;
+        userCodeFailed = true;
+      }
+      const executionTimeMs = Date.now() - executionStartTime;
+      span?.setAttributes({
+        ...QueueExecutionTimeMs(executionTimeMs)
+      });
+      if (userCodeFailed) {
+        const err = userCodeError;
+        if (WorkflowAPIError.is(err)) {
+          if (err.status === 410) {
+            stepLogger.info("Workflow run already completed, skipping step", {
+              workflowRunId,
+              stepId,
+              message: err.message
+            });
+            return;
+          }
+          if (err.status !== void 0 && err.status >= 500) {
             throw err;
-          }),
-          serializeTraceCarrier()
-        ]);
-        if (stepCompleted409) {
-          return;
+          }
         }
-        span?.setAttributes({
-          ...StepStatus("completed"),
-          ...StepResultType(typeof result)
-        });
-        await queueMessage(world, getWorkflowQueueName(workflowName), {
-          runId: workflowRunId,
-          traceCarrier,
-          requestedAt: /* @__PURE__ */ new Date()
-        });
-        return;
-      } catch (err) {
         const normalizedError = await normalizeUnknownError(err);
         const normalizedStack = normalizedError.stack || getErrorStack(err) || "";
         if (err instanceof Error) {
@@ -58492,26 +58444,6 @@ var stepHandler = getWorldHandlers().createQueueHandler("__wkf_step_", async (me
           ...ErrorCategory(errorCategory),
           ...ErrorRetryable(!isFatal)
         });
-        if (WorkflowAPIError.is(err)) {
-          if (err.status === 410) {
-            stepLogger.info("Workflow run already completed, skipping step", {
-              workflowRunId,
-              stepId,
-              message: err.message
-            });
-            return;
-          }
-          if (err.status !== void 0 && err.status >= 500) {
-            runtimeLogger.warn("Persistent server error (5xx) during step, deferring to queue retry", {
-              status: err.status,
-              workflowRunId,
-              stepId,
-              error: err.message,
-              url: err.url
-            });
-            throw err;
-          }
-        }
         if (isFatal) {
           stepLogger.error("Encountered FatalError while executing step, bubbling up to parent workflow", {
             workflowRunId,
@@ -58519,7 +58451,7 @@ var stepHandler = getWorldHandlers().createQueueHandler("__wkf_step_", async (me
             errorStack: normalizedStack
           });
           try {
-            await withServerErrorRetry(() => world.events.create(workflowRunId, {
+            await world.events.create(workflowRunId, {
               eventType: "step_failed",
               specVersion: SPEC_VERSION_CURRENT,
               correlationId: stepId,
@@ -58527,7 +58459,7 @@ var stepHandler = getWorldHandlers().createQueueHandler("__wkf_step_", async (me
                 error: normalizedError.message,
                 stack: normalizedStack
               }
-            }));
+            });
           } catch (stepFailErr) {
             if (WorkflowAPIError.is(stepFailErr) && stepFailErr.status === 409) {
               runtimeLogger.warn("Tried failing step, but step has already finished.", {
@@ -58562,7 +58494,7 @@ var stepHandler = getWorldHandlers().createQueueHandler("__wkf_step_", async (me
             });
             const errorMessage = `Step "${stepName}" failed after ${maxRetries2} ${pluralize("retry", "retries", maxRetries2)}: ${normalizedError.message}`;
             try {
-              await withServerErrorRetry(() => world.events.create(workflowRunId, {
+              await world.events.create(workflowRunId, {
                 eventType: "step_failed",
                 specVersion: SPEC_VERSION_CURRENT,
                 correlationId: stepId,
@@ -58570,7 +58502,7 @@ var stepHandler = getWorldHandlers().createQueueHandler("__wkf_step_", async (me
                   error: errorMessage,
                   stack: normalizedStack
                 }
-              }));
+              });
             } catch (stepFailErr) {
               if (WorkflowAPIError.is(stepFailErr) && stepFailErr.status === 409) {
                 runtimeLogger.warn("Tried failing step, but step has already finished.", {
@@ -58604,7 +58536,7 @@ var stepHandler = getWorldHandlers().createQueueHandler("__wkf_step_", async (me
               });
             }
             try {
-              await withServerErrorRetry(() => world.events.create(workflowRunId, {
+              await world.events.create(workflowRunId, {
                 eventType: "step_retrying",
                 specVersion: SPEC_VERSION_CURRENT,
                 correlationId: stepId,
@@ -58615,7 +58547,7 @@ var stepHandler = getWorldHandlers().createQueueHandler("__wkf_step_", async (me
                     retryAfter: err.retryAfter
                   }
                 }
-              }));
+              });
             } catch (stepRetryErr) {
               if (WorkflowAPIError.is(stepRetryErr) && stepRetryErr.status === 409) {
                 runtimeLogger.warn("Tried retrying step, but step has already finished.", {
@@ -58641,10 +58573,62 @@ var stepHandler = getWorldHandlers().createQueueHandler("__wkf_step_", async (me
             return { timeoutSeconds };
           }
         }
+        await queueMessage(world, getWorkflowQueueName(workflowName), {
+          runId: workflowRunId,
+          traceCarrier: await serializeTraceCarrier(),
+          requestedAt: /* @__PURE__ */ new Date()
+        });
+        return;
+      }
+      result = await trace2("step.dehydrate", {}, async (dehydrateSpan) => {
+        const startTime = Date.now();
+        const dehydrated = await dehydrateStepReturnValue(result, workflowRunId, encryptionKey, ops);
+        const durationMs = Date.now() - startTime;
+        dehydrateSpan?.setAttributes({
+          ...QueueSerializeTimeMs(durationMs),
+          ...StepResultType(typeof dehydrated)
+        });
+        return dehydrated;
+      });
+      (0, import_functions5.waitUntil)(Promise.all(ops).catch((err) => {
+        const isAbortError = err?.name === "AbortError" || err?.name === "ResponseAborted";
+        if (!isAbortError)
+          throw err;
+      }));
+      let stepCompleted409 = false;
+      const [, traceCarrier] = await Promise.all([
+        world.events.create(workflowRunId, {
+          eventType: "step_completed",
+          specVersion: SPEC_VERSION_CURRENT,
+          correlationId: stepId,
+          eventData: {
+            result
+          }
+        }).catch((err) => {
+          if (WorkflowAPIError.is(err) && err.status === 409) {
+            runtimeLogger.warn("Tried completing step, but step has already finished.", {
+              workflowRunId,
+              stepId,
+              stepName,
+              message: err.message
+            });
+            stepCompleted409 = true;
+            return;
+          }
+          throw err;
+        }),
+        serializeTraceCarrier()
+      ]);
+      if (stepCompleted409) {
+        return;
       }
+      span?.setAttributes({
+        ...StepStatus("completed"),
+        ...StepResultType(typeof result)
+      });
       await queueMessage(world, getWorkflowQueueName(workflowName), {
         runId: workflowRunId,
-        traceCarrier: await serializeTraceCarrier(),
+        traceCarrier,
         requestedAt: /* @__PURE__ */ new Date()
       });
     });
@@ -58659,7 +58643,7 @@ function workflowEntrypoint(workflowCode2) {
       await handleHealthCheckMessage(healthCheck2, "workflow");
       return;
     }
-    const { runId, traceCarrier: traceContext, requestedAt, serverErrorRetryCount } = WorkflowInvokePayloadSchema.parse(message_);
+    const { runId, traceCarrier: traceContext, requestedAt } = WorkflowInvokePayloadSchema.parse(message_);
     const workflowName = metadata.queueName.slice("__wkf_workflow_".length);
     const spanLinks = await linkToCurrentContext();
     return await withTraceContext(traceContext, async () => {
@@ -58680,179 +58664,183 @@ function workflowEntrypoint(workflowCode2) {
             ...WorkflowRunId(runId),
             ...WorkflowTracePropagated(!!traceContext)
           });
-          return await withThrottleRetry(async () => {
-            let workflowStartedAt = -1;
-            let workflowRun = await world.runs.get(runId);
-            try {
-              if (workflowRun.status === "pending") {
-                const result2 = await world.events.create(runId, {
-                  eventType: "run_started",
-                  specVersion: SPEC_VERSION_CURRENT
-                });
-                if (!result2.run) {
-                  throw new WorkflowRuntimeError(`Event creation for 'run_started' did not return the run entity for run "${runId}"`);
-                }
-                workflowRun = result2.run;
-              }
-              if (!workflowRun.startedAt) {
-                throw new WorkflowRuntimeError(`Workflow run "${runId}" has no "startedAt" timestamp`);
-              }
-              workflowStartedAt = +workflowRun.startedAt;
-              span?.setAttributes({
-                ...WorkflowRunStatus(workflowRun.status),
-                ...WorkflowStartedAt(workflowStartedAt)
+          let workflowStartedAt = -1;
+          let workflowRun = await world.runs.get(runId);
+          try {
+            if (workflowRun.status === "pending") {
+              const result = await world.events.create(runId, {
+                eventType: "run_started",
+                specVersion: SPEC_VERSION_CURRENT
               });
-              if (workflowRun.status !== "running") {
-                runtimeLogger.info("Workflow already completed or failed, skipping", {
-                  workflowRunId: runId,
-                  status: workflowRun.status
-                });
-                return;
-              }
-              const events = await getAllWorkflowRunEvents(workflowRun.runId);
-              const now = Date.now();
-              const completedWaitIds = new Set(events.filter((e) => e.eventType === "wait_completed").map((e) => e.correlationId));
-              const waitsToComplete = events.filter((e) => e.eventType === "wait_created" && e.correlationId !== void 0 && !completedWaitIds.has(e.correlationId) && now >= e.eventData.resumeAt.getTime()).map((e) => ({
-                eventType: "wait_completed",
-                specVersion: SPEC_VERSION_CURRENT,
-                correlationId: e.correlationId
-              }));
-              for (const waitEvent of waitsToComplete) {
-                try {
-                  const result2 = await world.events.create(runId, waitEvent);
-                  events.push(result2.event);
-                } catch (err) {
-                  if (WorkflowAPIError.is(err) && err.status === 409) {
-                    runtimeLogger.info("Wait already completed, skipping", {
-                      workflowRunId: runId,
-                      correlationId: waitEvent.correlationId
-                    });
-                    continue;
-                  }
-                  throw err;
-                }
+              if (!result.run) {
+                throw new WorkflowRuntimeError(`Event creation for 'run_started' did not return the run entity for run "${runId}"`);
               }
-              const result = await trace2("workflow.replay", {}, async (replaySpan) => {
-                replaySpan?.setAttributes({
-                  ...WorkflowEventsCount(events.length)
-                });
-                const rawKey = await world.getEncryptionKeyForRun?.(workflowRun);
-                const encryptionKey = rawKey ? await importKey(rawKey) : void 0;
-                return await runWorkflow(workflowCode2, workflowRun, events, encryptionKey);
-              });
+              workflowRun = result.run;
+            }
+            if (!workflowRun.startedAt) {
+              throw new WorkflowRuntimeError(`Workflow run "${runId}" has no "startedAt" timestamp`);
+            }
+          } catch (err) {
+            if (err instanceof WorkflowRuntimeError) {
+              runtimeLogger.error("Fatal runtime error during workflow setup", { workflowRunId: runId, error: err.message });
               try {
                 await world.events.create(runId, {
-                  eventType: "run_completed",
+                  eventType: "run_failed",
                   specVersion: SPEC_VERSION_CURRENT,
                   eventData: {
-                    output: result
+                    error: {
+                      message: err.message,
+                      stack: err.stack
+                    }
                   }
                 });
-              } catch (err) {
-                if (WorkflowAPIError.is(err) && (err.status === 409 || err.status === 410)) {
-                  runtimeLogger.warn("Tried completing workflow run, but run has already finished.", {
-                    workflowRunId: runId,
-                    message: err.message
-                  });
+              } catch (failErr) {
+                if (WorkflowAPIError.is(failErr) && (failErr.status === 409 || failErr.status === 410)) {
                   return;
-                } else {
-                  throw err;
                 }
+                throw failErr;
               }
-              span?.setAttributes({
-                ...WorkflowRunStatus("completed"),
-                ...WorkflowEventsCount(events.length)
-              });
+              return;
+            }
+            throw err;
+          }
+          workflowStartedAt = +workflowRun.startedAt;
+          span?.setAttributes({
+            ...WorkflowRunStatus(workflowRun.status),
+            ...WorkflowStartedAt(workflowStartedAt)
+          });
+          if (workflowRun.status !== "running") {
+            runtimeLogger.info("Workflow already completed or failed, skipping", {
+              workflowRunId: runId,
+              status: workflowRun.status
+            });
+            return;
+          }
+          const events = await getAllWorkflowRunEvents(workflowRun.runId);
+          const now = Date.now();
+          const completedWaitIds = new Set(events.filter((e) => e.eventType === "wait_completed").map((e) => e.correlationId));
+          const waitsToComplete = events.filter((e) => e.eventType === "wait_created" && e.correlationId !== void 0 && !completedWaitIds.has(e.correlationId) && now >= e.eventData.resumeAt.getTime()).map((e) => ({
+            eventType: "wait_completed",
+            specVersion: SPEC_VERSION_CURRENT,
+            correlationId: e.correlationId
+          }));
+          for (const waitEvent of waitsToComplete) {
+            try {
+              const result = await world.events.create(runId, waitEvent);
+              events.push(result.event);
             } catch (err) {
-              if (WorkflowSuspension.is(err)) {
-                const suspensionMessage = buildWorkflowSuspensionMessage(runId, err.stepCount, err.hookCount, err.waitCount);
-                if (suspensionMessage) {
-                  runtimeLogger.debug(suspensionMessage);
-                }
-                const result = await handleSuspension({
-                  suspension: err,
-                  world,
-                  run: workflowRun,
-                  span
+              if (WorkflowAPIError.is(err) && err.status === 409) {
+                runtimeLogger.info("Wait already completed, skipping", {
+                  workflowRunId: runId,
+                  correlationId: waitEvent.correlationId
                 });
-                if (result.timeoutSeconds !== void 0) {
-                  return { timeoutSeconds: result.timeoutSeconds };
-                }
-              } else {
-                if (WorkflowAPIError.is(err) && err.status !== void 0 && err.status >= 500) {
-                  const retryCount = serverErrorRetryCount ?? 0;
-                  const delaySecondSteps = [5, 30, 120];
-                  if (retryCount < delaySecondSteps.length) {
-                    runtimeLogger.warn("Server error (5xx), re-enqueueing workflow with backoff", {
-                      workflowRunId: runId,
-                      retryCount,
-                      delaySeconds: delaySecondSteps[retryCount],
-                      error: err.message
-                    });
-                    await queueMessage(world, getWorkflowQueueName(workflowName), {
-                      runId,
-                      serverErrorRetryCount: retryCount + 1,
-                      traceCarrier: await serializeTraceCarrier(),
-                      requestedAt: /* @__PURE__ */ new Date()
-                    }, { delaySeconds: delaySecondSteps[retryCount] });
-                    return;
+                continue;
+              }
+              throw err;
+            }
+          }
+          const rawKey = await world.getEncryptionKeyForRun?.(workflowRun);
+          const encryptionKey = rawKey ? await importKey(rawKey) : void 0;
+          let workflowResult;
+          try {
+            workflowResult = await trace2("workflow.replay", {}, async (replaySpan) => {
+              replaySpan?.setAttributes({
+                ...WorkflowEventsCount(events.length)
+              });
+              return await runWorkflow(workflowCode2, workflowRun, events, encryptionKey);
+            });
+          } catch (err) {
+            if (WorkflowSuspension.is(err)) {
+              const suspensionMessage = buildWorkflowSuspensionMessage(runId, err.stepCount, err.hookCount, err.waitCount);
+              if (suspensionMessage) {
+                runtimeLogger.debug(suspensionMessage);
+              }
+              const result = await handleSuspension({
+                suspension: err,
+                world,
+                run: workflowRun,
+                span
+              });
+              if (result.timeoutSeconds !== void 0) {
+                return { timeoutSeconds: result.timeoutSeconds };
+              }
+              return;
+            }
+            if (err instanceof Error) {
+              span?.recordException?.(err);
+            }
+            const normalizedError = await normalizeUnknownError(err);
+            const errorName = normalizedError.name || getErrorName(err);
+            const errorMessage = normalizedError.message;
+            let errorStack = normalizedError.stack || getErrorStack(err);
+            if (errorStack) {
+              const parsedName = parseWorkflowName(workflowName);
+              const filename = parsedName?.moduleSpecifier || workflowName;
+              errorStack = remapErrorStack(errorStack, filename, workflowCode2);
+            }
+            runtimeLogger.error("Error while running workflow", {
+              workflowRunId: runId,
+              errorName,
+              errorStack
+            });
+            try {
+              await world.events.create(runId, {
+                eventType: "run_failed",
+                specVersion: SPEC_VERSION_CURRENT,
+                eventData: {
+                  error: {
+                    message: errorMessage,
+                    stack: errorStack
                   }
-                } else if (WorkflowAPIError.is(err) && err.status === 429) {
-                  throw err;
+                  // TODO: include error codes when we define them
                 }
-                if (err instanceof Error) {
-                  span?.recordException?.(err);
-                }
-                const normalizedError = await normalizeUnknownError(err);
-                const errorName = normalizedError.name || getErrorName(err);
-                const errorMessage = normalizedError.message;
-                let errorStack = normalizedError.stack || getErrorStack(err);
-                if (errorStack) {
-                  const parsedName = parseWorkflowName(workflowName);
-                  const filename = parsedName?.moduleSpecifier || workflowName;
-                  errorStack = remapErrorStack(errorStack, filename, workflowCode2);
-                }
-                runtimeLogger.error("Error while running workflow", {
+              });
+            } catch (failErr) {
+              if (WorkflowAPIError.is(failErr) && (failErr.status === 409 || failErr.status === 410)) {
+                runtimeLogger.warn("Tried failing workflow run, but run has already finished.", {
                   workflowRunId: runId,
-                  errorName,
-                  errorStack
+                  message: failErr.message
                 });
-                try {
-                  await world.events.create(runId, {
-                    eventType: "run_failed",
-                    specVersion: SPEC_VERSION_CURRENT,
-                    eventData: {
-                      error: {
-                        message: errorMessage,
-                        stack: errorStack
-                      }
-                      // TODO: include error codes when we define them
-                    }
-                  });
-                } catch (err2) {
-                  if (WorkflowAPIError.is(err2) && (err2.status === 409 || err2.status === 410)) {
-                    runtimeLogger.warn("Tried failing workflow run, but run has already finished.", {
-                      workflowRunId: runId,
-                      message: err2.message
-                    });
-                    span?.setAttributes({
-                      ...WorkflowErrorName(errorName),
-                      ...WorkflowErrorMessage(errorMessage),
-                      ...ErrorType(errorName)
-                    });
-                    return;
-                  } else {
-                    throw err2;
-                  }
-                }
                 span?.setAttributes({
-                  ...WorkflowRunStatus("failed"),
                   ...WorkflowErrorName(errorName),
                   ...WorkflowErrorMessage(errorMessage),
                   ...ErrorType(errorName)
                 });
+                return;
+              } else {
+                throw failErr;
+              }
+            }
+            span?.setAttributes({
+              ...WorkflowRunStatus("failed"),
+              ...WorkflowErrorName(errorName),
+              ...WorkflowErrorMessage(errorMessage),
+              ...ErrorType(errorName)
+            });
+            return;
+          }
+          try {
+            await world.events.create(runId, {
+              eventType: "run_completed",
+              specVersion: SPEC_VERSION_CURRENT,
+              eventData: {
+                output: workflowResult
               }
+            });
+          } catch (err) {
+            if (WorkflowAPIError.is(err) && (err.status === 409 || err.status === 410)) {
+              runtimeLogger.warn("Tried completing workflow run, but run has already finished.", {
+                workflowRunId: runId,
+                message: err.message
+              });
+              return;
+            } else {
+              throw err;
             }
+          }
+          span?.setAttributes({
+            ...WorkflowRunStatus("completed"),
+            ...WorkflowEventsCount(events.length)
           });
         });
       });
@@ -59090,7 +59078,7 @@ function getWritable(options = {}) {
 __name(getWritable, "getWritable");
 
 // ../workflow/dist/stdlib.js
-var fetch = globalThis[/* @__PURE__ */ Symbol.for("WORKFLOW_USE_STEP")]("step//workflow@4.2.0-beta.69//fetch");
+var fetch = globalThis[/* @__PURE__ */ Symbol.for("WORKFLOW_USE_STEP")]("step//workflow@4.2.0-beta.70//fetch");
 
 // ../../node_modules/.pnpm/zod@4.3.6/node_modules/zod/v4/core/core.js
 var NEVER = Object.freeze({