@mastra/inngest 0.0.0-message-list-update-20250715150321 → 0.0.0-model-router-unknown-provider-20251017212006

package/dist/index.cjs

@@ -2,17 +2,24 @@
 
 var crypto = require('crypto');
 var realtime = require('@inngest/realtime');
+var aiTracing = require('@mastra/core/ai-tracing');
 var di = require('@mastra/core/di');
 var tools = require('@mastra/core/tools');
 var workflows = require('@mastra/core/workflows');
 var _constants = require('@mastra/core/workflows/_constants');
+var inngest = require('inngest');
 var hono = require('inngest/hono');
 var zod = require('zod');
 
 // src/index.ts
-function serve({ mastra, inngest }) {
+function serve({
+  mastra,
+  inngest,
+  functions: userFunctions = [],
+  registerOptions
+}) {
   const wfs = mastra.getWorkflows();
-  const functions = Array.from(
+  const workflowFunctions = Array.from(
     new Set(
       Object.values(wfs).flatMap((wf) => {
         if (wf instanceof InngestWorkflow) {
@@ -24,8 +31,9 @@ function serve({ mastra, inngest }) {
     )
   );
   return hono.serve({
+    ...registerOptions,
     client: inngest,
-    functions
+    functions: [...workflowFunctions, ...userFunctions]
   });
 }
 var InngestRun = class extends workflows.Run {
@@ -53,9 +61,15 @@ var InngestRun = class extends workflows.Run {
       await new Promise((resolve) => setTimeout(resolve, 1e3));
       runs = await this.getRuns(eventId);
       if (runs?.[0]?.status === "Failed") {
-        console.log("run", runs?.[0]);
-        throw new Error(`Function run ${runs?.[0]?.status}`);
-      } else if (runs?.[0]?.status === "Cancelled") {
+        const snapshot = await this.#mastra?.storage?.loadWorkflowSnapshot({
+          workflowName: this.workflowId,
+          runId: this.runId
+        });
+        return {
+          output: { result: { steps: snapshot?.context, status: "failed", error: runs?.[0]?.output?.message } }
+        };
+      }
+      if (runs?.[0]?.status === "Cancelled") {
         const snapshot = await this.#mastra?.storage?.loadWorkflowSnapshot({
           workflowName: this.workflowId,
           runId: this.runId
@@ -86,6 +100,7 @@ var InngestRun = class extends workflows.Run {
       await this.#mastra?.storage?.persistWorkflowSnapshot({
         workflowName: this.workflowId,
         runId: this.runId,
+        resourceId: this.resourceId,
         snapshot: {
           ...snapshot,
           status: "canceled"
@@ -94,11 +109,13 @@ var InngestRun = class extends workflows.Run {
     }
   }
   async start({
-    inputData
+    inputData,
+    initialState
   }) {
     await this.#mastra.getStorage()?.persistWorkflowSnapshot({
       workflowName: this.workflowId,
       runId: this.runId,
+      resourceId: this.resourceId,
       snapshot: {
         runId: this.runId,
         serializedStepGraph: this.serializedStepGraph,
@@ -106,15 +123,21 @@ var InngestRun = class extends workflows.Run {
         context: {},
         activePaths: [],
         suspendedPaths: {},
+        resumeLabels: {},
+        waitingPaths: {},
         timestamp: Date.now(),
         status: "running"
       }
     });
+    const inputDataToUse = await this._validateInput(inputData);
+    const initialStateToUse = await this._validateInitialState(initialState ?? {});
     const eventOutput = await this.inngest.send({
       name: `workflow.${this.workflowId}`,
       data: {
-        inputData,
-        runId: this.runId
+        inputData: inputDataToUse,
+        initialState: initialStateToUse,
+        runId: this.runId,
+        resourceId: this.resourceId
       }
     });
     const eventId = eventOutput.ids[0];
@@ -150,16 +173,20 @@ var InngestRun = class extends workflows.Run {
       workflowName: this.workflowId,
       runId: this.runId
     });
+    const suspendedStep = this.workflowSteps[steps?.[0] ?? ""];
+    const resumeDataToUse = await this._validateResumeData(params.resumeData, suspendedStep);
     const eventOutput = await this.inngest.send({
       name: `workflow.${this.workflowId}`,
       data: {
-        inputData: params.resumeData,
+        inputData: resumeDataToUse,
+        initialState: snapshot?.value ?? {},
         runId: this.runId,
+        workflowId: this.workflowId,
         stepResults: snapshot?.context,
         resume: {
           steps,
           stepResults: snapshot?.context,
-          resumePayload: params.resumeData,
+          resumePayload: resumeDataToUse,
           // @ts-ignore
           resumePath: snapshot?.suspendedPaths?.[steps?.[0]]
         }
@@ -204,7 +231,11 @@ var InngestRun = class extends workflows.Run {
     const writer = writable.getWriter();
     const unwatch = this.watch(async (event) => {
       try {
-        await writer.write(event);
+        const e = {
+          ...event,
+          type: event.type.replace("workflow-", "")
+        };
+        await writer.write(e);
       } catch {
       }
     }, "watch-v2");
@@ -235,8 +266,14 @@ var InngestWorkflow = class _InngestWorkflow extends workflows.Workflow {
   #mastra;
   inngest;
   function;
+  flowControlConfig;
   constructor(params, inngest) {
-    super(params);
+    const { concurrency, rateLimit, throttle, debounce, priority, ...workflowParams } = params;
+    super(workflowParams);
+    const flowControlEntries = Object.entries({ concurrency, rateLimit, throttle, debounce, priority }).filter(
+      ([_, value]) => value !== void 0
+    );
+    this.flowControlConfig = flowControlEntries.length > 0 ? Object.fromEntries(flowControlEntries) : void 0;
     this.#mastra = params.mastra;
     this.inngest = inngest;
   }
@@ -257,27 +294,6 @@ var InngestWorkflow = class _InngestWorkflow extends workflows.Workflow {
     const run = await storage.getWorkflowRunById({ runId, workflowName: this.id });
     return run ?? (this.runs.get(runId) ? { ...this.runs.get(runId), workflowName: this.id } : null);
   }
-  async getWorkflowRunExecutionResult(runId) {
-    const storage = this.#mastra?.getStorage();
-    if (!storage) {
-      this.logger.debug("Cannot get workflow run execution result. Mastra storage is not initialized");
-      return null;
-    }
-    const run = await storage.getWorkflowRunById({ runId, workflowName: this.id });
-    if (!run?.snapshot) {
-      return null;
-    }
-    if (typeof run.snapshot === "string") {
-      return null;
-    }
-    return {
-      status: run.snapshot.status,
-      result: run.snapshot.result,
-      error: run.snapshot.error,
-      payload: run.snapshot.context?.input,
-      steps: run.snapshot.context
-    };
-  }
   __registerMastra(mastra) {
     this.#mastra = mastra;
     this.executionEngine.__registerMastra(mastra);
@@ -296,23 +312,14 @@ var InngestWorkflow = class _InngestWorkflow extends workflows.Workflow {
       }
     }
   }
-  createRun(options) {
-    const runIdToUse = options?.runId || crypto.randomUUID();
-    const run = this.runs.get(runIdToUse) ?? new InngestRun(
-      {
-        workflowId: this.id,
-        runId: runIdToUse,
-        executionEngine: this.executionEngine,
-        executionGraph: this.executionGraph,
-        serializedStepGraph: this.serializedStepGraph,
-        mastra: this.#mastra,
-        retryConfig: this.retryConfig,
-        cleanup: () => this.runs.delete(runIdToUse)
-      },
-      this.inngest
+  /**
+   * @deprecated Use createRunAsync() instead.
+   * @throws {Error} Always throws an error directing users to use createRunAsync()
+   */
+  createRun(_options) {
+    throw new Error(
+      "createRun() has been deprecated. Please use createRunAsync() instead.\n\nMigration guide:\n  Before: const run = workflow.createRun();\n  After:  const run = await workflow.createRunAsync();\n\nNote: createRunAsync() is an async method, so make sure your calling function is async."
     );
-    this.runs.set(runIdToUse, run);
-    return run;
   }
   async createRunAsync(options) {
     const runIdToUse = options?.runId || crypto.randomUUID();
@@ -320,29 +327,38 @@ var InngestWorkflow = class _InngestWorkflow extends workflows.Workflow {
       {
         workflowId: this.id,
         runId: runIdToUse,
+        resourceId: options?.resourceId,
         executionEngine: this.executionEngine,
         executionGraph: this.executionGraph,
         serializedStepGraph: this.serializedStepGraph,
         mastra: this.#mastra,
         retryConfig: this.retryConfig,
-        cleanup: () => this.runs.delete(runIdToUse)
+        cleanup: () => this.runs.delete(runIdToUse),
+        workflowSteps: this.steps
       },
       this.inngest
     );
     this.runs.set(runIdToUse, run);
-    const workflowSnapshotInStorage = await this.getWorkflowRunExecutionResult(runIdToUse);
-    if (!workflowSnapshotInStorage) {
+    const shouldPersistSnapshot = this.options.shouldPersistSnapshot({
+      workflowStatus: run.workflowRunStatus,
+      stepResults: {}
+    });
+    const workflowSnapshotInStorage = await this.getWorkflowRunExecutionResult(runIdToUse, false);
+    if (!workflowSnapshotInStorage && shouldPersistSnapshot) {
       await this.mastra?.getStorage()?.persistWorkflowSnapshot({
         workflowName: this.id,
         runId: runIdToUse,
+        resourceId: options?.resourceId,
         snapshot: {
           runId: runIdToUse,
           status: "pending",
           value: {},
           context: {},
           activePaths: [],
+          waitingPaths: {},
           serializedStepGraph: this.serializedStepGraph,
           suspendedPaths: {},
+          resumeLabels: {},
           result: void 0,
           error: void 0,
           // @ts-ignore
@@ -361,11 +377,13 @@ var InngestWorkflow = class _InngestWorkflow extends workflows.Workflow {
         id: `workflow.${this.id}`,
         // @ts-ignore
         retries: this.retryConfig?.attempts ?? 0,
-        cancelOn: [{ event: `cancel.workflow.${this.id}` }]
+        cancelOn: [{ event: `cancel.workflow.${this.id}` }],
+        // Spread flow control configuration
+        ...this.flowControlConfig
       },
       { event: `workflow.${this.id}` },
       async ({ event, step, attempt, publish }) => {
-        let { inputData, runId, resume } = event.data;
+        let { inputData, initialState, runId, resourceId, resume, outputOptions } = event.data;
         if (!runId) {
           runId = await step.run(`workflow.${this.id}.runIdGen`, async () => {
             return crypto.randomUUID();
@@ -393,19 +411,32 @@ var InngestWorkflow = class _InngestWorkflow extends workflows.Workflow {
           once: (_event, _callback) => {
           }
         };
-        const engine = new InngestExecutionEngine(this.#mastra, step, attempt);
+        const engine = new InngestExecutionEngine(this.#mastra, step, attempt, this.options);
         const result = await engine.execute({
           workflowId: this.id,
           runId,
+          resourceId,
           graph: this.executionGraph,
           serializedStepGraph: this.serializedStepGraph,
           input: inputData,
+          initialState,
           emitter,
           retryConfig: this.retryConfig,
           runtimeContext: new di.RuntimeContext(),
           // TODO
           resume,
-          abortController: new AbortController()
+          abortController: new AbortController(),
+          currentSpan: void 0,
+          // TODO: Pass actual parent AI span from workflow execution context
+          outputOptions
+        });
+        await step.run(`workflow.${this.id}.finalize`, async () => {
+          if (result.status === "failed") {
+            throw new inngest.NonRetriableError(`Workflow failed`, {
+              cause: result
+            });
+          }
+          return result;
         });
         return { result, runId };
       }
@@ -439,17 +470,16 @@ function createStep(params) {
   if (isAgent(params)) {
     return {
       id: params.name,
+      description: params.getDescription(),
       // @ts-ignore
       inputSchema: zod.z.object({
         prompt: zod.z.string()
-        // resourceId: z.string().optional(),
-        // threadId: z.string().optional(),
       }),
       // @ts-ignore
       outputSchema: zod.z.object({
         text: zod.z.string()
       }),
-      execute: async ({ inputData, [_constants.EMITTER_SYMBOL]: emitter, runtimeContext, abortSignal, abort }) => {
+      execute: async ({ inputData, [_constants.EMITTER_SYMBOL]: emitter, runtimeContext, abortSignal, abort, tracingContext }) => {
         let streamPromise = {};
         streamPromise.promise = new Promise((resolve, reject) => {
           streamPromise.resolve = resolve;
@@ -459,50 +489,66 @@ function createStep(params) {
           name: params.name,
           args: inputData
         };
-        await emitter.emit("watch-v2", {
-          type: "tool-call-streaming-start",
-          ...toolData
-        });
-        const { fullStream } = await params.stream(inputData.prompt, {
-          // resourceId: inputData.resourceId,
-          // threadId: inputData.threadId,
-          runtimeContext,
-          onFinish: (result) => {
-            streamPromise.resolve(result.text);
-          },
-          abortSignal
-        });
-        if (abortSignal.aborted) {
-          return abort();
-        }
-        for await (const chunk of fullStream) {
-          switch (chunk.type) {
-            case "text-delta":
+        if ((await params.getLLM()).getModel().specificationVersion === `v2`) {
+          const { fullStream } = await params.stream(inputData.prompt, {
+            runtimeContext,
+            tracingContext,
+            onFinish: (result) => {
+              streamPromise.resolve(result.text);
+            },
+            abortSignal
+          });
+          if (abortSignal.aborted) {
+            return abort();
+          }
+          await emitter.emit("watch-v2", {
+            type: "tool-call-streaming-start",
+            ...toolData ?? {}
+          });
+          for await (const chunk of fullStream) {
+            if (chunk.type === "text-delta") {
+              await emitter.emit("watch-v2", {
+                type: "tool-call-delta",
+                ...toolData ?? {},
+                argsTextDelta: chunk.payload.text
+              });
+            }
+          }
+        } else {
+          const { fullStream } = await params.streamLegacy(inputData.prompt, {
+            runtimeContext,
+            tracingContext,
+            onFinish: (result) => {
+              streamPromise.resolve(result.text);
+            },
+            abortSignal
+          });
+          if (abortSignal.aborted) {
+            return abort();
+          }
+          await emitter.emit("watch-v2", {
+            type: "tool-call-streaming-start",
+            ...toolData ?? {}
+          });
+          for await (const chunk of fullStream) {
+            if (chunk.type === "text-delta") {
               await emitter.emit("watch-v2", {
                 type: "tool-call-delta",
-                ...toolData,
+                ...toolData ?? {},
                 argsTextDelta: chunk.textDelta
               });
-              break;
-            case "step-start":
-            case "step-finish":
-            case "finish":
-              break;
-            case "tool-call":
-            case "tool-result":
-            case "tool-call-streaming-start":
-            case "tool-call-delta":
-            case "source":
-            case "file":
-            default:
-              await emitter.emit("watch-v2", chunk);
-              break;
+            }
           }
         }
+        await emitter.emit("watch-v2", {
+          type: "tool-call-streaming-finish",
+          ...toolData ?? {}
+        });
         return {
           text: await streamPromise.promise
         };
-      }
+      },
+      component: params.component
     };
   }
   if (isTool(params)) {
@@ -513,15 +559,20 @@ function createStep(params) {
       // TODO: tool probably should have strong id type
       // @ts-ignore
       id: params.id,
+      description: params.description,
       inputSchema: params.inputSchema,
       outputSchema: params.outputSchema,
-      execute: async ({ inputData, mastra, runtimeContext }) => {
+      execute: async ({ inputData, mastra, runtimeContext, tracingContext, suspend, resumeData }) => {
         return params.execute({
           context: inputData,
-          mastra,
-          runtimeContext
+          mastra: aiTracing.wrapMastra(mastra, tracingContext),
+          runtimeContext,
+          tracingContext,
+          suspend,
+          resumeData
         });
-      }
+      },
+      component: "TOOL"
     };
   }
   return {
@@ -537,7 +588,10 @@ function createStep(params) {
 function init(inngest) {
   return {
     createWorkflow(params) {
-      return new InngestWorkflow(params, inngest);
+      return new InngestWorkflow(
+        params,
+        inngest
+      );
     },
     createStep,
     cloneStep(step, opts) {
@@ -546,7 +600,11 @@ function init(inngest) {
         description: step.description,
         inputSchema: step.inputSchema,
         outputSchema: step.outputSchema,
-        execute: step.execute
+        resumeSchema: step.resumeSchema,
+        suspendSchema: step.suspendSchema,
+        stateSchema: step.stateSchema,
+        execute: step.execute,
+        component: step.component
       };
     },
     cloneWorkflow(workflow, opts) {
@@ -566,19 +624,19 @@ function init(inngest) {
 var InngestExecutionEngine = class extends workflows.DefaultExecutionEngine {
   inngestStep;
   inngestAttempts;
-  constructor(mastra, inngestStep, inngestAttempts = 0) {
-    super({ mastra });
+  constructor(mastra, inngestStep, inngestAttempts = 0, options) {
+    super({ mastra, options });
     this.inngestStep = inngestStep;
     this.inngestAttempts = inngestAttempts;
   }
   async execute(params) {
     await params.emitter.emit("watch-v2", {
-      type: "start",
+      type: "workflow-start",
       payload: { runId: params.runId }
     });
     const result = await super.execute(params);
     await params.emitter.emit("watch-v2", {
-      type: "finish",
+      type: "workflow-finish",
       payload: { runId: params.runId }
     });
     return result;
@@ -630,7 +688,7 @@ var InngestExecutionEngine = class extends workflows.DefaultExecutionEngine {
       });
       const suspendedStepIds = Object.entries(stepResults).flatMap(([stepId, stepResult]) => {
         if (stepResult?.status === "suspended") {
-          const nestedPath = stepResult?.payload?.__workflow_meta?.path;
+          const nestedPath = stepResult?.suspendPayload?.__workflow_meta?.path;
           return nestedPath ? [[stepId, ...nestedPath]] : [[stepId]];
         }
         return [];
@@ -640,31 +698,6 @@ var InngestExecutionEngine = class extends workflows.DefaultExecutionEngine {
     executionSpan?.end();
     return base;
   }
-  async superExecuteStep({
-    workflowId,
-    runId,
-    step,
-    stepResults,
-    executionContext,
-    resume,
-    prevOutput,
-    emitter,
-    abortController,
-    runtimeContext
-  }) {
-    return super.executeStep({
-      workflowId,
-      runId,
-      step,
-      stepResults,
-      executionContext,
-      resume,
-      prevOutput,
-      emitter,
-      abortController,
-      runtimeContext
-    });
-  }
   // async executeSleep({ id, duration }: { id: string; duration: number }): Promise<void> {
   //   await this.inngestStep.sleep(id, duration);
   // }
@@ -676,28 +709,40 @@ var InngestExecutionEngine = class extends workflows.DefaultExecutionEngine {
     stepResults,
     emitter,
     abortController,
-    runtimeContext
+    runtimeContext,
+    executionContext,
+    writableStream,
+    tracingContext
   }) {
     let { duration, fn } = entry;
+    const sleepSpan = tracingContext?.currentSpan?.createChildSpan({
+      type: aiTracing.AISpanType.WORKFLOW_SLEEP,
+      name: `sleep: ${duration ? `${duration}ms` : "dynamic"}`,
+      attributes: {
+        durationMs: duration,
+        sleepType: fn ? "dynamic" : "fixed"
+      },
+      tracingPolicy: this.options?.tracingPolicy
+    });
     if (fn) {
+      const stepCallId = crypto.randomUUID();
       duration = await this.inngestStep.run(`workflow.${workflowId}.sleep.${entry.id}`, async () => {
         return await fn({
           runId,
+          workflowId,
           mastra: this.mastra,
           runtimeContext,
           inputData: prevOutput,
+          state: executionContext.state,
+          setState: (state) => {
+            executionContext.state = state;
+          },
           runCount: -1,
-          getInitData: () => stepResults?.input,
-          getStepResult: (step) => {
-            if (!step?.id) {
-              return null;
-            }
-            const result = stepResults[step.id];
-            if (result?.status === "success") {
-              return result.output;
-            }
-            return null;
+          tracingContext: {
+            currentSpan: sleepSpan
           },
+          getInitData: () => stepResults?.input,
+          getStepResult: workflows.getStepResult.bind(this, stepResults),
           // TODO: this function shouldn't have suspend probably?
           suspend: async (_suspendPayload) => {
           },
@@ -707,12 +752,34 @@ var InngestExecutionEngine = class extends workflows.DefaultExecutionEngine {
             abortController?.abort();
           },
           [_constants.EMITTER_SYMBOL]: emitter,
+          // TODO: add streamVNext support
+          [_constants.STREAM_FORMAT_SYMBOL]: executionContext.format,
           engine: { step: this.inngestStep },
-          abortSignal: abortController?.signal
+          abortSignal: abortController?.signal,
+          writer: new tools.ToolStream(
+            {
+              prefix: "workflow-step",
+              callId: stepCallId,
+              name: "sleep",
+              runId
+            },
+            writableStream
+          )
         });
       });
+      sleepSpan?.update({
+        attributes: {
+          durationMs: duration
+        }
+      });
+    }
+    try {
+      await this.inngestStep.sleep(entry.id, !duration || duration < 0 ? 0 : duration);
+      sleepSpan?.end();
+    } catch (e) {
+      sleepSpan?.error({ error: e });
+      throw e;
     }
-    await this.inngestStep.sleep(entry.id, !duration || duration < 0 ? 0 : duration);
   }
   async executeSleepUntil({
     workflowId,
@@ -722,28 +789,41 @@ var InngestExecutionEngine = class extends workflows.DefaultExecutionEngine {
     stepResults,
     emitter,
     abortController,
-    runtimeContext
+    runtimeContext,
+    executionContext,
+    writableStream,
+    tracingContext
   }) {
     let { date, fn } = entry;
+    const sleepUntilSpan = tracingContext?.currentSpan?.createChildSpan({
+      type: aiTracing.AISpanType.WORKFLOW_SLEEP,
+      name: `sleepUntil: ${date ? date.toISOString() : "dynamic"}`,
+      attributes: {
+        untilDate: date,
+        durationMs: date ? Math.max(0, date.getTime() - Date.now()) : void 0,
+        sleepType: fn ? "dynamic" : "fixed"
+      },
+      tracingPolicy: this.options?.tracingPolicy
+    });
     if (fn) {
       date = await this.inngestStep.run(`workflow.${workflowId}.sleepUntil.${entry.id}`, async () => {
+        const stepCallId = crypto.randomUUID();
         return await fn({
           runId,
+          workflowId,
           mastra: this.mastra,
           runtimeContext,
           inputData: prevOutput,
+          state: executionContext.state,
+          setState: (state) => {
+            executionContext.state = state;
+          },
           runCount: -1,
-          getInitData: () => stepResults?.input,
-          getStepResult: (step) => {
-            if (!step?.id) {
-              return null;
-            }
-            const result = stepResults[step.id];
-            if (result?.status === "success") {
-              return result.output;
-            }
-            return null;
+          tracingContext: {
+            currentSpan: sleepUntilSpan
           },
+          getInitData: () => stepResults?.input,
+          getStepResult: workflows.getStepResult.bind(this, stepResults),
           // TODO: this function shouldn't have suspend probably?
           suspend: async (_suspendPayload) => {
           },
@@ -753,15 +833,42 @@ var InngestExecutionEngine = class extends workflows.DefaultExecutionEngine {
             abortController?.abort();
           },
           [_constants.EMITTER_SYMBOL]: emitter,
+          [_constants.STREAM_FORMAT_SYMBOL]: executionContext.format,
+          // TODO: add streamVNext support
           engine: { step: this.inngestStep },
-          abortSignal: abortController?.signal
+          abortSignal: abortController?.signal,
+          writer: new tools.ToolStream(
+            {
+              prefix: "workflow-step",
+              callId: stepCallId,
+              name: "sleep",
+              runId
+            },
+            writableStream
+          )
         });
       });
+      if (date && !(date instanceof Date)) {
+        date = new Date(date);
+      }
+      const time = !date ? 0 : date.getTime() - Date.now();
+      sleepUntilSpan?.update({
+        attributes: {
+          durationMs: Math.max(0, time)
+        }
+      });
     }
     if (!(date instanceof Date)) {
+      sleepUntilSpan?.end();
       return;
     }
-    await this.inngestStep.sleepUntil(entry.id, date);
+    try {
+      await this.inngestStep.sleepUntil(entry.id, date);
+      sleepUntilSpan?.end();
+    } catch (e) {
+      sleepUntilSpan?.error({ error: e });
+      throw e;
+    }
   }
   async executeWaitForEvent({ event, timeout }) {
     const eventData = await this.inngestStep.waitForEvent(`user-event-${event}`, {
@@ -781,8 +888,25 @@ var InngestExecutionEngine = class extends workflows.DefaultExecutionEngine {
     prevOutput,
     emitter,
     abortController,
-    runtimeContext
+    runtimeContext,
+    tracingContext,
+    writableStream,
+    disableScorers
   }) {
+    const stepAISpan = tracingContext?.currentSpan?.createChildSpan({
+      name: `workflow step: '${step.id}'`,
+      type: aiTracing.AISpanType.WORKFLOW_STEP,
+      input: prevOutput,
+      attributes: {
+        stepId: step.id
+      },
+      tracingPolicy: this.options?.tracingPolicy
+    });
+    const { inputData, validationError } = await workflows.validateStepInput({
+      prevOutput,
+      step,
+      validateInputs: this.options?.validateInputs ?? false
+    });
     const startedAt = await this.inngestStep.run(
       `workflow.${executionContext.workflowId}.run.${executionContext.runId}.step.${step.id}.running_ev`,
       async () => {
@@ -809,11 +933,11 @@ var InngestExecutionEngine = class extends workflows.DefaultExecutionEngine {
           eventTimestamp: Date.now()
         });
         await emitter.emit("watch-v2", {
-          type: "step-start",
+          type: "workflow-step-start",
           payload: {
             id: step.id,
             status: "running",
-            payload: prevOutput,
+            payload: inputData,
             startedAt: startedAt2
           }
         });
@@ -824,38 +948,60 @@ var InngestExecutionEngine = class extends workflows.DefaultExecutionEngine {
       const isResume = !!resume?.steps?.length;
       let result;
       let runId;
-      if (isResume) {
-        runId = stepResults[resume?.steps?.[0]]?.payload?.__workflow_meta?.runId ?? crypto.randomUUID();
-        const snapshot = await this.mastra?.getStorage()?.loadWorkflowSnapshot({
-          workflowName: step.id,
-          runId
-        });
-        const invokeResp = await this.inngestStep.invoke(`workflow.${executionContext.workflowId}.step.${step.id}`, {
-          function: step.getFunction(),
-          data: {
-            inputData: prevOutput,
-            runId,
-            resume: {
+      try {
+        if (isResume) {
+          runId = stepResults[resume?.steps?.[0]]?.suspendPayload?.__workflow_meta?.runId ?? crypto.randomUUID();
+          const snapshot = await this.mastra?.getStorage()?.loadWorkflowSnapshot({
+            workflowName: step.id,
+            runId
+          });
+          const invokeResp = await this.inngestStep.invoke(`workflow.${executionContext.workflowId}.step.${step.id}`, {
+            function: step.getFunction(),
+            data: {
+              inputData,
+              initialState: executionContext.state ?? snapshot?.value ?? {},
               runId,
-              steps: resume.steps.slice(1),
-              stepResults: snapshot?.context,
-              resumePayload: resume.resumePayload,
-              // @ts-ignore
-              resumePath: snapshot?.suspendedPaths?.[resume.steps?.[1]]
+              resume: {
+                runId,
+                steps: resume.steps.slice(1),
+                stepResults: snapshot?.context,
+                resumePayload: resume.resumePayload,
+                // @ts-ignore
+                resumePath: snapshot?.suspendedPaths?.[resume.steps?.[1]]
+              },
+              outputOptions: { includeState: true }
             }
-          }
-        });
-        result = invokeResp.result;
-        runId = invokeResp.runId;
-      } else {
-        const invokeResp = await this.inngestStep.invoke(`workflow.${executionContext.workflowId}.step.${step.id}`, {
-          function: step.getFunction(),
-          data: {
-            inputData: prevOutput
-          }
-        });
-        result = invokeResp.result;
-        runId = invokeResp.runId;
+          });
+          result = invokeResp.result;
+          runId = invokeResp.runId;
+          executionContext.state = invokeResp.result.state;
+        } else {
+          const invokeResp = await this.inngestStep.invoke(`workflow.${executionContext.workflowId}.step.${step.id}`, {
+            function: step.getFunction(),
+            data: {
+              inputData,
+              initialState: executionContext.state ?? {},
+              outputOptions: { includeState: true }
+            }
+          });
+          result = invokeResp.result;
+          runId = invokeResp.runId;
+          executionContext.state = invokeResp.result.state;
+        }
+      } catch (e) {
+        const errorCause = e?.cause;
+        if (errorCause && typeof errorCause === "object") {
+          result = errorCause;
+          runId = errorCause.runId || crypto.randomUUID();
+        } else {
+          runId = crypto.randomUUID();
+          result = {
+            status: "failed",
+            error: e instanceof Error ? e : new Error(String(e)),
+            steps: {},
+            input: inputData
+          };
+        }
       }
       const res = await this.inngestStep.run(
         `workflow.${executionContext.workflowId}.step.${step.id}.nestedwf-results`,
@@ -879,7 +1025,7 @@ var InngestExecutionEngine = class extends workflows.DefaultExecutionEngine {
               eventTimestamp: Date.now()
             });
             await emitter.emit("watch-v2", {
-              type: "step-result",
+              type: "workflow-step-result",
               payload: {
                 id: step.id,
                 status: "failed",
@@ -894,7 +1040,7 @@ var InngestExecutionEngine = class extends workflows.DefaultExecutionEngine {
               return stepRes2?.status === "suspended";
             });
             for (const [stepName, stepResult] of suspendedSteps) {
-              const suspendPath = [stepName, ...stepResult?.payload?.__workflow_meta?.path ?? []];
+              const suspendPath = [stepName, ...stepResult?.suspendPayload?.__workflow_meta?.path ?? []];
               executionContext.suspendedPaths[step.id] = executionContext.executionPath;
               await emitter.emit("watch", {
                 type: "watch",
@@ -902,7 +1048,11 @@ var InngestExecutionEngine = class extends workflows.DefaultExecutionEngine {
                   currentStep: {
                     id: step.id,
                     status: "suspended",
-                    payload: { ...stepResult?.payload, __workflow_meta: { runId, path: suspendPath } }
+                    payload: stepResult.payload,
+                    suspendPayload: {
+                      ...stepResult?.suspendPayload,
+                      __workflow_meta: { runId, path: suspendPath }
+                    }
                   },
                   workflowState: {
                     status: "running",
@@ -914,7 +1064,7 @@ var InngestExecutionEngine = class extends workflows.DefaultExecutionEngine {
                 eventTimestamp: Date.now()
               });
               await emitter.emit("watch-v2", {
-                type: "step-suspended",
+                type: "workflow-step-suspended",
                 payload: {
                   id: step.id,
                   status: "suspended"
@@ -924,7 +1074,11 @@ var InngestExecutionEngine = class extends workflows.DefaultExecutionEngine {
                 executionContext,
                 result: {
                   status: "suspended",
-                  payload: { ...stepResult?.payload, __workflow_meta: { runId, path: suspendPath } }
+                  payload: stepResult.payload,
+                  suspendPayload: {
+                    ...stepResult?.suspendPayload,
+                    __workflow_meta: { runId, path: suspendPath }
+                  }
                 }
               };
             }
@@ -971,7 +1125,7 @@ var InngestExecutionEngine = class extends workflows.DefaultExecutionEngine {
             eventTimestamp: Date.now()
           });
           await emitter.emit("watch-v2", {
-            type: "step-result",
+            type: "workflow-step-result",
             payload: {
               id: step.id,
               status: "success",
@@ -979,7 +1133,7 @@ var InngestExecutionEngine = class extends workflows.DefaultExecutionEngine {
             }
           });
           await emitter.emit("watch-v2", {
-            type: "step-finish",
+            type: "workflow-step-finish",
             payload: {
               id: step.id,
               metadata: {}
@@ -989,135 +1143,196 @@ var InngestExecutionEngine = class extends workflows.DefaultExecutionEngine {
         }
       );
       Object.assign(executionContext, res.executionContext);
-      return res.result;
+      return {
+        ...res.result,
+        startedAt,
+        endedAt: Date.now(),
+        payload: inputData,
+        resumedAt: resume?.steps[0] === step.id ? startedAt : void 0,
+        resumePayload: resume?.steps[0] === step.id ? resume?.resumePayload : void 0
+      };
     }
-    const stepRes = await this.inngestStep.run(`workflow.${executionContext.workflowId}.step.${step.id}`, async () => {
-      let execResults;
-      let suspended;
-      let bailed;
-      try {
-        const result = await step.execute({
-          runId: executionContext.runId,
-          mastra: this.mastra,
-          runtimeContext,
-          inputData: prevOutput,
-          resumeData: resume?.steps[0] === step.id ? resume?.resumePayload : void 0,
-          getInitData: () => stepResults?.input,
-          getStepResult: (step2) => {
-            const result2 = stepResults[step2.id];
-            if (result2?.status === "success") {
-              return result2.output;
+    let stepRes;
+    try {
+      stepRes = await this.inngestStep.run(`workflow.${executionContext.workflowId}.step.${step.id}`, async () => {
+        let execResults;
+        let suspended;
+        let bailed;
+        try {
+          if (validationError) {
+            throw validationError;
+          }
+          const result = await step.execute({
+            runId: executionContext.runId,
+            mastra: this.mastra,
+            runtimeContext,
+            writableStream,
+            state: executionContext?.state ?? {},
+            setState: (state) => {
+              executionContext.state = state;
+            },
+            inputData,
+            resumeData: resume?.steps[0] === step.id ? resume?.resumePayload : void 0,
+            tracingContext: {
+              currentSpan: stepAISpan
+            },
+            getInitData: () => stepResults?.input,
+            getStepResult: workflows.getStepResult.bind(this, stepResults),
+            suspend: async (suspendPayload, suspendOptions) => {
+              executionContext.suspendedPaths[step.id] = executionContext.executionPath;
+              if (suspendOptions?.resumeLabel) {
+                executionContext.resumeLabels[suspendOptions.resumeLabel] = step.id;
+              }
+              suspended = { payload: suspendPayload };
+            },
+            bail: (result2) => {
+              bailed = { payload: result2 };
+            },
+            resume: {
+              steps: resume?.steps?.slice(1) || [],
+              resumePayload: resume?.resumePayload,
+              // @ts-ignore
+              runId: stepResults[step.id]?.suspendPayload?.__workflow_meta?.runId
+            },
+            [_constants.EMITTER_SYMBOL]: emitter,
+            engine: {
+              step: this.inngestStep
+            },
+            abortSignal: abortController.signal
+          });
+          const endedAt = Date.now();
+          execResults = {
+            status: "success",
+            output: result,
+            startedAt,
+            endedAt,
+            payload: inputData,
+            resumedAt: resume?.steps[0] === step.id ? startedAt : void 0,
+            resumePayload: resume?.steps[0] === step.id ? resume?.resumePayload : void 0
+          };
+        } catch (e) {
+          const stepFailure = {
+            status: "failed",
+            payload: inputData,
+            error: e instanceof Error ? e.message : String(e),
+            endedAt: Date.now(),
+            startedAt,
+            resumedAt: resume?.steps[0] === step.id ? startedAt : void 0,
+            resumePayload: resume?.steps[0] === step.id ? resume?.resumePayload : void 0
+          };
+          execResults = stepFailure;
+          const fallbackErrorMessage = `Step ${step.id} failed`;
+          stepAISpan?.error({ error: new Error(execResults.error ?? fallbackErrorMessage) });
+          throw new inngest.RetryAfterError(execResults.error ?? fallbackErrorMessage, executionContext.retryConfig.delay, {
+            cause: execResults
+          });
+        }
+        if (suspended) {
+          execResults = {
+            status: "suspended",
+            suspendPayload: suspended.payload,
+            payload: inputData,
+            suspendedAt: Date.now(),
+            startedAt,
+            resumedAt: resume?.steps[0] === step.id ? startedAt : void 0,
+            resumePayload: resume?.steps[0] === step.id ? resume?.resumePayload : void 0
+          };
+        } else if (bailed) {
+          execResults = {
+            status: "bailed",
+            output: bailed.payload,
+            payload: inputData,
+            endedAt: Date.now(),
+            startedAt
+          };
+        }
+        await emitter.emit("watch", {
+          type: "watch",
+          payload: {
+            currentStep: {
+              id: step.id,
+              ...execResults
+            },
+            workflowState: {
+              status: "running",
+              steps: { ...stepResults, [step.id]: execResults },
+              result: null,
+              error: null
             }
-            return null;
-          },
-          suspend: async (suspendPayload) => {
-            executionContext.suspendedPaths[step.id] = executionContext.executionPath;
-            suspended = { payload: suspendPayload };
-          },
-          bail: (result2) => {
-            bailed = { payload: result2 };
-          },
-          resume: {
-            steps: resume?.steps?.slice(1) || [],
-            resumePayload: resume?.resumePayload,
-            // @ts-ignore
-            runId: stepResults[step.id]?.payload?.__workflow_meta?.runId
-          },
-          [_constants.EMITTER_SYMBOL]: emitter,
-          engine: {
-            step: this.inngestStep
           },
-          abortSignal: abortController.signal
+          eventTimestamp: Date.now()
         });
-        const endedAt = Date.now();
-        execResults = {
-          status: "success",
-          output: result,
-          startedAt,
-          endedAt,
-          payload: prevOutput,
-          resumedAt: resume?.steps[0] === step.id ? startedAt : void 0,
-          resumePayload: resume?.steps[0] === step.id ? resume?.resumePayload : void 0
-        };
-      } catch (e) {
-        execResults = {
-          status: "failed",
-          payload: prevOutput,
-          error: e instanceof Error ? e.message : String(e),
-          endedAt: Date.now(),
-          startedAt,
-          resumedAt: resume?.steps[0] === step.id ? startedAt : void 0,
-          resumePayload: resume?.steps[0] === step.id ? resume?.resumePayload : void 0
-        };
-      }
-      if (suspended) {
-        execResults = {
-          status: "suspended",
-          suspendedPayload: suspended.payload,
-          payload: prevOutput,
-          suspendedAt: Date.now(),
-          startedAt,
-          resumedAt: resume?.steps[0] === step.id ? startedAt : void 0,
-          resumePayload: resume?.steps[0] === step.id ? resume?.resumePayload : void 0
-        };
-      } else if (bailed) {
-        execResults = { status: "bailed", output: bailed.payload, payload: prevOutput, endedAt: Date.now(), startedAt };
-      }
-      if (execResults.status === "failed") {
-        if (executionContext.retryConfig.attempts > 0 && this.inngestAttempts < executionContext.retryConfig.attempts) {
-          throw execResults.error;
+        if (execResults.status === "suspended") {
+          await emitter.emit("watch-v2", {
+            type: "workflow-step-suspended",
+            payload: {
+              id: step.id,
+              ...execResults
+            }
+          });
+        } else {
+          await emitter.emit("watch-v2", {
+            type: "workflow-step-result",
+            payload: {
+              id: step.id,
+              ...execResults
+            }
+          });
+          await emitter.emit("watch-v2", {
+            type: "workflow-step-finish",
+            payload: {
+              id: step.id,
+              metadata: {}
+            }
+          });
         }
-      }
-      await emitter.emit("watch", {
-        type: "watch",
-        payload: {
-          currentStep: {
-            id: step.id,
-            ...execResults
-          },
-          workflowState: {
-            status: "running",
-            steps: { ...stepResults, [step.id]: execResults },
-            result: null,
-            error: null
-          }
-        },
-        eventTimestamp: Date.now()
+        stepAISpan?.end({ output: execResults });
+        return { result: execResults, executionContext, stepResults };
       });
-      if (execResults.status === "suspended") {
-        await emitter.emit("watch-v2", {
-          type: "step-suspended",
-          payload: {
-            id: step.id,
-            ...execResults
-          }
-        });
-      } else {
-        await emitter.emit("watch-v2", {
-          type: "step-result",
-          payload: {
-            id: step.id,
-            ...execResults
-          }
-        });
-        await emitter.emit("watch-v2", {
-          type: "step-finish",
-          payload: {
-            id: step.id,
-            metadata: {}
-          }
-        });
-      }
-      return { result: execResults, executionContext, stepResults };
-    });
+    } catch (e) {
+      const stepFailure = e instanceof Error ? e?.cause : {
+        status: "failed",
+        error: e instanceof Error ? e.message : String(e),
+        payload: inputData,
+        startedAt,
+        endedAt: Date.now()
+      };
+      stepRes = {
+        result: stepFailure,
+        executionContext,
+        stepResults: {
+          ...stepResults,
+          [step.id]: stepFailure
+        }
+      };
+    }
+    if (disableScorers !== false && stepRes.result.status === "success") {
+      await this.inngestStep.run(`workflow.${executionContext.workflowId}.step.${step.id}.score`, async () => {
+        if (step.scorers) {
+          await this.runScorers({
+            scorers: step.scorers,
+            runId: executionContext.runId,
+            input: inputData,
+            output: stepRes.result,
+            workflowId: executionContext.workflowId,
+            stepId: step.id,
+            runtimeContext,
+            disableScorers,
+            tracingContext: { currentSpan: stepAISpan }
+          });
+        }
+      });
+    }
     Object.assign(executionContext.suspendedPaths, stepRes.executionContext.suspendedPaths);
     Object.assign(stepResults, stepRes.stepResults);
+    executionContext.state = stepRes.executionContext.state;
     return stepRes.result;
   }
   async persistStepUpdate({
     workflowId,
     runId,
     stepResults,
+    resourceId,
     executionContext,
     serializedStepGraph,
     workflowStatus,
@@ -1127,15 +1342,22 @@ var InngestExecutionEngine = class extends workflows.DefaultExecutionEngine {
     await this.inngestStep.run(
       `workflow.${workflowId}.run.${runId}.path.${JSON.stringify(executionContext.executionPath)}.stepUpdate`,
       async () => {
+        const shouldPersistSnapshot = this.options.shouldPersistSnapshot({ stepResults, workflowStatus });
+        if (!shouldPersistSnapshot) {
+          return;
+        }
         await this.mastra?.getStorage()?.persistWorkflowSnapshot({
           workflowName: workflowId,
           runId,
+          resourceId,
           snapshot: {
             runId,
-            value: {},
+            value: executionContext.state,
             context: stepResults,
             activePaths: [],
             suspendedPaths: executionContext.suspendedPaths,
+            resumeLabels: executionContext.resumeLabels,
+            waitingPaths: {},
             serializedStepGraph,
             status: workflowStatus,
             result,
@@ -1159,30 +1381,50 @@ var InngestExecutionEngine = class extends workflows.DefaultExecutionEngine {
     executionContext,
     emitter,
     abortController,
-    runtimeContext
+    runtimeContext,
+    writableStream,
+    disableScorers,
+    tracingContext
   }) {
+    const conditionalSpan = tracingContext?.currentSpan?.createChildSpan({
+      type: aiTracing.AISpanType.WORKFLOW_CONDITIONAL,
+      name: `conditional: '${entry.conditions.length} conditions'`,
+      input: prevOutput,
+      attributes: {
+        conditionCount: entry.conditions.length
+      },
+      tracingPolicy: this.options?.tracingPolicy
+    });
     let execResults;
     const truthyIndexes = (await Promise.all(
       entry.conditions.map(
         (cond, index) => this.inngestStep.run(`workflow.${workflowId}.conditional.${index}`, async () => {
+          const evalSpan = conditionalSpan?.createChildSpan({
+            type: aiTracing.AISpanType.WORKFLOW_CONDITIONAL_EVAL,
+            name: `condition: '${index}'`,
+            input: prevOutput,
+            attributes: {
+              conditionIndex: index
+            },
+            tracingPolicy: this.options?.tracingPolicy
+          });
           try {
             const result = await cond({
               runId,
+              workflowId,
               mastra: this.mastra,
               runtimeContext,
               runCount: -1,
               inputData: prevOutput,
-              getInitData: () => stepResults?.input,
-              getStepResult: (step) => {
-                if (!step?.id) {
-                  return null;
-                }
-                const result2 = stepResults[step.id];
-                if (result2?.status === "success") {
-                  return result2.output;
-                }
-                return null;
+              state: executionContext.state,
+              setState: (state) => {
+                executionContext.state = state;
+              },
+              tracingContext: {
+                currentSpan: evalSpan
               },
+              getInitData: () => stepResults?.input,
+              getStepResult: workflows.getStepResult.bind(this, stepResults),
               // TODO: this function shouldn't have suspend probably?
               suspend: async (_suspendPayload) => {
               },
@@ -1192,40 +1434,76 @@ var InngestExecutionEngine = class extends workflows.DefaultExecutionEngine {
                 abortController.abort();
               },
               [_constants.EMITTER_SYMBOL]: emitter,
+              [_constants.STREAM_FORMAT_SYMBOL]: executionContext.format,
+              // TODO: add streamVNext support
               engine: {
                 step: this.inngestStep
               },
-              abortSignal: abortController.signal
+              abortSignal: abortController.signal,
+              writer: new tools.ToolStream(
+                {
+                  prefix: "workflow-step",
+                  callId: crypto.randomUUID(),
+                  name: "conditional",
+                  runId
+                },
+                writableStream
+              )
+            });
+            evalSpan?.end({
+              output: result,
+              attributes: {
+                result: !!result
+              }
             });
             return result ? index : null;
           } catch (e) {
+            evalSpan?.error({
+              error: e instanceof Error ? e : new Error(String(e)),
+              attributes: {
+                result: false
+              }
+            });
             return null;
           }
         })
       )
     )).filter((index) => index !== null);
     const stepsToRun = entry.steps.filter((_, index) => truthyIndexes.includes(index));
+    conditionalSpan?.update({
+      attributes: {
+        truthyIndexes,
+        selectedSteps: stepsToRun.map((s) => s.type === "step" ? s.step.id : `control-${s.type}`)
+      }
+    });
     const results = await Promise.all(
       stepsToRun.map(
         (step, index) => this.executeEntry({
           workflowId,
           runId,
           entry: step,
+          serializedStepGraph,
           prevStep,
           stepResults,
           resume,
-          serializedStepGraph,
           executionContext: {
             workflowId,
             runId,
             executionPath: [...executionContext.executionPath, index],
             suspendedPaths: executionContext.suspendedPaths,
+            resumeLabels: executionContext.resumeLabels,
             retryConfig: executionContext.retryConfig,
-            executionSpan: executionContext.executionSpan
+            executionSpan: executionContext.executionSpan,
+            state: executionContext.state
           },
           emitter,
           abortController,
-          runtimeContext
+          runtimeContext,
+          writableStream,
+          disableScorers,
+          tracingContext: {
+            currentSpan: conditionalSpan
+          }
         })
       )
     );
@@ -1234,7 +1512,7 @@ var InngestExecutionEngine = class extends workflows.DefaultExecutionEngine {
     if (hasFailed) {
       execResults = { status: "failed", error: hasFailed.result.error };
     } else if (hasSuspended) {
-      execResults = { status: "suspended", payload: hasSuspended.result.suspendPayload };
+      execResults = { status: "suspended", suspendPayload: hasSuspended.result.suspendPayload };
     } else {
       execResults = {
         status: "success",
@@ -1246,6 +1524,15 @@ var InngestExecutionEngine = class extends workflows.DefaultExecutionEngine {
         }, {})
       };
     }
+    if (execResults.status === "failed") {
+      conditionalSpan?.error({
+        error: new Error(execResults.error)
+      });
+    } else {
+      conditionalSpan?.end({
+        output: execResults.output || execResults
+      });
+    }
     return execResults;
   }
 };
@@ -1256,3 +1543,5 @@ exports.InngestWorkflow = InngestWorkflow;
 exports.createStep = createStep;
 exports.init = init;
 exports.serve = serve;
+//# sourceMappingURL=index.cjs.map
+//# sourceMappingURL=index.cjs.map