@vellumai/assistant 0.9.0-dev.202606181513.fd39213 → 0.9.0-dev.202606181735.ae6e1d8

package/src/workflows/engine.ts

@@ -73,6 +73,30 @@ export type WorkflowProgressEvent =
   | { type: "phase"; title: string }
   | { type: "log"; message: string };
 
+/**
+ * An observe-only lifecycle event for a single leaf — the source of a live leaf
+ * tree. Emitted once when a leaf STARTS (after the cap check + spawn increment)
+ * and once when it FINISHES (completed or failed). A journal-replayed leaf and
+ * an aborted in-flight leaf emit neither.
+ */
+export type WorkflowLeafEvent =
+  | {
+      type: "leaf_started";
+      seq: number;
+      label?: string;
+      phase?: string;
+      promptSummary: string;
+    }
+  | {
+      type: "leaf_finished";
+      seq: number;
+      status: "completed" | "failed";
+      label?: string;
+      inputTokens: number;
+      outputTokens: number;
+      resultSummary: string;
+    };
+
 /** The journal-store surface the engine depends on (injectable for tests). */
 export interface WorkflowJournal {
   appendJournalEntry: typeof JournalStore.appendJournalEntry;
@@ -108,6 +132,13 @@ export interface ExecuteWorkflowOptions {
   trustContext: TrustContext;
   /** Receives `phase`/`log` progress events from the script. */
   onProgress?: (event: WorkflowProgressEvent) => void;
+  /**
+   * Observe-only per-leaf lifecycle hook. Emits once on START (after the cap
+   * check + spawn increment; NEVER on a journal-replay short-circuit) and once
+   * on FINISH (completed or failed). An aborted in-flight leaf emits NEITHER a
+   * finish nor a spurious failed.
+   */
+  onLeaf?: (event: WorkflowLeafEvent) => void;
   /** Cooperative cancellation for the whole run. */
   signal?: AbortSignal;
 }
@@ -451,6 +482,26 @@ function skipRegexLiteral(source: string, start: number): number {
   return source.length - 1;
 }
 
+/**
+ * A bounded string summary of a leaf prompt or result, for an observe-only
+ * {@link WorkflowLeafEvent}. Strings pass through; everything else is
+ * JSON-serialized (falling back to `String(value)` if it cannot be). Truncated
+ * to `max` chars with a trailing ellipsis.
+ */
+function summarizeForLeafEvent(value: unknown, max = 200): string {
+  let text: string;
+  if (typeof value === "string") {
+    text = value;
+  } else {
+    try {
+      text = JSON.stringify(value);
+    } catch {
+      text = String(value);
+    }
+  }
+  return text.length > max ? `${text.slice(0, max)}…` : text;
+}
+
 function callHashOf(prompt: string, opts: LeafCallOptions): string {
   return createHash("sha256")
     .update(deterministicStringify({ prompt, opts }))
@@ -542,6 +593,7 @@ export async function executeWorkflow(
     leafRunner,
     trustContext,
     onProgress,
+    onLeaf,
     signal,
   } = opts;
 
@@ -630,6 +682,14 @@ export async function executeWorkflow(
     }
     agentsSpawned += 1;
 
+    onLeaf?.({
+      type: "leaf_started",
+      seq,
+      ...(leafOpts.label !== undefined ? { label: leafOpts.label } : {}),
+      ...(leafOpts.phase !== undefined ? { phase: leafOpts.phase } : {}),
+      promptSummary: summarizeForLeafEvent(prompt),
+    });
+
     try {
       // A leaf is EITHER a schema leaf (structured output via forced
       // tool-choice — `schema` set, no tools) OR a tool leaf (free-form with
@@ -666,6 +726,17 @@ export async function executeWorkflow(
         request: { prompt, opts: leafOpts },
         result: result.output,
         status: "completed",
+        inputTokens: result.inputTokens,
+        outputTokens: result.outputTokens,
+      });
+      onLeaf?.({
+        type: "leaf_finished",
+        seq,
+        status: "completed",
+        ...(leafOpts.label !== undefined ? { label: leafOpts.label } : {}),
+        inputTokens: result.inputTokens,
+        outputTokens: result.outputTokens,
+        resultSummary: summarizeForLeafEvent(result.output),
       });
       return { output: result.output, failed: false };
     } catch (err) {
@@ -693,6 +764,17 @@ export async function executeWorkflow(
         { err, runId, seq, label: leafOpts.label },
         "Workflow leaf failed",
       );
+      onLeaf?.({
+        type: "leaf_finished",
+        seq,
+        status: "failed",
+        ...(leafOpts.label !== undefined ? { label: leafOpts.label } : {}),
+        inputTokens: 0,
+        outputTokens: 0,
+        resultSummary: summarizeForLeafEvent(
+          err instanceof Error ? err.message : String(err),
+        ),
+      });
       return { output: null, failed: true };
     }
   };

package/src/workflows/journal-store.test.ts

@@ -197,6 +197,76 @@ describe("workflow journal store", () => {
     });
   });
 
+  test("appendJournalEntry round-trips per-leaf token usage", () => {
+    createRun({ id: "wf-tok", scriptSource: "s", scriptHash: "h" });
+
+    appendJournalEntry({
+      runId: "wf-tok",
+      seq: 0,
+      callHash: "c0",
+      kind: "agent",
+      request: { prompt: "with tokens" },
+      result: { text: "r" },
+      status: "completed",
+      inputTokens: 120,
+      outputTokens: 45,
+    });
+    // A leaf with no recorded usage stores NULL → reads back as undefined.
+    appendJournalEntry({
+      runId: "wf-tok",
+      seq: 1,
+      callHash: "c1",
+      kind: "agent",
+      request: { prompt: "no tokens" },
+      result: { error: "boom" },
+      status: "failed",
+    });
+
+    const journal = getJournal("wf-tok");
+    expect(journal[0]).toMatchObject({
+      seq: 0,
+      inputTokens: 120,
+      outputTokens: 45,
+    });
+    expect(journal[1].inputTokens).toBeUndefined();
+    expect(journal[1].outputTokens).toBeUndefined();
+  });
+
+  test("appendJournalEntry upserts token usage on a changed-hash re-run", () => {
+    createRun({ id: "wf-tok-resume", scriptSource: "s", scriptHash: "h" });
+
+    appendJournalEntry({
+      runId: "wf-tok-resume",
+      seq: 0,
+      callHash: "hash-old",
+      kind: "agent",
+      result: { text: "old" },
+      status: "completed",
+      inputTokens: 10,
+      outputTokens: 5,
+    });
+    // Resume re-runs the leaf with new usage; the upsert overwrites the tokens.
+    appendJournalEntry({
+      runId: "wf-tok-resume",
+      seq: 0,
+      callHash: "hash-new",
+      kind: "agent",
+      result: { text: "new" },
+      status: "completed",
+      inputTokens: 30,
+      outputTokens: 12,
+    });
+
+    const journal = getJournal("wf-tok-resume");
+    expect(journal).toHaveLength(1);
+    expect(journal[0]).toMatchObject({
+      seq: 0,
+      callHash: "hash-new",
+      inputTokens: 30,
+      outputTokens: 12,
+    });
+  });
+
   test("appendJournalEntry is idempotent on (run_id, seq)", () => {
     createRun({ id: "wf-dup", scriptSource: "s", scriptHash: "h" });
 

package/src/workflows/journal-store.ts

@@ -68,6 +68,9 @@ export interface WorkflowJournalEntry {
   request: unknown;
   result: unknown;
   status: string;
+  /** Per-leaf token usage for a completed agent leaf; undefined otherwise. */
+  inputTokens?: number;
+  outputTokens?: number;
   createdAt: number | null;
 }
 
@@ -111,6 +114,9 @@ export interface AppendJournalEntryInput {
   request?: unknown;
   result?: unknown;
   status: string;
+  /** Per-leaf token usage; set for a completed agent leaf, omitted otherwise. */
+  inputTokens?: number;
+  outputTokens?: number;
 }
 
 // ---------------------------------------------------------------------------
@@ -145,6 +151,8 @@ interface WorkflowJournalRow {
   request_json: string | null;
   result_json: string | null;
   status: string;
+  input_tokens: number | null;
+  output_tokens: number | null;
   created_at: number | null;
 }
 
@@ -194,6 +202,8 @@ function rowToJournalEntry(row: WorkflowJournalRow): WorkflowJournalEntry {
     request: parseJsonColumn(row.request_json),
     result: parseJsonColumn(row.result_json),
     status: row.status,
+    inputTokens: row.input_tokens ?? undefined,
+    outputTokens: row.output_tokens ?? undefined,
     createdAt: row.created_at,
   };
 }
@@ -318,14 +328,17 @@ export function appendJournalEntry(
   rawRun(
     /*sql*/ `
     INSERT INTO workflow_journal (
-      run_id, seq, call_hash, kind, request_json, result_json, status, created_at
-    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+      run_id, seq, call_hash, kind, request_json, result_json, status,
+      input_tokens, output_tokens, created_at
+    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
     ON CONFLICT(run_id, seq) DO UPDATE SET
       call_hash = excluded.call_hash,
       kind = excluded.kind,
       request_json = excluded.request_json,
       result_json = excluded.result_json,
-      status = excluded.status
+      status = excluded.status,
+      input_tokens = excluded.input_tokens,
+      output_tokens = excluded.output_tokens
     `,
     input.runId,
     input.seq,
@@ -334,6 +347,8 @@ export function appendJournalEntry(
     serializeJsonColumn(input.request),
     serializeJsonColumn(input.result),
     input.status,
+    input.inputTokens ?? null,
+    input.outputTokens ?? null,
     Date.now(),
   );
   return getJournalEntry(input.runId, input.seq)!;

package/src/workflows/run-manager.test.ts

@@ -292,6 +292,7 @@ describe("WorkflowRunManager.abort", () => {
       scriptSource: "export const meta = { name: 'x', description: 'y' }",
       args: {},
       manifest: { tools: [], hostFunctions: [], persona: false },
+      conversationId: "conv-1",
       trustContext: TRUST,
     });
 
@@ -318,6 +319,7 @@ describe("WorkflowRunManager — progress events", () => {
       scriptSource: "export const meta = { name: 'x', description: 'y' }",
       args: {},
       manifest: { tools: [], hostFunctions: [], persona: false },
+      conversationId: "conv-1",
       label: "My Flow",
       trustContext: TRUST,
     });
@@ -330,14 +332,164 @@ describe("WorkflowRunManager — progress events", () => {
     expect(progress).toHaveLength(2);
     expect(progress[0]).toMatchObject({
       type: "workflow_progress",
+      conversationId: "conv-1",
       label: "My Flow",
       phase: "Gathering",
     });
     expect(progress[1]).toMatchObject({
       type: "workflow_progress",
+      conversationId: "conv-1",
       message: "step done",
     });
   });
+
+  test("a run with no conversation broadcasts progress unscoped (no conversationId)", () => {
+    const h = makeHarness();
+    h.manager.start({
+      scriptSource: "export const meta = { name: 'x', description: 'y' }",
+      args: {},
+      manifest: { tools: [], hostFunctions: [], persona: false },
+      label: "My Flow",
+      trustContext: TRUST,
+    });
+
+    const onProgress = h.executeCalls[0]!.onProgress!;
+    onProgress({ type: "phase", title: "Gathering" });
+    onProgress({ type: "log", message: "step done" });
+
+    // `workflow_progress` is a pre-existing event; it still broadcasts for a
+    // conversationless run, just without a `conversationId` (unscoped).
+    const progress = h.broadcasts.filter((b) => b.type === "workflow_progress");
+    expect(progress).toHaveLength(2);
+    expect(progress[0]).toMatchObject({
+      type: "workflow_progress",
+      label: "My Flow",
+      phase: "Gathering",
+    });
+    expect(progress[0]).not.toHaveProperty("conversationId");
+  });
+
+  test("onLeaf start/finish are republished as scoped leaf events", () => {
+    const fakeEngine: WorkflowRunManagerDeps["executeWorkflow"] = (options) => {
+      options.onLeaf?.({
+        type: "leaf_started",
+        seq: 1,
+        label: "Research",
+        phase: "Gather",
+        promptSummary: "look it up",
+      });
+      options.onLeaf?.({
+        type: "leaf_finished",
+        seq: 1,
+        status: "completed",
+        label: "Research",
+        inputTokens: 30,
+        outputTokens: 12,
+        resultSummary: "found it",
+      });
+      return new Promise(() => {}) as ReturnType<
+        WorkflowRunManagerDeps["executeWorkflow"]
+      >;
+    };
+    const h = makeHarness({ engine: fakeEngine });
+
+    const { runId } = h.manager.start({
+      scriptSource: "export const meta = { name: 'x', description: 'y' }",
+      args: {},
+      manifest: { tools: [], hostFunctions: [], persona: false },
+      conversationId: "conv-1",
+      trustContext: TRUST,
+    });
+
+    const started = h.broadcasts.find(
+      (b) => b.type === "workflow_leaf_started",
+    );
+    expect(started).toMatchObject({
+      type: "workflow_leaf_started",
+      runId,
+      conversationId: "conv-1",
+      seq: 1,
+      label: "Research",
+      phase: "Gather",
+      promptSummary: "look it up",
+    });
+
+    const finished = h.broadcasts.find(
+      (b) => b.type === "workflow_leaf_finished",
+    );
+    expect(finished).toMatchObject({
+      type: "workflow_leaf_finished",
+      runId,
+      conversationId: "conv-1",
+      seq: 1,
+      status: "completed",
+      label: "Research",
+      inputTokens: 30,
+      outputTokens: 12,
+      resultSummary: "found it",
+    });
+  });
+
+  test("a run with no conversation gets no onLeaf callback", () => {
+    const h = makeHarness();
+    h.manager.start({
+      scriptSource: "export const meta = { name: 'x', description: 'y' }",
+      args: {},
+      manifest: { tools: [], hostFunctions: [], persona: false },
+      trustContext: TRUST,
+    });
+
+    expect(h.executeCalls[0]!.onLeaf).toBeUndefined();
+  });
+});
+
+describe("WorkflowRunManager.start — workflow_started", () => {
+  test("start with a conversation broadcasts workflow_started carrying toolUseId", () => {
+    const h = makeHarness();
+    const { runId } = h.manager.start({
+      scriptSource: "export const meta = { name: 'x', description: 'y' }",
+      args: {},
+      manifest: { tools: [], hostFunctions: [], persona: false },
+      conversationId: "conv-1",
+      toolUseId: "toolu-abc",
+      label: "My Flow",
+      trustContext: TRUST,
+    });
+
+    const started = h.broadcasts.find((b) => b.type === "workflow_started");
+    expect(started).toMatchObject({
+      type: "workflow_started",
+      runId,
+      conversationId: "conv-1",
+      toolUseId: "toolu-abc",
+      label: "My Flow",
+    });
+  });
+
+  test("start without a conversation broadcasts no workflow_started", () => {
+    const h = makeHarness();
+    h.manager.start({
+      scriptSource: "export const meta = { name: 'x', description: 'y' }",
+      args: {},
+      manifest: { tools: [], hostFunctions: [], persona: false },
+      trustContext: TRUST,
+    });
+
+    expect(h.broadcasts.some((b) => b.type === "workflow_started")).toBe(false);
+  });
+
+  test("resume never broadcasts workflow_started", () => {
+    const h = makeHarness({});
+    seedRun(h, {
+      id: "run-x",
+      status: "interrupted",
+      conversationId: "conv-1",
+    });
+
+    h.manager.resume("run-x");
+
+    expect(h.broadcasts.some((b) => b.type === "workflow_started")).toBe(false);
+  });
 });
 
 describe("WorkflowRunManager — completion", () => {
@@ -364,6 +516,7 @@ describe("WorkflowRunManager — completion", () => {
     expect(completed).toMatchObject({
       type: "workflow_completed",
       runId,
+      conversationId: "conv-1",
       status: "completed",
       agentsSpawned: 4,
       inputTokens: 100,
@@ -418,7 +571,7 @@ describe("WorkflowRunManager — completion", () => {
     expect(h.manager.status(runId)?.result).toBe(big);
   });
 
-  test("no conversationId → completion events fire but no wake", async () => {
+  test("no conversationId → pre-existing events broadcast unscoped; UI events and wake are gated", async () => {
     const h = makeHarness();
     h.manager.start({
       scriptSource: "export const meta = { name: 'x', description: 'y' }",
@@ -435,8 +588,23 @@ describe("WorkflowRunManager — completion", () => {
       outputTokens: 1,
     });
 
-    expect(h.broadcasts.some((b) => b.type === "workflow_completed")).toBe(
-      true,
+    // The pre-existing terminal event still broadcasts (unscoped) so raw SSE
+    // listeners and conversationless scheduled runs are still surfaced.
+    const completed = h.broadcasts.find((b) => b.type === "workflow_completed");
+    expect(completed).toMatchObject({
+      type: "workflow_completed",
+      status: "completed",
+    });
+    expect(completed).not.toHaveProperty("conversationId");
+
+    // The conversation-only signals stay gated: no `workflow_started`, no leaf
+    // events, and no completion wake without an originating conversation.
+    expect(h.broadcasts.some((b) => b.type === "workflow_started")).toBe(false);
+    expect(h.broadcasts.some((b) => b.type === "workflow_leaf_started")).toBe(
+      false,
+    );
+    expect(h.broadcasts.some((b) => b.type === "workflow_leaf_finished")).toBe(
+      false,
     );
     expect(h.wakes).toHaveLength(0);
   });

package/src/workflows/run-manager.ts

@@ -114,6 +114,11 @@ interface StartWorkflowCommon {
    * scheduled run with no chat surface) — the summary is then events-only.
    */
   conversationId?: string;
+  /**
+   * The `skill_execute` tool-use block id that launched this run; forwarded on
+   * `workflow_started` so the client anchors the inline card to the spawn call.
+   */
+  toolUseId?: string;
   /** Human-readable label for display; defaults to the run id. */
   label?: string;
   /** Trust/auth context forwarded to every leaf. */
@@ -215,6 +220,19 @@ export class WorkflowRunManager {
     const controller = new AbortController();
     this.inflight.set(runId, controller);
 
+    // Announce the launch only when there's a conversation to scope it to (the
+    // same gate the in-flight/terminal broadcasts use). `resume` never emits
+    // this — the run already announced itself on its original `start`.
+    if (opts.conversationId) {
+      this.deps.broadcast({
+        type: "workflow_started",
+        runId,
+        conversationId: opts.conversationId,
+        ...(opts.toolUseId ? { toolUseId: opts.toolUseId } : {}),
+        label,
+      });
+    }
+
     // Fire-and-forget: the engine owns its own try/catch and always finishes
     // the run row. We never await it — `start` must return synchronously.
     void this.runToCompletion(
@@ -376,6 +394,14 @@ export class WorkflowRunManager {
     ctx: RunContext,
   ): Promise<void> {
     const config = this.deps.getConfig();
+    // `workflow_progress` / `workflow_completed` always broadcast: a run with no
+    // originating conversation emits them unscoped (no `conversationId`) for raw
+    // SSE listeners and the DB record, preserving the pre-scoping contract that
+    // surfaces conversationless scheduled runs. The conversation-only signals —
+    // `workflow_started`, the `onLeaf` leaf events, and the completion wake —
+    // stay gated on `conversationId` since they exist solely to drive the inline
+    // workflow card in that conversation.
+    const conversationId = ctx.conversationId;
     try {
       const result = await this.deps.executeWorkflow({
         runId,
@@ -393,6 +419,7 @@ export class WorkflowRunManager {
           this.deps.broadcast({
             type: "workflow_progress",
             runId,
+            ...(conversationId ? { conversationId } : {}),
             label,
             agentsSpawned,
             ...(event.type === "phase"
@@ -400,6 +427,41 @@ export class WorkflowRunManager {
               : { message: event.message }),
           });
         },
+        ...(conversationId
+          ? {
+              onLeaf: (event) => {
+                if (event.type === "leaf_started") {
+                  this.deps.broadcast({
+                    type: "workflow_leaf_started",
+                    runId,
+                    conversationId,
+                    seq: event.seq,
+                    ...(event.label !== undefined
+                      ? { label: event.label }
+                      : {}),
+                    ...(event.phase !== undefined
+                      ? { phase: event.phase }
+                      : {}),
+                    promptSummary: event.promptSummary,
+                  });
+                } else {
+                  this.deps.broadcast({
+                    type: "workflow_leaf_finished",
+                    runId,
+                    conversationId,
+                    seq: event.seq,
+                    status: event.status,
+                    ...(event.label !== undefined
+                      ? { label: event.label }
+                      : {}),
+                    inputTokens: event.inputTokens,
+                    outputTokens: event.outputTokens,
+                    resultSummary: event.resultSummary,
+                  });
+                }
+              },
+            }
+          : {}),
       });
 
       const summary = buildCompletionSummary(
@@ -412,6 +474,7 @@ export class WorkflowRunManager {
       this.deps.broadcast({
         type: "workflow_completed",
         runId,
+        ...(conversationId ? { conversationId } : {}),
         status: result.status,
         agentsSpawned: result.agentsSpawned,
         inputTokens: result.inputTokens,