@vellumai/assistant 0.9.0-dev.202606181513.fd39213 → 0.9.0-dev.202606181735.ae6e1d8

package/src/runtime/routes/workflow-routes.test.ts

@@ -1,6 +1,9 @@
 import { afterEach, beforeEach, describe, expect, test } from "bun:test";
 
-import type { WorkflowRun } from "../../workflows/journal-store.js";
+import type {
+  WorkflowJournalEntry,
+  WorkflowRun,
+} from "../../workflows/journal-store.js";
 import type { SavedWorkflowEntry } from "../../workflows/library.js";
 import {
   WorkflowResumeNotPossibleError,
@@ -58,6 +61,22 @@ interface FakeManager {
   resume: (id: string) => { runId: string };
 }
 
+function makeJournalEntry(
+  overrides: Partial<WorkflowJournalEntry> = {},
+): WorkflowJournalEntry {
+  return {
+    runId: "run-1",
+    seq: 0,
+    callHash: "call-0",
+    kind: "agent",
+    request: null,
+    result: null,
+    status: "completed",
+    createdAt: 1000,
+    ...overrides,
+  };
+}
+
 function setup(opts: {
   runs?: WorkflowRun[];
   saved?: SavedWorkflowEntry[];
@@ -65,6 +84,8 @@ function setup(opts: {
   resume?: (id: string) => { runId: string };
   /** Resolved auto-approve threshold for the resume posture gate. */
   threshold?: "none" | "low" | "medium" | "high";
+  /** Journal entries returned by getJournal, keyed by run id. */
+  journal?: Record<string, WorkflowJournalEntry[]>;
 }): {
   aborted: string[];
   resumed: string[];
@@ -97,6 +118,7 @@ function setup(opts: {
     getManager: () => manager,
     listWorkflows: () => opts.saved ?? [],
     getAutoApproveThreshold: async () => opts.threshold ?? "none",
+    getJournal: (runId) => opts.journal?.[runId] ?? [],
   });
   return { aborted, resumed, listCalls };
 }
@@ -303,6 +325,208 @@ describe("workflow routes (happy paths)", () => {
   });
 });
 
+// ---------------------------------------------------------------------------
+// Journal projection
+// ---------------------------------------------------------------------------
+
+interface WireLeaf {
+  seq: number;
+  kind: string;
+  label?: string;
+  phase?: string;
+  promptSummary?: string;
+  status: string;
+  resultSummary?: string;
+  inputTokens?: number;
+  outputTokens?: number;
+  createdAt: number | null;
+}
+
+interface WireJournal {
+  runId: string;
+  status: string;
+  agentsSpawned: number;
+  inputTokens: number;
+  outputTokens: number;
+  leaves: WireLeaf[];
+}
+
+describe("getWorkflowRunJournal", () => {
+  test("projects agent leaves with label/phase/promptSummary/resultSummary extracted", async () => {
+    setup({
+      runs: [makeRun({ id: "run-1" })],
+      journal: {
+        "run-1": [
+          makeJournalEntry({
+            seq: 0,
+            kind: "agent",
+            request: {
+              prompt: "Investigate the failing build",
+              opts: { label: "investigate", phase: "triage" },
+            },
+            result: { summary: "found the flaky test" },
+            status: "completed",
+            inputTokens: 120,
+            outputTokens: 45,
+          }),
+        ],
+      },
+    });
+    const result = (await route("getWorkflowRunJournal").handler({
+      pathParams: { id: "run-1" },
+    })) as WireJournal;
+
+    expect(result.runId).toBe("run-1");
+    expect(result.status).toBe("running");
+    expect(result.agentsSpawned).toBe(2);
+    expect(result.leaves.map((l) => l.seq)).toEqual([0]);
+
+    const first = result.leaves[0];
+    expect(first.kind).toBe("agent");
+    expect(first.label).toBe("investigate");
+    expect(first.phase).toBe("triage");
+    expect(first.promptSummary).toBe("Investigate the failing build");
+    expect(first.resultSummary).toContain("found the flaky test");
+    // Per-leaf token usage is projected onto the wire leaf.
+    expect(first.inputTokens).toBe(120);
+    expect(first.outputTokens).toBe(45);
+    // The bulky raw request/result payloads are dropped.
+    expect(first).not.toHaveProperty("request");
+    expect(first).not.toHaveProperty("result");
+  });
+
+  test("omits per-leaf token fields when the journal entry carries none", async () => {
+    setup({
+      runs: [makeRun({ id: "run-1" })],
+      journal: {
+        "run-1": [
+          makeJournalEntry({
+            seq: 0,
+            kind: "agent",
+            request: { prompt: "no tokens recorded", opts: {} },
+            status: "completed",
+          }),
+        ],
+      },
+    });
+    const result = (await route("getWorkflowRunJournal").handler({
+      pathParams: { id: "run-1" },
+    })) as WireJournal;
+
+    const leaf = result.leaves[0];
+    expect(leaf).not.toHaveProperty("inputTokens");
+    expect(leaf).not.toHaveProperty("outputTokens");
+  });
+
+  test("excludes kind:workflow entries so the backfill matches the live agent-only stream", async () => {
+    // The live `workflow_leaf_*` stream is emitted only for agent leaves; nested
+    // `workflow(name)` resolutions never reach it. The journal route must filter
+    // to agent leaves too, or a nested-workflow run renders a phantom unlabeled
+    // node on backfill and a different leaf set than live.
+    setup({
+      runs: [makeRun({ id: "run-1" })],
+      journal: {
+        "run-1": [
+          makeJournalEntry({
+            seq: 0,
+            kind: "agent",
+            request: { prompt: "first agent", opts: { label: "first" } },
+            status: "completed",
+          }),
+          makeJournalEntry({
+            seq: 1,
+            kind: "workflow",
+            request: { name: "child", args: { foo: 1 } },
+            result: "child done",
+            status: "completed",
+          }),
+          makeJournalEntry({
+            seq: 2,
+            kind: "agent",
+            request: { prompt: "second agent", opts: { label: "second" } },
+            status: "completed",
+          }),
+        ],
+      },
+    });
+    const result = (await route("getWorkflowRunJournal").handler({
+      pathParams: { id: "run-1" },
+    })) as WireJournal;
+
+    // Only the agent leaves survive, in seq order; the workflow-kind entry (seq
+    // 1) is dropped entirely.
+    expect(result.leaves.map((l) => l.seq)).toEqual([0, 2]);
+    expect(result.leaves.every((l) => l.kind === "agent")).toBe(true);
+    expect(result.leaves.map((l) => l.label)).toEqual(["first", "second"]);
+  });
+
+  test("surfaces a failed leaf's { error } result as resultSummary", async () => {
+    setup({
+      runs: [makeRun({ id: "run-1" })],
+      journal: {
+        "run-1": [
+          makeJournalEntry({
+            seq: 0,
+            status: "failed",
+            request: { prompt: "do the thing", opts: {} },
+            result: { error: "agent crashed: out of tokens" },
+          }),
+        ],
+      },
+    });
+    const result = (await route("getWorkflowRunJournal").handler({
+      pathParams: { id: "run-1" },
+    })) as WireJournal;
+
+    expect(result.leaves[0].status).toBe("failed");
+    expect(result.leaves[0].resultSummary).toContain(
+      "agent crashed: out of tokens",
+    );
+  });
+
+  test("truncates a long prompt/result to ~200 chars", async () => {
+    const longPrompt = "x".repeat(500);
+    const longResult = "y".repeat(500);
+    setup({
+      runs: [makeRun({ id: "run-1" })],
+      journal: {
+        "run-1": [
+          makeJournalEntry({
+            seq: 0,
+            request: { prompt: longPrompt, opts: {} },
+            result: longResult,
+          }),
+        ],
+      },
+    });
+    const result = (await route("getWorkflowRunJournal").handler({
+      pathParams: { id: "run-1" },
+    })) as WireJournal;
+
+    const leaf = result.leaves[0];
+    expect(leaf.promptSummary!.length).toBeLessThanOrEqual(201);
+    expect(leaf.promptSummary!.startsWith("x")).toBe(true);
+    // A truncated summary ends with the ellipsis marker.
+    expect(leaf.promptSummary!.endsWith("…")).toBe(true);
+    expect(leaf.resultSummary!.length).toBeLessThanOrEqual(201);
+  });
+
+  test("returns an empty leaf list for a run with no journal entries", async () => {
+    setup({ runs: [makeRun({ id: "run-1" })] });
+    const result = (await route("getWorkflowRunJournal").handler({
+      pathParams: { id: "run-1" },
+    })) as WireJournal;
+    expect(result.leaves).toEqual([]);
+  });
+
+  test("throws NotFoundError for an unknown id", () => {
+    setup({ runs: [] });
+    expect(() =>
+      route("getWorkflowRunJournal").handler({ pathParams: { id: "nope" } }),
+    ).toThrow(NotFoundError);
+  });
+});
+
 // ---------------------------------------------------------------------------
 // Unknown run → 404
 // ---------------------------------------------------------------------------

package/src/runtime/routes/workflow-routes.ts

@@ -12,7 +12,11 @@ import { z } from "zod";
 import { getAutoApproveThreshold } from "../../permissions/gateway-threshold-reader.js";
 import { isFullAccessThreshold } from "../../permissions/threshold.js";
 import { manifestGrantsSideEffects } from "../../workflows/capabilities.js";
-import type { WorkflowRun } from "../../workflows/journal-store.js";
+import {
+  getJournal,
+  type WorkflowJournalEntry,
+  type WorkflowRun,
+} from "../../workflows/journal-store.js";
 import { listWorkflows } from "../../workflows/library.js";
 import {
   getWorkflowRunManager,
@@ -40,6 +44,7 @@ export interface WorkflowRoutesDeps {
   >;
   listWorkflows: typeof listWorkflows;
   getAutoApproveThreshold: typeof getAutoApproveThreshold;
+  getJournal: typeof getJournal;
 }
 
 function defaultDeps(): WorkflowRoutesDeps {
@@ -47,6 +52,7 @@ function defaultDeps(): WorkflowRoutesDeps {
     getManager: getWorkflowRunManager,
     listWorkflows,
     getAutoApproveThreshold,
+    getJournal,
   };
 }
 
@@ -93,6 +99,29 @@ const savedWorkflowSchema = z.object({
   path: z.string(),
 });
 
+const workflowLeafSchema = z.object({
+  seq: z.number(),
+  kind: z.enum(["agent", "workflow"]),
+  label: z.string().optional(),
+  phase: z.string().optional(),
+  promptSummary: z.string().optional(),
+  status: z.string(),
+  resultSummary: z.string().optional(),
+  inputTokens: z.number().optional(),
+  outputTokens: z.number().optional(),
+  createdAt: z.number().nullable(),
+});
+
+const workflowJournalSchema = z.object({
+  runId: z.string(),
+  status: z.enum(VALID_RUN_STATUSES).optional(),
+  agentsSpawned: z.number().optional(),
+  inputTokens: z.number().optional(),
+  outputTokens: z.number().optional(),
+  phase: z.string().optional(),
+  leaves: z.array(workflowLeafSchema),
+});
+
 // ---------------------------------------------------------------------------
 // Handlers (transport-agnostic)
 // ---------------------------------------------------------------------------
@@ -122,6 +151,64 @@ export function toWireRun(run: WorkflowRun): WorkflowRunWire {
   };
 }
 
+/**
+ * Render an arbitrary journal value (prompt, result, `{ error }`) into a short
+ * single-string preview, truncating to `max` characters. Objects are
+ * JSON-stringified so a structured result still produces a readable summary.
+ */
+function summarize(value: unknown, max = 200): string | undefined {
+  if (value === undefined || value === null) return undefined;
+  let text: string;
+  if (typeof value === "string") {
+    text = value;
+  } else {
+    try {
+      text = JSON.stringify(value);
+    } catch {
+      return undefined;
+    }
+  }
+  return text.length > max ? `${text.slice(0, max)}…` : text;
+}
+
+/**
+ * Project a journal entry into the bounded leaf wire shape, dropping the bulky
+ * raw `request`/`result` payloads in favor of short summaries. The `request`
+ * and `result` columns are `unknown`, so read their fields defensively.
+ */
+export function toWireLeaf(
+  entry: WorkflowJournalEntry,
+): z.infer<typeof workflowLeafSchema> {
+  const request =
+    entry.request && typeof entry.request === "object"
+      ? (entry.request as Record<string, unknown>)
+      : undefined;
+  const opts =
+    request?.opts && typeof request.opts === "object"
+      ? (request.opts as Record<string, unknown>)
+      : undefined;
+  const label = typeof opts?.label === "string" ? opts.label : undefined;
+  const phase = typeof opts?.phase === "string" ? opts.phase : undefined;
+  const promptSummary = summarize(request?.prompt);
+  const resultSummary = summarize(entry.result);
+  return {
+    seq: entry.seq,
+    kind: entry.kind,
+    ...(label !== undefined ? { label } : {}),
+    ...(phase !== undefined ? { phase } : {}),
+    ...(promptSummary !== undefined ? { promptSummary } : {}),
+    status: entry.status,
+    ...(resultSummary !== undefined ? { resultSummary } : {}),
+    ...(entry.inputTokens !== undefined
+      ? { inputTokens: entry.inputTokens }
+      : {}),
+    ...(entry.outputTokens !== undefined
+      ? { outputTokens: entry.outputTokens }
+      : {}),
+    createdAt: entry.createdAt,
+  };
+}
+
 function parseLimit(raw: string | undefined): number | undefined {
   if (raw === undefined) return undefined;
   const n = Number(raw);
@@ -156,6 +243,28 @@ function handleGetRun(id: string) {
   return toWireRun(run);
 }
 
+function handleGetRunJournal(id: string) {
+  const run = deps.getManager().status(id);
+  if (!run) {
+    throw new NotFoundError(`Workflow run ${id} not found`);
+  }
+  return {
+    runId: run.id,
+    status: run.status,
+    agentsSpawned: run.agentsSpawned,
+    inputTokens: run.inputTokens,
+    outputTokens: run.outputTokens,
+    // Only agent leaves; the live `workflow_leaf_*` stream is emitted solely for
+    // agent leaves (nested workflow resolutions never reach it), so the
+    // backfilled set must match. `kind: "workflow"` entries carry no label and
+    // would render as phantom unlabeled nodes.
+    leaves: deps
+      .getJournal(id)
+      .filter((entry) => entry.kind === "agent")
+      .map(toWireLeaf),
+  };
+}
+
 function handleAbortRun(id: string) {
   // status() is the source of truth for existence; abort() itself is a no-op
   // for unknown/finished runs, so 404 on an unknown id is surfaced here.
@@ -271,6 +380,27 @@ export const ROUTES: RouteDefinition[] = [
     additionalResponses: { "404": { description: "Run not found" } },
     handler: ({ pathParams }: RouteHandlerArgs) => handleGetRun(pathParams!.id),
   },
+  {
+    operationId: "getWorkflowRunJournal",
+    endpoint: "workflows/runs/:id/journal",
+    method: "GET",
+    // Unlike the run list/status routes (settings metadata), the journal returns
+    // per-leaf prompt/result summaries — conversation/work-product content — so it
+    // requires `chat.read`, matching the subagent-detail route (the analogous
+    // content surface), not the `settings.read` used by the management routes.
+    policy: {
+      requiredScopes: ["chat.read"],
+      allowedPrincipalTypes: ACTOR_PRINCIPALS,
+    },
+    summary: "Get workflow run journal",
+    description:
+      "Return a workflow run's leaf journal as bounded per-leaf summaries (one entry per finished leaf).",
+    tags: ["workflows"],
+    responseBody: workflowJournalSchema,
+    additionalResponses: { "404": { description: "Run not found" } },
+    handler: ({ pathParams }: RouteHandlerArgs) =>
+      handleGetRunJournal(pathParams!.id),
+  },
   {
     operationId: "abortWorkflowRun",
     endpoint: "workflows/runs/:id/abort",

package/src/tools/workflows/run-workflow.ts

@@ -68,6 +68,7 @@ export async function executeRunWorkflow(
       args,
       manifest,
       conversationId: context.conversationId,
+      ...(context.toolUseId ? { toolUseId: context.toolUseId } : {}),
       ...(label ? { label } : {}),
       trustContext,
     });

package/src/workflows/engine.test.ts

@@ -16,7 +16,11 @@ import type { TrustContext } from "../daemon/trust-context.js";
 import { getSqlite } from "../memory/db-connection.js";
 import { initializeDb } from "../memory/db-init.js";
 import type { ResolvedCapabilities } from "./capabilities.js";
-import { executeWorkflow, extractWorkflowMeta } from "./engine.js";
+import {
+  executeWorkflow,
+  extractWorkflowMeta,
+  type WorkflowLeafEvent,
+} from "./engine.js";
 import * as journalStore from "./journal-store.js";
 import type { LeafResult, RunLeafOptions } from "./leaf-runner.js";
 
@@ -677,6 +681,21 @@ describe("executeWorkflow — resume", () => {
     );
     expect(journalA.map((e) => e.seq)).toEqual([0, 1, 2]);
   });
+
+  test("a completed leaf persists its per-leaf token usage in the journal", async () => {
+    const runner = makeFakeRunner();
+    await execute("wf-leaf-tokens", `return agent("alpha");`, runner.runner);
+
+    const journal = journalStore.getJournal("wf-leaf-tokens");
+    expect(journal).toHaveLength(1);
+    // The fake runner reports 10/5 per leaf; the completed entry carries them.
+    expect(journal[0]).toMatchObject({
+      seq: 0,
+      status: "completed",
+      inputTokens: 10,
+      outputTokens: 5,
+    });
+  });
 });
 
 describe("executeWorkflow — schema vs tool leaf tool forwarding", () => {
@@ -1142,3 +1161,158 @@ describe("executeWorkflow — nested workflow()", () => {
     expect(second.prompts).toEqual([]);
   });
 });
+
+describe("executeWorkflow — onLeaf observe-only hook", () => {
+  beforeEach(resetTables);
+
+  /** Run a workflow, collecting every onLeaf event. */
+  function executeWithLeafEvents(
+    runId: string,
+    scriptSource: string,
+    runner: typeof import("./leaf-runner.js").runLeaf,
+    config: WorkflowsConfig = configWith(),
+    signal?: AbortSignal,
+  ): {
+    promise: ReturnType<typeof executeWorkflow>;
+    events: WorkflowLeafEvent[];
+  } {
+    const events: WorkflowLeafEvent[] = [];
+    const promise = executeWorkflow({
+      runId,
+      scriptSource: META + scriptSource,
+      args: null,
+      capabilities: CAPABILITIES,
+      config,
+      journal: journalStore,
+      leafRunner: runner,
+      trustContext: TRUST,
+      onLeaf: (e) => events.push(e),
+      ...(signal ? { signal } : {}),
+    });
+    return { promise, events };
+  }
+
+  test("a single agent() emits leaf_started then leaf_finished:completed at seq 0", async () => {
+    const fake = makeFakeRunner();
+    const { promise, events } = executeWithLeafEvents(
+      "wf-leaf-single",
+      `return agent("solo");`,
+      fake.runner,
+    );
+    const res = await promise;
+    expect(res.status).toBe("completed");
+
+    expect(events).toEqual([
+      { type: "leaf_started", seq: 0, promptSummary: "solo" },
+      {
+        type: "leaf_finished",
+        seq: 0,
+        status: "completed",
+        inputTokens: 10,
+        outputTokens: 5,
+        resultSummary: "out:solo",
+      },
+    ]);
+  });
+
+  test("a 3-spec parallel() emits 3 started + 3 finished across seqs 0,1,2", async () => {
+    const fake = makeFakeRunner();
+    const { promise, events } = executeWithLeafEvents(
+      "wf-leaf-parallel",
+      `return parallel([leaf("a"), leaf("b"), leaf("c")]);`,
+      fake.runner,
+    );
+    const res = await promise;
+    expect(res.status).toBe("completed");
+
+    const started = events.filter((e) => e.type === "leaf_started");
+    const finished = events.filter((e) => e.type === "leaf_finished");
+    expect(started.map((e) => e.seq).sort((a, b) => a - b)).toEqual([0, 1, 2]);
+    expect(finished.map((e) => e.seq).sort((a, b) => a - b)).toEqual([0, 1, 2]);
+    expect(finished.every((e) => e.status === "completed")).toBe(true);
+  });
+
+  test("a throwing (non-abort) leaf emits leaf_started then leaf_finished:failed", async () => {
+    const fake = makeFakeRunner({ failOn: (p) => p === "boom" });
+    const { promise, events } = executeWithLeafEvents(
+      "wf-leaf-fail",
+      `return parallel([leaf("boom")]);`,
+      fake.runner,
+    );
+    await promise;
+
+    expect(events).toEqual([
+      { type: "leaf_started", seq: 0, promptSummary: "boom" },
+      {
+        type: "leaf_finished",
+        seq: 0,
+        status: "failed",
+        inputTokens: 0,
+        outputTokens: 0,
+        resultSummary: "leaf failed: boom",
+      },
+    ]);
+  });
+
+  test("an aborted in-flight leaf emits leaf_started but NO leaf_finished", async () => {
+    const controller = new AbortController();
+    let started = false;
+    const runner = (async (): Promise<
+      import("./leaf-runner.js").LeafResult
+    > => {
+      started = true;
+      await new Promise<void>((resolve) => {
+        if (controller.signal.aborted) return resolve();
+        controller.signal.addEventListener("abort", () => resolve(), {
+          once: true,
+        });
+      });
+      const err = new Error("The operation was aborted");
+      err.name = "AbortError";
+      throw err;
+    }) as unknown as typeof import("./leaf-runner.js").runLeaf;
+
+    const { promise, events } = executeWithLeafEvents(
+      "wf-leaf-abort",
+      `return agent("long-running");`,
+      runner,
+      configWith(),
+      controller.signal,
+    );
+
+    await waitUntil(() => started);
+    controller.abort();
+
+    const res = await promise;
+    expect(res.status).toBe("aborted");
+    expect(events).toEqual([
+      { type: "leaf_started", seq: 0, promptSummary: "long-running" },
+    ]);
+  });
+
+  test("a journal-replayed leaf emits neither leaf_started nor leaf_finished", async () => {
+    const script = `return agent("alpha");`;
+
+    // First run journals the leaf as completed.
+    const first = makeFakeRunner();
+    const res1 = await executeWithLeafEvents(
+      "wf-leaf-replay",
+      script,
+      first.runner,
+    ).promise;
+    expect(res1.status).toBe("completed");
+    expect(first.prompts).toEqual(["alpha"]);
+
+    // Resume: the leaf replays from the journal, so no onLeaf events fire.
+    const second = makeFakeRunner();
+    const { promise, events } = executeWithLeafEvents(
+      "wf-leaf-replay",
+      script,
+      second.runner,
+    );
+    const res2 = await promise;
+    expect(res2.status).toBe("completed");
+    expect(second.prompts).toEqual([]);
+    expect(events).toEqual([]);
+  });
+});