@vellumai/assistant 0.8.12 → 0.9.0-staging.2

package/src/workflows/fanout-load.test.ts

@@ -0,0 +1,168 @@
+/**
+ * OPT-IN, real-provider fan-out load & cost guard for the workflow engine.
+ *
+ * ─── Manual / opt-in only ───────────────────────────────────────────────────
+ * This suite is SKIPPED unless `ANTHROPIC_API_KEY` is set. When a key IS
+ * present it makes ~200 REAL leaf provider calls (one forced-schema
+ * `workflowLeaf` call per leaf), so it is never run in CI. Run it by hand
+ * when you want to validate the engine against the live provider:
+ *
+ *   ANTHROPIC_API_KEY=sk-... bun test src/workflows/fanout-load.test.ts
+ *
+ * Cost: each leaf is a tiny Haiku-class (cost-optimized) structured-output
+ * call with a small per-leaf context. One full run is ~200 such calls and
+ * typically costs a few US cents (well under $0.25). The assertion band below
+ * caps it at $1 as a blunt runaway-cost tripwire — if a run ever approaches
+ * that, something is mis-resolved (wrong model/profile) and the guard should
+ * fail loudly.
+ *
+ * ─── What it guards against ─────────────────────────────────────────────────
+ * It reproduces the shape that previously caused rate-limit collapse: many
+ * concurrent leaves with non-trivial per-leaf context. The historical failure
+ * mode was a `StructuredOutput` fast-fail cluster — leaves returning null /
+ * zero output tokens en masse, sub-minute "completion", because the token/min
+ * ceiling was blown by an over-wide fan-out. At the default
+ * `maxConcurrentLeaves: 6` the engine throttles the fan-out, so the collapse
+ * must NOT recur. This test asserts that: every leaf returns real non-zero
+ * output, the run completes (not `cap_exceeded` / `failed`), wall-clock lands
+ * in a generous band, and summed cost stays within a sane envelope.
+ */
+
+import { describe, expect, test } from "bun:test";
+
+import { z } from "zod";
+
+import { getConfig } from "../config/loader.js";
+import type { TrustContext } from "../daemon/trust-context.js";
+import { initializeDb } from "../memory/db-init.js";
+import { listUsageEvents } from "../memory/llm-usage-store.js";
+import { resolveCapabilities } from "./capabilities.js";
+import { executeWorkflow } from "./engine.js";
+import * as journalStore from "./journal-store.js";
+import { runLeaf } from "./leaf-runner.js";
+
+const apiKey = process.env.ANTHROPIC_API_KEY;
+
+// Number of concurrent leaves the historical collapse fanned out over; the
+// guard runs the same shape at the engine's default concurrency.
+const LEAF_COUNT = 200;
+
+// Generous wall-clock ceiling. At concurrency 6 with ~200 short leaves this
+// is well under a couple minutes in practice; the band only fails on a true
+// stall, not on normal provider jitter.
+const MAX_WALL_CLOCK_MS = 8 * 60_000;
+
+// Runaway-cost tripwire (see header). A healthy run is a few cents.
+const MAX_COST_USD = 1;
+
+const TRUST: TrustContext = { sourceChannel: "vellum", trustClass: "guardian" };
+
+describe.skipIf(!apiKey)(
+  "Workflow fan-out load & cost guard — real provider",
+  () => {
+    test(
+      "200-leaf fan-out at default concurrency does not rate-limit-collapse",
+      async () => {
+        initializeDb();
+
+        // Non-trivial per-leaf context mirrors the real fan-out shape: each
+        // leaf judges a distinct synthetic record, not a one-token prompt.
+        const items = Array.from({ length: LEAF_COUNT }, (_, i) => ({
+          id: i,
+          subject: `synthetic-record-${i}`,
+          body:
+            `Incident report #${i}. A background job processed batch ${i % 7} ` +
+            `with status code ${200 + (i % 5)} and emitted ${3 + (i % 11)} ` +
+            `warnings about resource contention on shard ${i % 13}. The ` +
+            `operator noted latency drift of ${(i % 9) * 4}ms over the window.`,
+        }));
+
+        // Schema-forced judgement per leaf: a small structured output that
+        // forces a real `workflowLeaf` provider call and a non-empty result.
+        const judgement = z.object({
+          severity: z.enum(["low", "medium", "high"]),
+          summary: z.string(),
+        });
+
+        const scriptSource = `
+export const meta = {
+  name: "fanout-load-guard",
+  description: "200-leaf real-provider fan-out load & cost guard",
+};
+
+const results = map(args.items, (item) =>
+  leaf(
+    "Classify the severity of this incident and summarize it in one short " +
+      "sentence.\\n\\n" +
+      "Subject: " + item.subject + "\\n" +
+      "Body: " + item.body,
+    { schema: args.schema, label: "judge-" + item.id },
+  ),
+);
+
+return results;
+`;
+
+        const runId = `fanout-load-${Date.now()}`;
+        const startedAt = Date.now();
+
+        const result = await executeWorkflow({
+          runId,
+          scriptSource,
+          args: { items, schema: judgement },
+          capabilities: resolveCapabilities({
+            tools: [],
+            hostFunctions: [],
+            persona: false,
+          }),
+          config: getConfig().workflows,
+          journal: journalStore,
+          leafRunner: runLeaf,
+          trustContext: TRUST,
+        });
+
+        const wallClockMs = Date.now() - startedAt;
+
+        // The engine must have throttled, not collapsed: the run completes.
+        expect(result.status).toBe("completed");
+        expect(result.agentsSpawned).toBe(LEAF_COUNT);
+
+        // No `StructuredOutput` fast-fail cluster: every leaf returns a real,
+        // schema-valid, non-null result (the collapse manifested as all-null).
+        const leafResults = result.result as Array<z.infer<
+          typeof judgement
+        > | null>;
+        expect(Array.isArray(leafResults)).toBe(true);
+        expect(leafResults).toHaveLength(LEAF_COUNT);
+        const nulls = leafResults.filter((r) => r == null);
+        expect(nulls).toHaveLength(0);
+        const invalid = leafResults.filter(
+          (r) => !judgement.safeParse(r).success,
+        );
+        expect(invalid).toHaveLength(0);
+
+        // Real, non-zero output tokens in aggregate (the collapse zeroed these
+        // out as leaves fast-failed before producing any tokens).
+        expect(result.outputTokens).toBeGreaterThan(0);
+        expect(result.inputTokens).toBeGreaterThan(0);
+
+        // Wall-clock within a generous band: not a sub-second mass-fail, and
+        // not an indefinite stall.
+        expect(wallClockMs).toBeLessThan(MAX_WALL_CLOCK_MS);
+
+        // Summed cost over this run's `workflowLeaf` usage events stays within
+        // a sane envelope. Filter by call site + the run's start time so we
+        // only sum events this run produced (the table is shared).
+        const events = listUsageEvents({ limit: 10_000 });
+        const runCostUsd = events
+          .filter(
+            (e) => e.callSite === "workflowLeaf" && e.createdAt >= startedAt,
+          )
+          .reduce((sum, e) => sum + (e.estimatedCostUsd ?? 0), 0);
+        expect(runCostUsd).toBeGreaterThan(0);
+        expect(runCostUsd).toBeLessThan(MAX_COST_USD);
+      },
+      MAX_WALL_CLOCK_MS + 60_000,
+    );
+  },
+);

package/src/workflows/journal-store.test.ts

@@ -0,0 +1,369 @@
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+// Silence the logger.
+mock.module("../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+import { getDb, getSqlite } from "../memory/db-connection.js";
+import { initializeDb } from "../memory/db-init.js";
+import { migrateWorkflowRuns } from "../memory/migrations/284-workflow-runs.js";
+import {
+  appendJournalEntry,
+  createRun,
+  finishRun,
+  getJournal,
+  getRun,
+  listRuns,
+  pruneRuns,
+  updateRun,
+} from "./journal-store.js";
+
+initializeDb();
+
+const DAY_MS = 24 * 60 * 60 * 1000;
+
+function resetTables(): void {
+  getSqlite().exec("DELETE FROM workflow_journal");
+  getSqlite().exec("DELETE FROM workflow_runs");
+}
+
+describe("workflow journal store", () => {
+  beforeEach(() => {
+    resetTables();
+  });
+
+  test("the migration is idempotent (re-running is a no-op)", () => {
+    // initializeDb (called at module load) already ran the migration once.
+    // Running it directly a second and third time must not throw and must
+    // leave the tables intact and usable.
+    expect(() => migrateWorkflowRuns(getDb())).not.toThrow();
+    expect(() => migrateWorkflowRuns(getDb())).not.toThrow();
+
+    createRun({
+      id: "wf-idem",
+      scriptSource: "console.log(1)",
+      scriptHash: "h1",
+    });
+    expect(getRun("wf-idem")?.id).toBe("wf-idem");
+  });
+
+  test("createRun round-trips all fields with sane defaults", () => {
+    const run = createRun({
+      id: "wf-1",
+      name: "nightly digest",
+      scriptSource: "await agent('x')",
+      scriptHash: "hash-abc",
+      args: { topic: "ai" },
+      capabilities: ["search", "email"],
+      conversationId: "conv-xyz",
+      trust: { sourceChannel: "slack", trustClass: "unknown" },
+    });
+
+    expect(run).toMatchObject({
+      id: "wf-1",
+      name: "nightly digest",
+      scriptSource: "await agent('x')",
+      scriptHash: "hash-abc",
+      args: { topic: "ai" },
+      capabilities: ["search", "email"],
+      status: "running",
+      conversationId: "conv-xyz",
+      trust: { sourceChannel: "slack", trustClass: "unknown" },
+      agentsSpawned: 0,
+      inputTokens: 0,
+      outputTokens: 0,
+      result: null,
+      error: null,
+      finishedAt: null,
+    });
+    expect(run.createdAt).toBeGreaterThan(0);
+    expect(run.updatedAt).toBeGreaterThan(0);
+
+    expect(getRun("wf-1")).toEqual(run);
+  });
+
+  test("optional fields default to null", () => {
+    const run = createRun({
+      id: "wf-min",
+      scriptSource: "noop",
+      scriptHash: "h",
+    });
+    expect(run).toMatchObject({
+      name: null,
+      args: null,
+      capabilities: null,
+      conversationId: null,
+      trust: null,
+    });
+  });
+
+  test("updateRun patches only provided fields and bumps updated_at", () => {
+    const created = createRun({
+      id: "wf-2",
+      scriptSource: "s",
+      scriptHash: "h",
+    });
+
+    const updated = updateRun("wf-2", {
+      status: "running",
+      agentsSpawned: 3,
+      inputTokens: 100,
+      outputTokens: 50,
+      conversationId: "conv-new",
+    });
+
+    expect(updated).toMatchObject({
+      agentsSpawned: 3,
+      inputTokens: 100,
+      outputTokens: 50,
+      conversationId: "conv-new",
+      // Untouched fields preserved.
+      scriptSource: "s",
+      scriptHash: "h",
+    });
+    expect(updated!.updatedAt!).toBeGreaterThanOrEqual(created.updatedAt!);
+  });
+
+  test("updateRun returns null for an unknown run", () => {
+    expect(updateRun("nope", { status: "failed" })).toBeNull();
+  });
+
+  test("finishRun stamps terminal status, result, and finished_at", () => {
+    createRun({ id: "wf-3", scriptSource: "s", scriptHash: "h" });
+
+    const finished = finishRun("wf-3", {
+      status: "completed",
+      result: { ok: true, count: 2 },
+    });
+
+    expect(finished).toMatchObject({
+      status: "completed",
+      result: { ok: true, count: 2 },
+      error: null,
+    });
+    expect(finished!.finishedAt).toBeGreaterThan(0);
+  });
+
+  test("finishRun records failure error text", () => {
+    createRun({ id: "wf-fail", scriptSource: "s", scriptHash: "h" });
+    const finished = finishRun("wf-fail", {
+      status: "failed",
+      error: "boom",
+    });
+    expect(finished).toMatchObject({ status: "failed", error: "boom" });
+  });
+
+  test("appendJournalEntry + getJournal round-trips in seq order", () => {
+    createRun({ id: "wf-j", scriptSource: "s", scriptHash: "h" });
+
+    appendJournalEntry({
+      runId: "wf-j",
+      seq: 1,
+      callHash: "c1",
+      kind: "agent",
+      request: { prompt: "a" },
+      result: { text: "ra" },
+      status: "completed",
+    });
+    appendJournalEntry({
+      runId: "wf-j",
+      seq: 0,
+      callHash: "c0",
+      kind: "agent",
+      request: { prompt: "b" },
+      result: 123,
+      status: "completed",
+    });
+
+    const journal = getJournal("wf-j");
+    expect(journal.map((e) => e.seq)).toEqual([0, 1]);
+    expect(journal[0]).toMatchObject({
+      runId: "wf-j",
+      seq: 0,
+      callHash: "c0",
+      kind: "agent",
+      request: { prompt: "b" },
+      result: 123,
+      status: "completed",
+    });
+    expect(journal[1]).toMatchObject({
+      kind: "agent",
+      request: { prompt: "a" },
+      result: { text: "ra" },
+    });
+  });
+
+  test("appendJournalEntry is idempotent on (run_id, seq)", () => {
+    createRun({ id: "wf-dup", scriptSource: "s", scriptHash: "h" });
+
+    appendJournalEntry({
+      runId: "wf-dup",
+      seq: 0,
+      callHash: "c0",
+      kind: "agent",
+      status: "completed",
+    });
+    // Replayed append for the same seq must not double-insert.
+    appendJournalEntry({
+      runId: "wf-dup",
+      seq: 0,
+      callHash: "c0",
+      kind: "agent",
+      status: "completed",
+    });
+
+    expect(getJournal("wf-dup")).toHaveLength(1);
+  });
+
+  test("appendJournalEntry upserts a changed-hash re-run at the same seq", () => {
+    createRun({ id: "wf-resume", scriptSource: "s", scriptHash: "h" });
+
+    // First run: leaf at seq 0 produces one (hash, result).
+    appendJournalEntry({
+      runId: "wf-resume",
+      seq: 0,
+      callHash: "hash-old",
+      kind: "agent",
+      request: { prompt: "old" },
+      result: { text: "old-result" },
+      status: "completed",
+    });
+
+    // Resume: the leaf's input CHANGED, so it re-runs and re-appends a new
+    // (hash, result) at the SAME seq. The stale row must be overwritten.
+    appendJournalEntry({
+      runId: "wf-resume",
+      seq: 0,
+      callHash: "hash-new",
+      kind: "agent",
+      request: { prompt: "new" },
+      result: { text: "new-result" },
+      status: "completed",
+    });
+
+    const journal = getJournal("wf-resume");
+    // Still exactly one row at seq 0 (no duplicate).
+    expect(journal).toHaveLength(1);
+    expect(journal[0]).toMatchObject({
+      seq: 0,
+      callHash: "hash-new",
+      request: { prompt: "new" },
+      result: { text: "new-result" },
+      status: "completed",
+    });
+  });
+
+  test("listRuns returns newest-first and honors limit + status filter", () => {
+    // created_at is wall-clock; force distinct ordering via direct timestamps.
+    createRun({ id: "old", scriptSource: "s", scriptHash: "h" });
+    getSqlite().exec(
+      "UPDATE workflow_runs SET created_at = 1000 WHERE id = 'old'",
+    );
+    createRun({ id: "mid", scriptSource: "s", scriptHash: "h" });
+    getSqlite().exec(
+      "UPDATE workflow_runs SET created_at = 2000 WHERE id = 'mid'",
+    );
+    createRun({ id: "new", scriptSource: "s", scriptHash: "h" });
+    getSqlite().exec(
+      "UPDATE workflow_runs SET created_at = 3000 WHERE id = 'new'",
+    );
+    finishRun("new", { status: "completed" });
+
+    expect(listRuns({ limit: 10 }).map((r) => r.id)).toEqual([
+      "new",
+      "mid",
+      "old",
+    ]);
+    expect(listRuns({ limit: 1 }).map((r) => r.id)).toEqual(["new"]);
+    expect(
+      listRuns({ limit: 10, status: "completed" }).map((r) => r.id),
+    ).toEqual(["new"]);
+  });
+
+  test("pruneRuns deletes finished runs past retention but keeps running ones", () => {
+    const now = Date.now();
+
+    // Old finished run — should be pruned.
+    createRun({ id: "old-done", scriptSource: "s", scriptHash: "h" });
+    finishRun("old-done", { status: "completed" });
+    appendJournalEntry({
+      runId: "old-done",
+      seq: 0,
+      callHash: "c",
+      kind: "agent",
+      status: "completed",
+    });
+    getSqlite().exec(
+      `UPDATE workflow_runs SET created_at = ${now - 10 * DAY_MS} WHERE id = 'old-done'`,
+    );
+
+    // Old but still running — must be kept.
+    createRun({ id: "old-running", scriptSource: "s", scriptHash: "h" });
+    getSqlite().exec(
+      `UPDATE workflow_runs SET created_at = ${now - 10 * DAY_MS} WHERE id = 'old-running'`,
+    );
+
+    // Recent finished run — within retention, kept.
+    createRun({ id: "recent-done", scriptSource: "s", scriptHash: "h" });
+    finishRun("recent-done", { status: "completed" });
+
+    const deleted = pruneRuns(7);
+    expect(deleted).toBe(1);
+    expect(getRun("old-done")).toBeNull();
+    expect(getJournal("old-done")).toHaveLength(0);
+    expect(getRun("old-running")).not.toBeNull();
+    expect(getRun("recent-done")).not.toBeNull();
+  });
+
+  test("pruneRuns never reaps a resumable 'interrupted' run, even past retention", () => {
+    const now = Date.now();
+
+    // Old interrupted run (crash-orphaned, awaiting resume) with a journal —
+    // pruning it would destroy resumability, so it MUST be kept.
+    createRun({ id: "old-interrupted", scriptSource: "s", scriptHash: "h" });
+    updateRun("old-interrupted", { status: "interrupted" });
+    appendJournalEntry({
+      runId: "old-interrupted",
+      seq: 0,
+      callHash: "c",
+      kind: "agent",
+      status: "completed",
+    });
+    getSqlite().exec(
+      `UPDATE workflow_runs SET created_at = ${now - 100 * DAY_MS} WHERE id = 'old-interrupted'`,
+    );
+
+    const deleted = pruneRuns(7);
+    expect(deleted).toBe(0);
+    expect(getRun("old-interrupted")).not.toBeNull();
+    // Its journal survives too, so resume can still replay.
+    expect(getJournal("old-interrupted")).toHaveLength(1);
+  });
+
+  test("pruneRuns reaps every TERMINAL status past retention", () => {
+    const now = Date.now();
+    const terminal = [
+      "completed",
+      "failed",
+      "aborted",
+      "cap_exceeded",
+    ] as const;
+    for (const status of terminal) {
+      createRun({ id: `old-${status}`, scriptSource: "s", scriptHash: "h" });
+      finishRun(`old-${status}`, { status });
+      getSqlite().exec(
+        `UPDATE workflow_runs SET created_at = ${now - 10 * DAY_MS} WHERE id = 'old-${status}'`,
+      );
+    }
+
+    const deleted = pruneRuns(7);
+    expect(deleted).toBe(terminal.length);
+    for (const status of terminal) {
+      expect(getRun(`old-${status}`)).toBeNull();
+    }
+  });
+});