@vellumai/assistant 0.6.5 → 0.6.6

package/src/__tests__/pipeline-runner.test.ts

@@ -0,0 +1,565 @@
+/**
+ * Unit tests for `plugins/pipeline.ts`.
+ *
+ * Covers:
+ * - Onion composition order (outer → inner → terminal → inner → outer).
+ * - Short-circuit (middleware that omits `next` — terminal is skipped).
+ * - Error propagation (no internal try/catch — errors flow unchanged).
+ * - Timeout (breached budget rejects with `PluginTimeoutError`).
+ * - Log shape (one structured record per invocation, every field typed).
+ */
+
+import { beforeEach, describe, expect, test } from "bun:test";
+
+import type { TrustContext } from "../daemon/conversation-runtime-assembly.js";
+import {
+  composeMiddleware,
+  DEFAULT_TIMEOUTS,
+  runPipeline,
+} from "../plugins/pipeline.js";
+import {
+  type Middleware,
+  PluginTimeoutError,
+  type TurnContext,
+} from "../plugins/types.js";
+
+// A minimal fake pino-compatible logger. The pipeline runner detects a
+// `logger` slot on the context (shape `{ info(record, msg?) }`) and falls
+// back to the module logger only when that slot is absent. Tests pass this
+// fake in via `ctx.logger` so the runner emits into our capture buffer
+// instead of real stderr.
+type LogCall = [record: Record<string, unknown>, msg?: string];
+
+function makeFakeLogger(): {
+  calls: LogCall[];
+  info: (record: Record<string, unknown>, msg?: string) => void;
+  warn: () => void;
+  error: () => void;
+  debug: () => void;
+  trace: () => void;
+  fatal: () => void;
+} {
+  const calls: LogCall[] = [];
+  return {
+    calls,
+    info: (record, msg) => {
+      calls.push([record, msg]);
+    },
+    warn: () => {},
+    error: () => {},
+    debug: () => {},
+    trace: () => {},
+    fatal: () => {},
+  };
+}
+
+let fakeLogger = makeFakeLogger();
+
+const trust: TrustContext = {
+  sourceChannel: "vellum",
+  trustClass: "guardian",
+};
+
+function makeCtx(overrides: Partial<TurnContext> = {}): TurnContext {
+  return {
+    requestId: "req-test",
+    conversationId: "conv-test",
+    turnIndex: 3,
+    trust,
+    // The runner reads `(ctx as { logger?: unknown }).logger` — we cast
+    // through the partial type to attach it without widening TurnContext.
+    ...({ logger: fakeLogger } as Partial<TurnContext>),
+    ...overrides,
+  };
+}
+
+beforeEach(() => {
+  fakeLogger = makeFakeLogger();
+});
+
+type Args = { value: number };
+type Result = { value: number };
+
+describe("composeMiddleware", () => {
+  test("invokes layers in outer→inner→terminal→inner→outer order", async () => {
+    const trace: string[] = [];
+
+    const outer: Middleware<Args, Result> = async (args, next) => {
+      trace.push("outer:before");
+      const result = await next(args);
+      trace.push("outer:after");
+      return result;
+    };
+
+    const inner: Middleware<Args, Result> = async (args, next) => {
+      trace.push("inner:before");
+      const result = await next(args);
+      trace.push("inner:after");
+      return result;
+    };
+
+    const terminal = async (args: Args): Promise<Result> => {
+      trace.push("terminal");
+      return { value: args.value * 2 };
+    };
+
+    const composed = composeMiddleware<Args, Result>([outer, inner], terminal);
+    const result = await composed({ value: 7 }, makeCtx());
+
+    expect(result).toEqual({ value: 14 });
+    expect(trace).toEqual([
+      "outer:before",
+      "inner:before",
+      "terminal",
+      "inner:after",
+      "outer:after",
+    ]);
+  });
+
+  test("middleware that omits `next` short-circuits the chain", async () => {
+    const trace: string[] = [];
+
+    const shortCircuit: Middleware<Args, Result> = async (_args, _next) => {
+      trace.push("short-circuit");
+      return { value: 99 };
+    };
+
+    const inner: Middleware<Args, Result> = async (args, next) => {
+      trace.push("inner");
+      return next(args);
+    };
+
+    const terminal = async (_args: Args): Promise<Result> => {
+      trace.push("terminal");
+      return { value: 0 };
+    };
+
+    const composed = composeMiddleware<Args, Result>(
+      [shortCircuit, inner],
+      terminal,
+    );
+    const result = await composed({ value: 1 }, makeCtx());
+
+    expect(result).toEqual({ value: 99 });
+    expect(trace).toEqual(["short-circuit"]);
+  });
+
+  test("empty middleware list reduces to the terminal handler", async () => {
+    const terminal = async (args: Args): Promise<Result> => ({
+      value: args.value + 1,
+    });
+    const composed = composeMiddleware<Args, Result>([], terminal);
+    const result = await composed({ value: 10 }, makeCtx());
+    expect(result).toEqual({ value: 11 });
+  });
+});
+
+describe("runPipeline — error propagation", () => {
+  test("errors thrown by middleware bubble through unchanged", async () => {
+    class Boom extends Error {
+      override readonly name = "Boom";
+    }
+
+    const thrower: Middleware<Args, Result> = async () => {
+      throw new Boom("detonated");
+    };
+
+    const terminal = async (_args: Args): Promise<Result> => {
+      throw new Error("terminal should not run");
+    };
+
+    await expect(
+      runPipeline(
+        "persistence",
+        [thrower],
+        terminal,
+        { value: 1 },
+        makeCtx(),
+        DEFAULT_TIMEOUTS.persistence,
+      ),
+    ).rejects.toBeInstanceOf(Boom);
+  });
+
+  test("errors thrown by the terminal handler bubble through unchanged", async () => {
+    const terminal = async (_args: Args): Promise<Result> => {
+      throw new TypeError("from terminal");
+    };
+
+    await expect(
+      runPipeline(
+        "persistence",
+        [],
+        terminal,
+        { value: 1 },
+        makeCtx(),
+        DEFAULT_TIMEOUTS.persistence,
+      ),
+    ).rejects.toBeInstanceOf(TypeError);
+  });
+});
+
+describe("runPipeline — timeout", () => {
+  test("breached budget rejects with PluginTimeoutError carrying pipeline + plugin name", async () => {
+    const sleeper: Middleware<Args, Result> = async (_args, _next) =>
+      new Promise<Result>((resolve) => {
+        setTimeout(() => resolve({ value: 0 }), 200);
+      });
+
+    const terminal = async (_args: Args): Promise<Result> => ({ value: 0 });
+
+    let caught: unknown;
+    try {
+      await runPipeline(
+        "memoryRetrieval",
+        [sleeper],
+        terminal,
+        { value: 1 },
+        makeCtx({ pluginName: "slow-plugin" }),
+        20,
+      );
+    } catch (err) {
+      caught = err;
+    }
+
+    expect(caught).toBeInstanceOf(PluginTimeoutError);
+    const tErr = caught as PluginTimeoutError;
+    expect(tErr.pipeline).toBe("memoryRetrieval");
+    expect(tErr.pluginName).toBe("slow-plugin");
+    expect(tErr.elapsedMs).toBeGreaterThanOrEqual(0);
+    expect(tErr.message).toContain("memoryRetrieval");
+    expect(tErr.message).toContain("slow-plugin");
+  });
+
+  test("fast pipeline does not arm the timer redundantly", async () => {
+    const terminal = async (args: Args): Promise<Result> => ({
+      value: args.value,
+    });
+    const result = await runPipeline(
+      "historyRepair",
+      [],
+      terminal,
+      { value: 42 },
+      makeCtx(),
+      DEFAULT_TIMEOUTS.historyRepair,
+    );
+    expect(result).toEqual({ value: 42 });
+  });
+
+  test("null timeout skips the race entirely", async () => {
+    // llmCall has DEFAULT_TIMEOUTS.llmCall === null — runner must not arm a
+    // timer. We verify by completing after an artificial 30ms wait and
+    // confirming success without interference.
+    const sleeper: Middleware<Args, Result> = async (args, next) =>
+      new Promise<Result>((resolve) => {
+        setTimeout(() => resolve(next(args)), 30);
+      });
+    const terminal = async (_args: Args): Promise<Result> => ({ value: 1 });
+    const result = await runPipeline(
+      "llmCall",
+      [sleeper],
+      terminal,
+      { value: 0 },
+      makeCtx(),
+      DEFAULT_TIMEOUTS.llmCall,
+    );
+    expect(result).toEqual({ value: 1 });
+  });
+});
+
+describe("runPipeline — timeout aborts linked signal", () => {
+  test("abort signal on args is fired when the timeout trips", async () => {
+    const callerController = new AbortController();
+    type SignalArgs = { value: number; signal: AbortSignal };
+
+    let observedAbortedAtCallStart = false;
+    let observedAbortedAtCallEnd = false;
+
+    const sleeper: Middleware<SignalArgs, Result> = async (
+      innerArgs,
+      _next,
+    ) => {
+      observedAbortedAtCallStart = innerArgs.signal.aborted;
+      return new Promise<Result>((resolve, reject) => {
+        innerArgs.signal.addEventListener("abort", () => {
+          observedAbortedAtCallEnd = innerArgs.signal.aborted;
+          reject(Object.assign(new Error("aborted"), { name: "AbortError" }));
+        });
+        // Keep running past the timeout if the signal doesn't fire.
+        setTimeout(() => resolve({ value: 0 }), 500);
+      });
+    };
+
+    const terminal = async (_args: SignalArgs): Promise<Result> => ({
+      value: 0,
+    });
+
+    await expect(
+      runPipeline<SignalArgs, Result>(
+        "compaction",
+        [sleeper],
+        terminal,
+        { value: 1, signal: callerController.signal },
+        makeCtx(),
+        15,
+      ),
+    ).rejects.toBeInstanceOf(PluginTimeoutError);
+
+    expect(observedAbortedAtCallStart).toBe(false);
+    expect(observedAbortedAtCallEnd).toBe(true);
+    // Caller's own signal must not be touched — the runner only aborts
+    // its internal linked signal, not the caller-owned controller.
+    expect(callerController.signal.aborted).toBe(false);
+
+    // Log record still reports timeout outcome + correct fields even though
+    // the inner middleware rejected with AbortError; the outer race still
+    // wins with the PluginTimeoutError.
+    const [record] = fakeLogger.calls[0]!;
+    expect(record.outcome).toBe("timeout");
+    expect(record.errorName).toBe("PluginTimeoutError");
+  });
+
+  test("caller-side abort still propagates to the inner call", async () => {
+    const callerController = new AbortController();
+    type SignalArgs = { signal: AbortSignal };
+
+    let innerSignalAborted = false;
+
+    const sleeper: Middleware<SignalArgs, Result> = async (innerArgs) => {
+      return new Promise<Result>((_resolve, reject) => {
+        innerArgs.signal.addEventListener("abort", () => {
+          innerSignalAborted = innerArgs.signal.aborted;
+          reject(Object.assign(new Error("aborted"), { name: "AbortError" }));
+        });
+      });
+    };
+
+    const terminal = async (_args: SignalArgs): Promise<Result> => ({
+      value: 0,
+    });
+
+    // Fire a caller-side abort after a short delay.
+    setTimeout(() => callerController.abort(), 10);
+
+    await expect(
+      runPipeline<SignalArgs, Result>(
+        "compaction",
+        [sleeper],
+        terminal,
+        { signal: callerController.signal },
+        makeCtx(),
+        10000,
+      ),
+    ).rejects.toMatchObject({ name: "AbortError" });
+
+    expect(innerSignalAborted).toBe(true);
+    expect(callerController.signal.aborted).toBe(true);
+  });
+
+  test("args without an AbortSignal property is passed through unchanged", async () => {
+    // Sanity — pipelines that don't carry a signal (persistence, tokenEstimate)
+    // see identical args identity as before the abort-linking change.
+    const args: Args = { value: 42 };
+    let seen: Args | undefined;
+    const terminal = async (innerArgs: Args): Promise<Result> => {
+      seen = innerArgs;
+      return { value: innerArgs.value };
+    };
+    await runPipeline(
+      "persistence",
+      [],
+      terminal,
+      args,
+      makeCtx(),
+      DEFAULT_TIMEOUTS.persistence,
+    );
+    expect(seen).toBe(args);
+  });
+});
+
+describe("runPipeline — structured log record", () => {
+  test("success emits one record with every documented field present", async () => {
+    const namedOuter: Middleware<Args, Result> = async function outerMw(
+      args,
+      next,
+    ) {
+      return next(args);
+    };
+    const terminal = async (args: Args): Promise<Result> => ({
+      value: args.value,
+    });
+
+    await runPipeline(
+      "compaction",
+      [namedOuter],
+      terminal,
+      { value: 7 },
+      makeCtx(),
+      DEFAULT_TIMEOUTS.compaction,
+    );
+
+    expect(fakeLogger.calls.length).toBe(1);
+    const [record, msg] = fakeLogger.calls[0]!;
+    expect(msg).toBe("plugin.pipeline");
+    expect(record.event).toBe("plugin.pipeline");
+    expect(record.pipeline).toBe("compaction");
+    expect(record.chain).toEqual(["outerMw"]);
+    expect(record.outcome).toBe("success");
+    expect(typeof record.durationMs).toBe("number");
+    expect(record.durationMs).toBeGreaterThanOrEqual(0);
+    expect(record.requestId).toBe("req-test");
+    expect(record.conversationId).toBe("conv-test");
+    expect(record.turnIndex).toBe(3);
+    // pluginName is only present when ctx carries one.
+    expect(record.pluginName).toBeUndefined();
+    // Error fields absent on success.
+    expect(record.errorName).toBeUndefined();
+    expect(record.errorMessage).toBeUndefined();
+    expect(record.errorStack).toBeUndefined();
+  });
+
+  test("error path records outcome=error + error fields + plugin name", async () => {
+    class Boom extends Error {
+      override readonly name = "BoomError";
+    }
+    const thrower: Middleware<Args, Result> = async () => {
+      throw new Boom("kaboom");
+    };
+    const terminal = async (_args: Args): Promise<Result> => ({ value: 0 });
+
+    await expect(
+      runPipeline(
+        "toolError",
+        [thrower],
+        terminal,
+        { value: 1 },
+        makeCtx({ pluginName: "noisy-plugin" }),
+        DEFAULT_TIMEOUTS.toolError,
+      ),
+    ).rejects.toBeInstanceOf(Boom);
+
+    expect(fakeLogger.calls.length).toBe(1);
+    const [record] = fakeLogger.calls[0]!;
+    expect(record.outcome).toBe("error");
+    expect(record.pipeline).toBe("toolError");
+    expect(record.errorName).toBe("BoomError");
+    expect(record.errorMessage).toBe("kaboom");
+    expect(typeof record.errorStack).toBe("string");
+    expect(record.pluginName).toBe("noisy-plugin");
+  });
+
+  test("timeout path records outcome=timeout + PluginTimeoutError fields", async () => {
+    const sleeper: Middleware<Args, Result> = async (_args, _next) =>
+      new Promise<Result>((resolve) => {
+        setTimeout(() => resolve({ value: 0 }), 200);
+      });
+    const terminal = async (_args: Args): Promise<Result> => ({ value: 0 });
+
+    await expect(
+      runPipeline(
+        "emptyResponse",
+        [sleeper],
+        terminal,
+        { value: 1 },
+        makeCtx({ pluginName: "slow-plugin" }),
+        15,
+      ),
+    ).rejects.toBeInstanceOf(PluginTimeoutError);
+
+    expect(fakeLogger.calls.length).toBe(1);
+    const [record] = fakeLogger.calls[0]!;
+    expect(record.outcome).toBe("timeout");
+    expect(record.pipeline).toBe("emptyResponse");
+    expect(record.errorName).toBe("PluginTimeoutError");
+    expect(String(record.errorMessage)).toContain("emptyResponse");
+    expect(String(record.errorMessage)).toContain("slow-plugin");
+    expect(record.timeoutMs).toBe(15);
+    expect(record.pluginName).toBe("slow-plugin");
+  });
+
+  test("null timeout omits timeoutMs field from the log record", async () => {
+    const terminal = async (args: Args): Promise<Result> => ({
+      value: args.value,
+    });
+    await runPipeline(
+      "llmCall",
+      [],
+      terminal,
+      { value: 5 },
+      makeCtx(),
+      DEFAULT_TIMEOUTS.llmCall,
+    );
+
+    expect(fakeLogger.calls.length).toBe(1);
+    const [record] = fakeLogger.calls[0]!;
+    expect(record.pipeline).toBe("llmCall");
+    expect(record.outcome).toBe("success");
+    expect(record.timeoutMs).toBeUndefined();
+  });
+
+  test("turnIndex is omitted when unset on the context", async () => {
+    const terminal = async (_args: Args): Promise<Result> => ({ value: 0 });
+    const ctxNoTurn = {
+      requestId: "r",
+      conversationId: "c",
+      turnIndex: undefined as unknown as number,
+      trust,
+      logger: fakeLogger,
+    } as TurnContext;
+    await runPipeline(
+      "persistence",
+      [],
+      terminal,
+      { value: 0 },
+      ctxNoTurn,
+      null,
+    );
+    const [record] = fakeLogger.calls[0]!;
+    expect(record.turnIndex).toBeUndefined();
+  });
+
+  test("chain list has one entry per middleware in registration order", async () => {
+    const a: Middleware<Args, Result> = async function outerA(args, next) {
+      return next(args);
+    };
+    const b: Middleware<Args, Result> = async function middleB(args, next) {
+      return next(args);
+    };
+    const c: Middleware<Args, Result> = async function innerC(args, next) {
+      return next(args);
+    };
+    const terminal = async (args: Args): Promise<Result> => ({
+      value: args.value,
+    });
+    await runPipeline(
+      "tokenEstimate",
+      [a, b, c],
+      terminal,
+      { value: 0 },
+      makeCtx(),
+      DEFAULT_TIMEOUTS.tokenEstimate,
+    );
+    const [record] = fakeLogger.calls[0]!;
+    expect(record.chain).toEqual(["outerA", "middleB", "innerC"]);
+  });
+});
+
+describe("DEFAULT_TIMEOUTS", () => {
+  test("matches the design-doc table exactly", () => {
+    expect(DEFAULT_TIMEOUTS).toEqual({
+      turn: null,
+      llmCall: null,
+      toolExecute: null,
+      memoryRetrieval: null,
+      historyRepair: null,
+      tokenEstimate: null,
+      compaction: null,
+      overflowReduce: null,
+      persistence: null,
+      titleGenerate: null,
+      toolResultTruncate: null,
+      emptyResponse: null,
+      toolError: null,
+      circuitBreaker: null,
+    });
+  });
+});

package/src/__tests__/platform.test.ts

@@ -59,8 +59,9 @@ afterEach(() => {
 // otherwise under ~/.vellum/workspace.
 describe("path characterization", () => {
   test("all path helpers resolve to expected locations", () => {
-    // Without VELLUM_WORKSPACE_DIR override, workspace is under ~/.vellum
+    // Without VELLUM_WORKSPACE_DIR or BASE_DATA_DIR override, everything is under ~/.vellum
     delete process.env.VELLUM_WORKSPACE_DIR;
+    delete process.env.BASE_DATA_DIR;
     const root = join(homedir(), ".vellum");
     const ws = getWorkspaceDir();
     const data = getDataDir();
@@ -87,6 +88,7 @@ describe("path characterization", () => {
   });
 
   test("VELLUM_WORKSPACE_DIR overrides workspace location", () => {
+    delete process.env.BASE_DATA_DIR;
     process.env.VELLUM_WORKSPACE_DIR = "/tmp/custom-workspace";
     expect(getWorkspaceDir()).toBe("/tmp/custom-workspace");
     expect(getDataDir()).toBe("/tmp/custom-workspace/data");
@@ -115,6 +117,7 @@ describe("path characterization", () => {
 
   test("hooks directory is inside the workspace boundary", () => {
     delete process.env.VELLUM_WORKSPACE_DIR;
+    delete process.env.BASE_DATA_DIR;
     expect(getWorkspaceHooksDir().startsWith(getWorkspaceDir())).toBe(true);
   });
 
@@ -122,6 +125,7 @@ describe("path characterization", () => {
     // Use a temp VELLUM_WORKSPACE_DIR so ensureDataDir writes to a temp dir
     // rather than the real ~/.vellum. Root-level dirs still go to ~/.vellum
     // but we only verify workspace dirs here to avoid side effects.
+    delete process.env.BASE_DATA_DIR;
     const wsDir = join(tmpdir(), `platform-test-ws-${Date.now()}`);
     process.env.VELLUM_WORKSPACE_DIR = wsDir;
 
@@ -133,7 +137,6 @@ describe("path characterization", () => {
 
     // Workspace dirs (in our temp location)
     expect(existsSync(wsDir)).toBe(true);
-    expect(existsSync(join(wsDir, "hooks"))).toBe(true);
     expect(existsSync(join(wsDir, "skills"))).toBe(true);
 
     // Data sub-dirs under workspace