@vellumai/assistant 0.8.7-dev.202606052232.2ddc989 → 0.8.8

package/src/__tests__/pipeline-runner.test.ts

@@ -0,0 +1,554 @@
+/**
+ * Unit tests for `plugins/pipeline.ts`.
+ *
+ * Covers:
+ * - Onion composition order (outer → inner → terminal → inner → outer).
+ * - Short-circuit (middleware that omits `next` — terminal is skipped).
+ * - Error propagation (no internal try/catch — errors flow unchanged).
+ * - Timeout (breached budget rejects with `PluginTimeoutError`).
+ * - Log shape (one structured record per invocation, every field typed).
+ */
+
+import { beforeEach, describe, expect, test } from "bun:test";
+
+import type { TrustContext } from "../daemon/trust-context.js";
+import {
+  composeMiddleware,
+  DEFAULT_TIMEOUTS,
+  runPipeline,
+} from "../plugins/pipeline.js";
+import {
+  type Middleware,
+  PluginTimeoutError,
+  type TurnContext,
+} from "../plugins/types.js";
+
+// A minimal fake pino-compatible logger. The pipeline runner detects a
+// `logger` slot on the context (shape `{ info(record, msg?) }`) and falls
+// back to the module logger only when that slot is absent. Tests pass this
+// fake in via `ctx.logger` so the runner emits into our capture buffer
+// instead of real stderr.
+type LogCall = [record: Record<string, unknown>, msg?: string];
+
+function makeFakeLogger(): {
+  calls: LogCall[];
+  info: (record: Record<string, unknown>, msg?: string) => void;
+  warn: () => void;
+  error: () => void;
+  debug: () => void;
+  trace: () => void;
+  fatal: () => void;
+} {
+  const calls: LogCall[] = [];
+  return {
+    calls,
+    info: (record, msg) => {
+      calls.push([record, msg]);
+    },
+    warn: () => {},
+    error: () => {},
+    debug: () => {},
+    trace: () => {},
+    fatal: () => {},
+  };
+}
+
+let fakeLogger = makeFakeLogger();
+
+const trust: TrustContext = {
+  sourceChannel: "vellum",
+  trustClass: "guardian",
+};
+
+function makeCtx(overrides: Partial<TurnContext> = {}): TurnContext {
+  return {
+    requestId: "req-test",
+    conversationId: "conv-test",
+    turnIndex: 3,
+    trust,
+    // The runner reads `(ctx as { logger?: unknown }).logger` — we cast
+    // through the partial type to attach it without widening TurnContext.
+    ...({ logger: fakeLogger } as Partial<TurnContext>),
+    ...overrides,
+  };
+}
+
+beforeEach(() => {
+  fakeLogger = makeFakeLogger();
+});
+
+type Args = { value: number };
+type Result = { value: number };
+
+describe("composeMiddleware", () => {
+  test("invokes layers in outer→inner→terminal→inner→outer order", async () => {
+    const trace: string[] = [];
+
+    const outer: Middleware<Args, Result> = async (args, next) => {
+      trace.push("outer:before");
+      const result = await next(args);
+      trace.push("outer:after");
+      return result;
+    };
+
+    const inner: Middleware<Args, Result> = async (args, next) => {
+      trace.push("inner:before");
+      const result = await next(args);
+      trace.push("inner:after");
+      return result;
+    };
+
+    const terminal = async (args: Args): Promise<Result> => {
+      trace.push("terminal");
+      return { value: args.value * 2 };
+    };
+
+    const composed = composeMiddleware<Args, Result>([outer, inner], terminal);
+    const result = await composed({ value: 7 }, makeCtx());
+
+    expect(result).toEqual({ value: 14 });
+    expect(trace).toEqual([
+      "outer:before",
+      "inner:before",
+      "terminal",
+      "inner:after",
+      "outer:after",
+    ]);
+  });
+
+  test("middleware that omits `next` short-circuits the chain", async () => {
+    const trace: string[] = [];
+
+    const shortCircuit: Middleware<Args, Result> = async (_args, _next) => {
+      trace.push("short-circuit");
+      return { value: 99 };
+    };
+
+    const inner: Middleware<Args, Result> = async (args, next) => {
+      trace.push("inner");
+      return next(args);
+    };
+
+    const terminal = async (_args: Args): Promise<Result> => {
+      trace.push("terminal");
+      return { value: 0 };
+    };
+
+    const composed = composeMiddleware<Args, Result>(
+      [shortCircuit, inner],
+      terminal,
+    );
+    const result = await composed({ value: 1 }, makeCtx());
+
+    expect(result).toEqual({ value: 99 });
+    expect(trace).toEqual(["short-circuit"]);
+  });
+
+  test("empty middleware list reduces to the terminal handler", async () => {
+    const terminal = async (args: Args): Promise<Result> => ({
+      value: args.value + 1,
+    });
+    const composed = composeMiddleware<Args, Result>([], terminal);
+    const result = await composed({ value: 10 }, makeCtx());
+    expect(result).toEqual({ value: 11 });
+  });
+});
+
+describe("runPipeline — error propagation", () => {
+  test("errors thrown by middleware bubble through unchanged", async () => {
+    class Boom extends Error {
+      override readonly name = "Boom";
+    }
+
+    const thrower: Middleware<Args, Result> = async () => {
+      throw new Boom("detonated");
+    };
+
+    const terminal = async (_args: Args): Promise<Result> => {
+      throw new Error("terminal should not run");
+    };
+
+    await expect(
+      runPipeline(
+        "circuitBreaker",
+        [thrower],
+        terminal,
+        { value: 1 },
+        makeCtx(),
+        DEFAULT_TIMEOUTS.circuitBreaker,
+      ),
+    ).rejects.toBeInstanceOf(Boom);
+  });
+
+  test("errors thrown by the terminal handler bubble through unchanged", async () => {
+    const terminal = async (_args: Args): Promise<Result> => {
+      throw new TypeError("from terminal");
+    };
+
+    await expect(
+      runPipeline(
+        "circuitBreaker",
+        [],
+        terminal,
+        { value: 1 },
+        makeCtx(),
+        DEFAULT_TIMEOUTS.circuitBreaker,
+      ),
+    ).rejects.toBeInstanceOf(TypeError);
+  });
+});
+
+describe("runPipeline — timeout", () => {
+  test("breached budget rejects with PluginTimeoutError carrying pipeline + plugin name", async () => {
+    const sleeper: Middleware<Args, Result> = async (_args, _next) =>
+      new Promise<Result>((resolve) => {
+        setTimeout(() => resolve({ value: 0 }), 200);
+      });
+
+    const terminal = async (_args: Args): Promise<Result> => ({ value: 0 });
+
+    let caught: unknown;
+    try {
+      await runPipeline(
+        "compaction",
+        [sleeper],
+        terminal,
+        { value: 1 },
+        makeCtx({ pluginName: "slow-plugin" }),
+        20,
+      );
+    } catch (err) {
+      caught = err;
+    }
+
+    expect(caught).toBeInstanceOf(PluginTimeoutError);
+    const tErr = caught as PluginTimeoutError;
+    expect(tErr.pipeline).toBe("compaction");
+    expect(tErr.pluginName).toBe("slow-plugin");
+    expect(tErr.elapsedMs).toBeGreaterThanOrEqual(0);
+    expect(tErr.message).toContain("compaction");
+    expect(tErr.message).toContain("slow-plugin");
+  });
+
+  test("fast pipeline does not arm the timer redundantly", async () => {
+    const terminal = async (args: Args): Promise<Result> => ({
+      value: args.value,
+    });
+    const result = await runPipeline(
+      "compaction",
+      [],
+      terminal,
+      { value: 42 },
+      makeCtx(),
+      DEFAULT_TIMEOUTS.compaction,
+    );
+    expect(result).toEqual({ value: 42 });
+  });
+
+  test("null timeout skips the race entirely", async () => {
+    // compaction has DEFAULT_TIMEOUTS.compaction === null — runner must
+    // not arm a timer. We verify by completing after an artificial 30ms wait
+    // and confirming success without interference.
+    const sleeper: Middleware<Args, Result> = async (args, next) =>
+      new Promise<Result>((resolve) => {
+        setTimeout(() => resolve(next(args)), 30);
+      });
+    const terminal = async (_args: Args): Promise<Result> => ({ value: 1 });
+    const result = await runPipeline(
+      "compaction",
+      [sleeper],
+      terminal,
+      { value: 0 },
+      makeCtx(),
+      DEFAULT_TIMEOUTS.compaction,
+    );
+    expect(result).toEqual({ value: 1 });
+  });
+});
+
+describe("runPipeline — timeout aborts linked signal", () => {
+  test("abort signal on args is fired when the timeout trips", async () => {
+    const callerController = new AbortController();
+    type SignalArgs = { value: number; signal: AbortSignal };
+
+    let observedAbortedAtCallStart = false;
+    let observedAbortedAtCallEnd = false;
+
+    const sleeper: Middleware<SignalArgs, Result> = async (
+      innerArgs,
+      _next,
+    ) => {
+      observedAbortedAtCallStart = innerArgs.signal.aborted;
+      return new Promise<Result>((resolve, reject) => {
+        innerArgs.signal.addEventListener("abort", () => {
+          observedAbortedAtCallEnd = innerArgs.signal.aborted;
+          reject(Object.assign(new Error("aborted"), { name: "AbortError" }));
+        });
+        // Keep running past the timeout if the signal doesn't fire.
+        setTimeout(() => resolve({ value: 0 }), 500);
+      });
+    };
+
+    const terminal = async (_args: SignalArgs): Promise<Result> => ({
+      value: 0,
+    });
+
+    await expect(
+      runPipeline<SignalArgs, Result>(
+        "compaction",
+        [sleeper],
+        terminal,
+        { value: 1, signal: callerController.signal },
+        makeCtx(),
+        15,
+      ),
+    ).rejects.toBeInstanceOf(PluginTimeoutError);
+
+    expect(observedAbortedAtCallStart).toBe(false);
+    expect(observedAbortedAtCallEnd).toBe(true);
+    // Caller's own signal must not be touched — the runner only aborts
+    // its internal linked signal, not the caller-owned controller.
+    expect(callerController.signal.aborted).toBe(false);
+
+    // Log record still reports timeout outcome + correct fields even though
+    // the inner middleware rejected with AbortError; the outer race still
+    // wins with the PluginTimeoutError.
+    const [record] = fakeLogger.calls[0]!;
+    expect(record.outcome).toBe("timeout");
+    expect(record.errorName).toBe("PluginTimeoutError");
+  });
+
+  test("caller-side abort still propagates to the inner call", async () => {
+    const callerController = new AbortController();
+    type SignalArgs = { signal: AbortSignal };
+
+    let innerSignalAborted = false;
+
+    const sleeper: Middleware<SignalArgs, Result> = async (innerArgs) => {
+      return new Promise<Result>((_resolve, reject) => {
+        innerArgs.signal.addEventListener("abort", () => {
+          innerSignalAborted = innerArgs.signal.aborted;
+          reject(Object.assign(new Error("aborted"), { name: "AbortError" }));
+        });
+      });
+    };
+
+    const terminal = async (_args: SignalArgs): Promise<Result> => ({
+      value: 0,
+    });
+
+    // Fire a caller-side abort after a short delay.
+    setTimeout(() => callerController.abort(), 10);
+
+    await expect(
+      runPipeline<SignalArgs, Result>(
+        "compaction",
+        [sleeper],
+        terminal,
+        { signal: callerController.signal },
+        makeCtx(),
+        10000,
+      ),
+    ).rejects.toMatchObject({ name: "AbortError" });
+
+    expect(innerSignalAborted).toBe(true);
+    expect(callerController.signal.aborted).toBe(true);
+  });
+
+  test("args without an AbortSignal property is passed through unchanged", async () => {
+    // Sanity — pipelines that don't carry a signal (circuitBreaker)
+    // see identical args identity as before the abort-linking change.
+    const args: Args = { value: 42 };
+    let seen: Args | undefined;
+    const terminal = async (innerArgs: Args): Promise<Result> => {
+      seen = innerArgs;
+      return { value: innerArgs.value };
+    };
+    await runPipeline(
+      "circuitBreaker",
+      [],
+      terminal,
+      args,
+      makeCtx(),
+      DEFAULT_TIMEOUTS.circuitBreaker,
+    );
+    expect(seen).toBe(args);
+  });
+});
+
+describe("runPipeline — structured log record", () => {
+  test("success emits one record with every documented field present", async () => {
+    const namedOuter: Middleware<Args, Result> = async function outerMw(
+      args,
+      next,
+    ) {
+      return next(args);
+    };
+    const terminal = async (args: Args): Promise<Result> => ({
+      value: args.value,
+    });
+
+    await runPipeline(
+      "compaction",
+      [namedOuter],
+      terminal,
+      { value: 7 },
+      makeCtx(),
+      DEFAULT_TIMEOUTS.compaction,
+    );
+
+    expect(fakeLogger.calls.length).toBe(1);
+    const [record, msg] = fakeLogger.calls[0]!;
+    expect(msg).toBe("plugin.pipeline");
+    expect(record.event).toBe("plugin.pipeline");
+    expect(record.pipeline).toBe("compaction");
+    expect(record.chain).toEqual(["outerMw"]);
+    expect(record.outcome).toBe("success");
+    expect(typeof record.durationMs).toBe("number");
+    expect(record.durationMs).toBeGreaterThanOrEqual(0);
+    expect(record.requestId).toBe("req-test");
+    expect(record.conversationId).toBe("conv-test");
+    expect(record.turnIndex).toBe(3);
+    // pluginName is only present when ctx carries one.
+    expect(record.pluginName).toBeUndefined();
+    // Error fields absent on success.
+    expect(record.errorName).toBeUndefined();
+    expect(record.errorMessage).toBeUndefined();
+    expect(record.errorStack).toBeUndefined();
+  });
+
+  test("error path records outcome=error + error fields + plugin name", async () => {
+    class Boom extends Error {
+      override readonly name = "BoomError";
+    }
+    const thrower: Middleware<Args, Result> = async () => {
+      throw new Boom("kaboom");
+    };
+    const terminal = async (_args: Args): Promise<Result> => ({ value: 0 });
+
+    await expect(
+      runPipeline(
+        "circuitBreaker",
+        [thrower],
+        terminal,
+        { value: 1 },
+        makeCtx({ pluginName: "noisy-plugin" }),
+        DEFAULT_TIMEOUTS.circuitBreaker,
+      ),
+    ).rejects.toBeInstanceOf(Boom);
+
+    expect(fakeLogger.calls.length).toBe(1);
+    const [record] = fakeLogger.calls[0]!;
+    expect(record.outcome).toBe("error");
+    expect(record.pipeline).toBe("circuitBreaker");
+    expect(record.errorName).toBe("BoomError");
+    expect(record.errorMessage).toBe("kaboom");
+    expect(typeof record.errorStack).toBe("string");
+    expect(record.pluginName).toBe("noisy-plugin");
+  });
+
+  test("timeout path records outcome=timeout + PluginTimeoutError fields", async () => {
+    const sleeper: Middleware<Args, Result> = async (_args, _next) =>
+      new Promise<Result>((resolve) => {
+        setTimeout(() => resolve({ value: 0 }), 200);
+      });
+    const terminal = async (_args: Args): Promise<Result> => ({ value: 0 });
+
+    await expect(
+      runPipeline(
+        "circuitBreaker",
+        [sleeper],
+        terminal,
+        { value: 1 },
+        makeCtx({ pluginName: "slow-plugin" }),
+        15,
+      ),
+    ).rejects.toBeInstanceOf(PluginTimeoutError);
+
+    expect(fakeLogger.calls.length).toBe(1);
+    const [record] = fakeLogger.calls[0]!;
+    expect(record.outcome).toBe("timeout");
+    expect(record.pipeline).toBe("circuitBreaker");
+    expect(record.errorName).toBe("PluginTimeoutError");
+    expect(String(record.errorMessage)).toContain("circuitBreaker");
+    expect(String(record.errorMessage)).toContain("slow-plugin");
+    expect(record.timeoutMs).toBe(15);
+    expect(record.pluginName).toBe("slow-plugin");
+  });
+
+  test("null timeout omits timeoutMs field from the log record", async () => {
+    const terminal = async (args: Args): Promise<Result> => ({
+      value: args.value,
+    });
+    await runPipeline(
+      "compaction",
+      [],
+      terminal,
+      { value: 5 },
+      makeCtx(),
+      DEFAULT_TIMEOUTS.compaction,
+    );
+
+    expect(fakeLogger.calls.length).toBe(1);
+    const [record] = fakeLogger.calls[0]!;
+    expect(record.pipeline).toBe("compaction");
+    expect(record.outcome).toBe("success");
+    expect(record.timeoutMs).toBeUndefined();
+  });
+
+  test("turnIndex is omitted when unset on the context", async () => {
+    const terminal = async (_args: Args): Promise<Result> => ({ value: 0 });
+    const ctxNoTurn = {
+      requestId: "r",
+      conversationId: "c",
+      turnIndex: undefined as unknown as number,
+      trust,
+      logger: fakeLogger,
+    } as TurnContext;
+    await runPipeline(
+      "circuitBreaker",
+      [],
+      terminal,
+      { value: 0 },
+      ctxNoTurn,
+      null,
+    );
+    const [record] = fakeLogger.calls[0]!;
+    expect(record.turnIndex).toBeUndefined();
+  });
+
+  test("chain list has one entry per middleware in registration order", async () => {
+    const a: Middleware<Args, Result> = async function outerA(args, next) {
+      return next(args);
+    };
+    const b: Middleware<Args, Result> = async function middleB(args, next) {
+      return next(args);
+    };
+    const c: Middleware<Args, Result> = async function innerC(args, next) {
+      return next(args);
+    };
+    const terminal = async (args: Args): Promise<Result> => ({
+      value: args.value,
+    });
+    await runPipeline(
+      "compaction",
+      [a, b, c],
+      terminal,
+      { value: 0 },
+      makeCtx(),
+      DEFAULT_TIMEOUTS.compaction,
+    );
+    const [record] = fakeLogger.calls[0]!;
+    expect(record.chain).toEqual(["outerA", "middleB", "innerC"]);
+  });
+});
+
+describe("DEFAULT_TIMEOUTS", () => {
+  test("matches the design-doc table exactly", () => {
+    expect(DEFAULT_TIMEOUTS).toEqual({
+      compaction: null,
+      overflowReduce: null,
+      circuitBreaker: null,
+    });
+  });
+});

package/src/__tests__/plugin-api-shim.test.ts

@@ -44,7 +44,10 @@ describe("buildShimSource", () => {
   });
 
   test("handles an empty export list (today's types-only surface)", () => {
-    const source = buildShimSource([], Symbol.for("vellum.plugin-api.v1"));
+    const source = buildShimSource(
+      [],
+      Symbol.for("vellum.plugin-api.v1"),
+    );
     expect(source).toBe(
       `const api = globalThis[Symbol.for("vellum.plugin-api.v1")];\n`,
     );
@@ -106,7 +109,7 @@ describe("ensurePluginApiShim", () => {
     const pluginDir = join(workspaceDir, "plugins", "fake-plugin");
     await mkdir(pluginDir, { recursive: true });
     await writeFile(
-      join(pluginDir, "probe.js"),
+      join(pluginDir, "register.js"),
       `import * as api from "@vellumai/plugin-api";\nexport { api };\n`,
     );
 
@@ -115,7 +118,7 @@ describe("ensurePluginApiShim", () => {
     // → plugin-api namespace. If any link in that chain is broken, this
     // import throws.
     const mod: { api: Record<string, unknown> } = await import(
-      join(pluginDir, "probe.js")
+      join(pluginDir, "register.js")
     );
     expect(mod.api).toBeDefined();
   });

package/src/__tests__/plugin-bootstrap.test.ts

@@ -38,11 +38,13 @@ import { RiskLevel } from "../permissions/types.js";
 import { registerDefaultPlugins } from "../plugins/defaults/index.js";
 import {
   closeRegistration,
+  getMiddlewaresFor,
   getRegisteredPlugins,
   registerPlugin,
   resetPluginRegistryForTests,
 } from "../plugins/registry.js";
 import {
+  type PipelineMiddlewareMap,
   type Plugin,
   PluginExecutionError,
   type PluginInitContext,
@@ -483,24 +485,33 @@ describe("plugin bootstrap", () => {
     expect(initFired).toBe(false);
   });
 
-  test("requiresFlag disabled: the skipped plugin is removed from the registry", async () => {
-    // Regression: a flag-gated skip must call `unregisterPlugin()` so the
-    // gated-off plugin does not linger in `registeredPlugins` with its
-    // `init()` never having fired to set up the state it depends on.
-    setOverridesForTesting({ "plugin-registry-disabled": false });
-
+  test("requiresFlag disabled: plugin middleware is dropped from the registry", async () => {
+    // Regression: prior to the unregisterPlugin() call on the flag-gated skip
+    // path, `getMiddlewaresFor()` iterated over every entry in
+    // `registeredPlugins` — so a gated-off plugin's middleware still ran on
+    // every pipeline invocation even though `init()` had never fired to set up
+    // the state it depended on.
+    setOverridesForTesting({ "plugin-middleware-disabled": false });
+
+    const gatedMiddleware: PipelineMiddlewareMap["compaction"] = async (
+      args,
+      next,
+    ) => next(args);
     const plugin = buildPlugin(
-      "gated-registry",
-      {},
-      { requiresFlag: ["plugin-registry-disabled"] },
+      "gated-middleware",
+      {
+        middleware: { compaction: gatedMiddleware },
+      },
+      { requiresFlag: ["plugin-middleware-disabled"] },
     );
     registerPlugin(plugin);
 
     await bootstrapPlugins();
 
-    // The gated-off plugin must not survive in the registry snapshot.
-    const names = getRegisteredPlugins().map((p) => p.manifest.name);
-    expect(names).not.toContain("gated-registry");
+    // The middleware slot must not expose the flag-gated plugin's contribution.
+    // The default plugins also contribute compaction middleware, so we key on
+    // identity rather than asserting an empty list.
+    expect(getMiddlewaresFor("compaction")).not.toContain(gatedMiddleware);
   });
 
   test("requiresFlag disabled: no shutdown hook entry installed for the skipped plugin", async () => {