@vellumai/assistant 0.8.8 → 0.8.9-staging.2

package/src/__tests__/compaction-pipeline.test.ts

@@ -1,210 +0,0 @@
-/**
- * Tests for the `compaction` plugin pipeline (PR 25).
- *
- * Covers:
- * - Default plugin delegates to the manager's `maybeCompact` and returns the
- *   same `ContextWindowResult` object the manager produced.
- * - A custom plugin layered on top can short-circuit before the terminal is
- *   reached and return a different summary, demonstrating that the pipeline
- *   slot is observable and replaceable without patching the manager.
- *
- * The tests drive `runPipeline` directly rather than going through the full
- * orchestrator — the integration path (conversation-agent-loop) is exercised
- * by `conversation-agent-loop-overflow.test.ts`, which must continue to pass
- * as the acceptance criterion for this PR.
- */
-
-import { describe, expect, test } from "bun:test";
-
-import type { TrustContext } from "../daemon/trust-context.js";
-import {
-  DEFAULT_COMPACTION_PLUGIN_NAME,
-  defaultCompactionTerminal,
-} from "../plugins/defaults/compaction/terminal.js";
-import { runPipeline } from "../plugins/pipeline.js";
-import {
-  type CompactionArgs,
-  type CompactionResult,
-  type Middleware,
-  PluginExecutionError,
-  type TurnContext,
-} from "../plugins/types.js";
-
-type ContextWindowResultShape = {
-  compacted: boolean;
-  summaryText: string;
-  messages: unknown[];
-  previousEstimatedInputTokens: number;
-  estimatedInputTokens: number;
-  maxInputTokens: number;
-  thresholdTokens: number;
-  compactedMessages: number;
-  compactedPersistedMessages: number;
-  summaryCalls: number;
-  summaryInputTokens: number;
-  summaryOutputTokens: number;
-  summaryModel: string;
-  reason?: string;
-};
-
-const trust: TrustContext = {
-  sourceChannel: "vellum",
-  trustClass: "guardian",
-};
-
-function makeTurnCtx(manager: {
-  maybeCompact: (...args: unknown[]) => Promise<unknown>;
-}): TurnContext {
-  return {
-    requestId: "req-compaction-test",
-    conversationId: "conv-compaction-test",
-    turnIndex: 0,
-    trust,
-    // `TurnContext.contextWindowManager` is a typed optional field; the
-    // default compaction plugin reads it directly without a cast.
-    contextWindowManager:
-      manager as unknown as TurnContext["contextWindowManager"],
-  };
-}
-
-function makeResult(
-  overrides: Partial<ContextWindowResultShape> = {},
-): ContextWindowResultShape {
-  return {
-    compacted: true,
-    summaryText: "default-summary",
-    messages: [],
-    previousEstimatedInputTokens: 1000,
-    estimatedInputTokens: 100,
-    maxInputTokens: 100000,
-    thresholdTokens: 80000,
-    compactedMessages: 3,
-    compactedPersistedMessages: 3,
-    summaryCalls: 1,
-    summaryInputTokens: 500,
-    summaryOutputTokens: 120,
-    summaryModel: "default-model",
-    ...overrides,
-  };
-}
-
-describe("compaction pipeline", () => {
-  test("default plugin delegates to the manager and returns its result unchanged", async () => {
-    const observed: {
-      messages: unknown;
-      signal: unknown;
-      options: unknown;
-    }[] = [];
-    const expected = makeResult({
-      summaryText: "manager-summary",
-      compactedMessages: 7,
-    });
-    const manager = {
-      maybeCompact: async (
-        messages: unknown,
-        signal: unknown,
-        options: unknown,
-      ) => {
-        observed.push({ messages, signal, options });
-        return expected;
-      },
-    };
-    const turnCtx = makeTurnCtx(manager);
-    const args: CompactionArgs = {
-      messages: [{ role: "user", content: "hi" }],
-      signal: new AbortController().signal,
-      options: { precomputedEstimate: 1234 },
-    };
-
-    // No middleware registered — the runner invokes the terminal directly.
-    const result = (await runPipeline<CompactionArgs, CompactionResult>(
-      "compaction",
-      [],
-      (innerArgs) => defaultCompactionTerminal(innerArgs, turnCtx),
-      args,
-      turnCtx,
-      30000,
-    )) as ContextWindowResultShape;
-
-    // Terminal forwarded args verbatim to the manager — except for
-    // `signal`, which the pipeline runner replaces with a signal linked
-    // to its internal timeout controller. The linked signal must forward
-    // caller-originated aborts, which is verified in the dedicated
-    // pipeline-runner abort-propagation tests.
-    expect(observed).toHaveLength(1);
-    expect(observed[0]!.messages).toBe(args.messages);
-    expect(observed[0]!.signal).toBeInstanceOf(AbortSignal);
-    expect(observed[0]!.options).toBe(args.options);
-
-    // Returned result is the manager's object, unmodified — no wrapping
-    // or shape transformation is allowed in the default path.
-    expect(result).toBe(expected);
-    expect(result.summaryText).toBe("manager-summary");
-    expect(result.compactedMessages).toBe(7);
-  });
-
-  test("custom plugin short-circuits to a different summary without touching the manager", async () => {
-    let managerCallCount = 0;
-    const manager = {
-      maybeCompact: async () => {
-        managerCallCount++;
-        return makeResult({ summaryText: "should-not-run" });
-      },
-    };
-    const turnCtx = makeTurnCtx(manager);
-
-    const custom: Middleware<CompactionArgs, CompactionResult> =
-      async function customCompaction(_args, _next, _ctx) {
-        // Short-circuit — omit the `next` call so the terminal never fires.
-        return makeResult({
-          summaryText: "custom-plugin-summary",
-          compactedMessages: 0,
-          summaryCalls: 0,
-          reason: "short-circuited by custom plugin",
-        });
-      };
-
-    const args: CompactionArgs = {
-      messages: [],
-      signal: undefined,
-      options: undefined,
-    };
-
-    const result = (await runPipeline<CompactionArgs, CompactionResult>(
-      "compaction",
-      [custom],
-      (innerArgs) => defaultCompactionTerminal(innerArgs, turnCtx),
-      args,
-      turnCtx,
-      30000,
-    )) as ContextWindowResultShape;
-
-    expect(managerCallCount).toBe(0);
-    expect(result.summaryText).toBe("custom-plugin-summary");
-    expect(result.reason).toBe("short-circuited by custom plugin");
-  });
-
-  test("default terminal surfaces a PluginExecutionError when the manager is missing", async () => {
-    // Build a turn context without the extension field so the default
-    // terminal's lenient read fails — this guards against a future refactor
-    // that removes the handle-attach helper in the orchestrator.
-    const turnCtxWithoutManager: TurnContext = {
-      requestId: "req-missing",
-      conversationId: "conv-missing",
-      turnIndex: 0,
-      trust,
-    };
-    const args: CompactionArgs = {
-      messages: [],
-      signal: undefined,
-      options: undefined,
-    };
-
-    await expect(
-      defaultCompactionTerminal(args, turnCtxWithoutManager),
-    ).rejects.toThrow(PluginExecutionError);
-    await expect(
-      defaultCompactionTerminal(args, turnCtxWithoutManager),
-    ).rejects.toThrow(DEFAULT_COMPACTION_PLUGIN_NAME);
-  });
-});

package/src/__tests__/compaction-timeout-recovery.test.ts

@@ -1,251 +0,0 @@
-/**
- * Tests for the compaction call-site recovery path added in JARVIS-587.
- *
- * When the `compaction` pipeline exceeds its 30s budget (manifest-wide
- * `DEFAULT_TIMEOUTS.compaction`), the pipeline runner throws
- * `PluginTimeoutError`. The three compaction call sites in
- * `conversation-agent-loop.ts` (start-of-turn, mid-loop, emergency) catch
- * that error, record the failure on the loop-held `CompactionCircuit`
- * (`recordOutcome(turn, true, ...)`) so the circuit breaker counts it, and degrade
- * gracefully.
- *
- * This file asserts the tight coupling between:
- *  (1) a `PluginTimeoutError`-driven failure and
- *  (2) the compaction circuit breaker's 3-strike threshold.
- *
- * The existing `circuit-breaker-pipeline.test.ts` already exercises the
- * breaker's transitions end-to-end. These tests verify that our new
- * catch-blocks feed the breaker the same `"failure"` outcome a normal
- * summary-LLM throw would, and that repeated timeouts therefore trip the
- * same 3-strike trip.
- */
-
-import { afterEach, beforeEach, describe, expect, test } from "bun:test";
-
-import {
-  type CircuitTurnInfo,
-  CompactionCircuit,
-} from "../agent/compaction-circuit.js";
-import type { ServerMessage } from "../daemon/message-protocol.js";
-import type { TrustContext } from "../daemon/trust-context.js";
-import { COMPACTION_CIRCUIT_FAILURE_THRESHOLD } from "../plugins/defaults/circuit-breaker/middlewares/circuitBreaker.js";
-import { defaultCircuitBreakerPlugin } from "../plugins/defaults/circuit-breaker/register.js";
-import { DEFAULT_TIMEOUTS, runPipeline } from "../plugins/pipeline.js";
-import {
-  getMiddlewaresFor,
-  registerPlugin,
-  resetPluginRegistryForTests,
-} from "../plugins/registry.js";
-import {
-  type CompactionArgs,
-  type CompactionResult,
-  type Middleware,
-  PluginTimeoutError,
-  type TurnContext,
-} from "../plugins/types.js";
-
-// ─── Fixtures ───────────────────────────────────────────────────────────────
-
-function makeCircuit(conversationId = "conv-timeout-test"): CompactionCircuit {
-  return new CompactionCircuit(conversationId);
-}
-
-const turn: CircuitTurnInfo = {
-  turnCount: 0,
-  trustContext: { sourceChannel: "vellum", trustClass: "guardian" },
-};
-
-const trust: TrustContext = {
-  sourceChannel: "vellum",
-  trustClass: "guardian",
-};
-
-function makeTurnCtx(conversationId: string): TurnContext {
-  return {
-    requestId: "req-timeout-test",
-    conversationId,
-    turnIndex: 0,
-    trust,
-  };
-}
-
-function collectEvents(): {
-  events: ServerMessage[];
-  onEvent: (msg: ServerMessage) => void;
-} {
-  const events: ServerMessage[] = [];
-  return { events, onEvent: (msg) => events.push(msg) };
-}
-
-// ─── Tests ──────────────────────────────────────────────────────────────────
-
-describe("compaction timeout recovery (JARVIS-587)", () => {
-  beforeEach(() => {
-    resetPluginRegistryForTests();
-    registerPlugin(defaultCircuitBreakerPlugin);
-  });
-
-  afterEach(() => {
-    resetPluginRegistryForTests();
-  });
-
-  test("runPipeline('compaction', ...) throws PluginTimeoutError on budget breach", async () => {
-    // Baseline: the compaction pipeline still surfaces PluginTimeoutError when
-    // its timer fires. This guards the outer race from silently swallowing
-    // the timeout when the inner call is aborted by our Part A wiring.
-    const hang: Middleware<CompactionArgs, CompactionResult> = async (
-      _args,
-      _next,
-    ) =>
-      new Promise<CompactionResult>(() => {
-        // intentionally never resolves
-      });
-
-    let caught: unknown;
-    try {
-      await runPipeline<CompactionArgs, CompactionResult>(
-        "compaction",
-        [hang],
-        async () => ({ compacted: false }) as unknown as CompactionResult,
-        { messages: [] as unknown, signal: undefined, options: undefined },
-        makeTurnCtx("conv-budget-breach"),
-        // Override the manifest timeout to keep the test fast.
-        20,
-      );
-    } catch (err) {
-      caught = err;
-    }
-    expect(caught).toBeInstanceOf(PluginTimeoutError);
-    expect((caught as PluginTimeoutError).pipeline).toBe("compaction");
-  });
-
-  test("recordOutcome(failed=true) driven by PluginTimeoutError trips the breaker at the 3rd strike", async () => {
-    // Simulates the production sequence: each mid-loop compaction hits the
-    // pipeline's 30s ceiling, the catch path calls
-    // `circuit.recordOutcome(turn, true, onEvent)`. After three such catches
-    // the circuit breaker must be open — matching the same invariant the
-    // existing breaker test file exercises for normal summary-LLM throws.
-    const circuit = makeCircuit();
-    const { onEvent, events } = collectEvents();
-
-    // First two timeouts — circuit still closed.
-    await circuit.recordOutcome(turn, true, onEvent);
-    await circuit.recordOutcome(turn, true, onEvent);
-    expect(circuit.consecutiveCompactionFailures).toBe(2);
-    expect(circuit.compactionCircuitOpenUntil).toBeNull();
-    expect(events).toHaveLength(0);
-
-    // Third timeout — breaker trips and emits the canonical transition event.
-    await circuit.recordOutcome(turn, true, onEvent);
-    expect(circuit.consecutiveCompactionFailures).toBe(
-      COMPACTION_CIRCUIT_FAILURE_THRESHOLD,
-    );
-    expect(circuit.compactionCircuitOpenUntil).not.toBeNull();
-    expect(events).toHaveLength(1);
-    expect(events[0]).toEqual({
-      type: "compaction_circuit_open",
-      conversationId: circuit.conversationId,
-      reason: "3_consecutive_failures",
-      openUntil: circuit.compactionCircuitOpenUntil as number,
-    });
-  });
-
-  test("a successful compaction after two timeouts resets the counter", async () => {
-    // The recovery path doesn't interfere with the breaker's normal reset —
-    // once a compaction call eventually succeeds, the streak is broken and
-    // the next failure starts counting from 1.
-    const circuit = makeCircuit();
-    const { onEvent } = collectEvents();
-
-    await circuit.recordOutcome(turn, true, onEvent);
-    await circuit.recordOutcome(turn, true, onEvent);
-    expect(circuit.consecutiveCompactionFailures).toBe(2);
-
-    await circuit.recordOutcome(turn, false, onEvent);
-    expect(circuit.consecutiveCompactionFailures).toBe(0);
-    expect(circuit.compactionCircuitOpenUntil).toBeNull();
-
-    await circuit.recordOutcome(turn, true, onEvent);
-    expect(circuit.consecutiveCompactionFailures).toBe(1);
-  });
-
-  test("compaction call-site recovery remains defense-in-depth even when DEFAULT_TIMEOUTS.compaction is null", () => {
-    // At the time this PR landed, `DEFAULT_TIMEOUTS.compaction` was null
-    // (pipeline timeouts globally disabled — see #27608). That makes the
-    // call-site catch blocks unreachable in production right now, but the
-    // catch blocks still matter: any future reintroduction of a per-pipeline
-    // compaction timeout immediately benefits from circuit-breaker recording
-    // and graceful-degradation without needing to re-touch every call site.
-    //
-    // This test just documents the current value so a future change that
-    // reintroduces a timeout must also decide (intentionally) whether the
-    // recovery path should continue to fire.
-    const value = DEFAULT_TIMEOUTS.compaction;
-    expect(value === null || typeof value === "number").toBe(true);
-  });
-});
-
-describe("abort propagation end-to-end (Part A + updateSummary fallback)", () => {
-  beforeEach(() => {
-    resetPluginRegistryForTests();
-    registerPlugin(defaultCircuitBreakerPlugin);
-  });
-
-  afterEach(() => {
-    resetPluginRegistryForTests();
-  });
-
-  test("caller-provided signal is replaced with a linked signal that fires on timeout", async () => {
-    // Minimal proof that Part A actually wires the signal through the
-    // compaction pipeline: when the pipeline runner's timer fires, the
-    // signal seen by the inner terminal is aborted — allowing
-    // `updateSummary`'s try/catch around `provider.sendMessage` to trigger
-    // the local-fallback path instead of the call hanging indefinitely.
-    let observedSignal: AbortSignal | undefined;
-    const waitForAbort: Middleware<CompactionArgs, CompactionResult> = async (
-      args,
-      _next,
-    ) => {
-      observedSignal = args.signal;
-      return new Promise<CompactionResult>((_resolve, reject) => {
-        args.signal?.addEventListener("abort", () => {
-          reject(
-            Object.assign(new Error("aborted by signal"), {
-              name: "AbortError",
-            }),
-          );
-        });
-      });
-    };
-
-    const callerController = new AbortController();
-
-    let caught: unknown;
-    try {
-      await runPipeline<CompactionArgs, CompactionResult>(
-        "compaction",
-        [waitForAbort, ...getMiddlewaresFor("compaction")],
-        async () => ({ compacted: false }) as unknown as CompactionResult,
-        {
-          messages: [] as unknown,
-          signal: callerController.signal,
-          options: undefined,
-        },
-        makeTurnCtx("conv-abort-test"),
-        20,
-      );
-    } catch (err) {
-      caught = err;
-    }
-
-    expect(caught).toBeInstanceOf(PluginTimeoutError);
-    expect(observedSignal).toBeDefined();
-    // The runner swaps the caller's signal for a linked signal — the two are
-    // distinct objects but both should end up aborted once the timer fires.
-    expect(observedSignal).not.toBe(callerController.signal);
-    expect(observedSignal!.aborted).toBe(true);
-    // Caller's own signal is untouched — pipeline timeout does not cascade
-    // outward onto the caller's controller.
-    expect(callerController.signal.aborted).toBe(false);
-  });
-});