@vellumai/assistant 0.5.16 → 0.6.1

package/src/__tests__/context-memory-e2e.test.ts

@@ -1,415 +0,0 @@
-import {
-  afterAll,
-  beforeAll,
-  beforeEach,
-  describe,
-  expect,
-  mock,
-  test,
-} from "bun:test";
-
-import { DEFAULT_CONFIG } from "../config/defaults.js";
-import { estimatePromptTokens } from "../context/token-estimator.js";
-import { ContextWindowManager } from "../context/window-manager.js";
-import { getDb, initializeDb, resetDb } from "../memory/db.js";
-import { buildMemoryQuery } from "../memory/query-builder.js";
-import { computeRecallBudget } from "../memory/retrieval-budget.js";
-import { buildMemoryRecall } from "../memory/retriever.js";
-import {
-  conversations,
-  memoryItems,
-  memoryItemSources,
-  messages,
-} from "../memory/schema.js";
-import type { Message, Provider } from "../providers/types.js";
-
-mock.module("../util/logger.js", () => ({
-  getLogger: () =>
-    new Proxy({} as Record<string, unknown>, {
-      get: () => () => {},
-    }),
-}));
-
-mock.module("../memory/qdrant-client.js", () => ({
-  getQdrantClient: () => ({
-    searchWithFilter: async () => [],
-    hybridSearch: async () => [],
-    upsertPoints: async () => {},
-    deletePoints: async () => {},
-  }),
-  initQdrantClient: () => {},
-}));
-
-function makeSummaryProvider(counter: { calls: number }): Provider {
-  return {
-    name: "mock",
-    async sendMessage() {
-      counter.calls += 1;
-      return {
-        content: [
-          {
-            type: "text",
-            text: "## Goals\n- Keep memory stable\n## Decisions\n- Use progressive rollouts",
-          },
-        ],
-        model: "mock-context-model",
-        usage: { inputTokens: 300, outputTokens: 60 },
-        stopReason: "end_turn",
-      };
-    },
-  };
-}
-
-function makeLongHistory(turns: number, finalUserText: string): Message[] {
-  const rows: Message[] = [];
-  for (let i = 0; i < turns; i++) {
-    rows.push({
-      role: "user",
-      content: [
-        {
-          type: "text",
-          text: `[U${i}] Apollo planning thread ${i}: discuss rollout safeguards and runbooks.`,
-        },
-      ],
-    });
-    rows.push({
-      role: "assistant",
-      content: [
-        {
-          type: "text",
-          text: `[A${i}] Captured safeguards ${i}: staged rollout, monitoring, and rollback readiness.`,
-        },
-      ],
-    });
-  }
-  rows.push({
-    role: "user",
-    content: [{ type: "text", text: finalUserText }],
-  });
-  return rows;
-}
-
-function insertSegment(
-  id: string,
-  messageId: string,
-  conversationId: string,
-  role: string,
-  text: string,
-  createdAt: number,
-): void {
-  const db = getDb();
-  db.run(`
-    INSERT INTO memory_segments (
-      id, message_id, conversation_id, role, segment_index, text, token_estimate, scope_id, created_at, updated_at
-    ) VALUES (
-      '${id}', '${messageId}', '${conversationId}', '${role}', 0, '${text.replace(
-        /'/g,
-        "''",
-      )}', ${Math.max(
-        6,
-        Math.ceil(text.split(/\s+/).length * 1.3),
-      )}, 'default', ${createdAt}, ${createdAt}
-    )
-  `);
-}
-
-function insertMemoryItem(opts: {
-  id: string;
-  kind: string;
-  subject: string;
-  statement: string;
-  confidence: number;
-  importance: number;
-  firstSeenAt: number;
-  lastSeenAt: number;
-}): void {
-  const db = getDb();
-  db.insert(memoryItems)
-    .values({
-      id: opts.id,
-      kind: opts.kind,
-      subject: opts.subject,
-      statement: opts.statement,
-      status: "active",
-      confidence: opts.confidence,
-      importance: opts.importance,
-      fingerprint: `fp-${opts.id}`,
-      firstSeenAt: opts.firstSeenAt,
-      lastSeenAt: opts.lastSeenAt,
-      lastUsedAt: null,
-      scopeId: "default",
-    })
-    .run();
-}
-
-describe("Context + Memory E2E regression", () => {
-  beforeAll(() => {
-    initializeDb();
-  });
-
-  beforeEach(() => {
-    const db = getDb();
-    db.run("DELETE FROM memory_item_sources");
-    db.run("DELETE FROM memory_embeddings");
-    db.run("DELETE FROM memory_items");
-
-    db.run("DELETE FROM memory_segments");
-    db.run("DELETE FROM messages");
-    db.run("DELETE FROM conversations");
-    db.run("DELETE FROM memory_jobs");
-    db.run("DELETE FROM memory_checkpoints");
-  });
-
-  afterAll(() => {
-    resetDb();
-  });
-
-  test("one-turn flow compacts context, enforces dynamic budget, and recalls relation-linked memory", async () => {
-    const db = getDb();
-    const conversationId = "conv-context-memory-e2e";
-    const now = 1_700_900_000_000;
-    const currentMessageId = "msg-e2e-current";
-    const currentUserText =
-      "For Apollo, what rollout should we use this week? Also tracking note: Apollo secret code 123XYZ.";
-
-    db.insert(conversations)
-      .values({
-        id: conversationId,
-        title: null,
-        createdAt: now,
-        updatedAt: now,
-        totalInputTokens: 0,
-        totalOutputTokens: 0,
-        totalEstimatedCost: 0,
-        contextSummary: null,
-        contextCompactedMessageCount: 0,
-        contextCompactedAt: null,
-      })
-      .run();
-
-    const seededMessages = [
-      {
-        id: "msg-e2e-user-2",
-        role: "user",
-        text: "Historical deployment note kept for audit context.",
-        createdAt: now + 2_000,
-      },
-      {
-        id: "msg-e2e-user-10",
-        role: "user",
-        text: "Sprint review confirmed Apollo rollout policy and guardrails.",
-        createdAt: now + 10_000,
-      },
-      {
-        id: "msg-e2e-user-12",
-        role: "user",
-        text: "Playbook update captured HermesGate ramp guidance.",
-        createdAt: now + 12_000,
-      },
-      {
-        id: "msg-e2e-assistant-12",
-        role: "assistant",
-        text: "Acknowledged rollout notes and linked them to operations memory.",
-        createdAt: now + 12_001,
-      },
-    ] as const;
-
-    for (const row of seededMessages) {
-      db.insert(messages)
-        .values({
-          id: row.id,
-          conversationId,
-          role: row.role,
-          content: JSON.stringify([{ type: "text", text: row.text }]),
-          createdAt: row.createdAt,
-        })
-        .run();
-      insertSegment(
-        `seg-${row.id}`,
-        row.id,
-        conversationId,
-        row.role,
-        row.text,
-        row.createdAt,
-      );
-    }
-
-    db.insert(messages)
-      .values({
-        id: currentMessageId,
-        conversationId,
-        role: "user",
-        content: JSON.stringify([{ type: "text", text: currentUserText }]),
-        createdAt: now + 100_000,
-      })
-      .run();
-    insertSegment(
-      "seg-e2e-current",
-      currentMessageId,
-      conversationId,
-      "user",
-      currentUserText,
-      now + 100_000,
-    );
-
-    insertMemoryItem({
-      id: "item-apollo-direct",
-      kind: "preference",
-      subject: "apollo rollout policy",
-      statement:
-        "Apollo rollout policy favors staged canary releases with guarded traffic ramps.",
-      confidence: 0.95,
-      importance: 0.95,
-      firstSeenAt: now + 10_000,
-      lastSeenAt: now + 50_000,
-    });
-    db.insert(memoryItemSources)
-      .values({
-        memoryItemId: "item-apollo-direct",
-        messageId: "msg-e2e-user-10",
-        evidence: "User confirmed policy in sprint review",
-        createdAt: now + 10_000,
-      })
-      .run();
-
-    insertMemoryItem({
-      id: "item-hermes-relation",
-      kind: "identity",
-      subject: "hermes rollout execution",
-      statement:
-        "HermesGate rollout guidance: start at 5% traffic and promote only after error budget checks pass.",
-      confidence: 0.8,
-      importance: 0.6,
-      firstSeenAt: now + 12_000,
-      lastSeenAt: now + 52_000,
-    });
-    db.insert(memoryItemSources)
-      .values({
-        memoryItemId: "item-hermes-relation",
-        messageId: "msg-e2e-user-12",
-        evidence: "Rollout playbook notes",
-        createdAt: now + 12_000,
-      })
-      .run();
-
-    insertMemoryItem({
-      id: "item-apollo-stale",
-      kind: "event",
-      subject: "legacy deploy process",
-      statement:
-        "Legacy note from years ago: Apollo deployments were manual and unmonitored.",
-      confidence: 0.6,
-      importance: 0.3,
-      firstSeenAt: now - 400 * 24 * 60 * 60 * 1000,
-      lastSeenAt: now - 390 * 24 * 60 * 60 * 1000,
-    });
-    db.insert(memoryItemSources)
-      .values({
-        memoryItemId: "item-apollo-stale",
-        messageId: "msg-e2e-user-2",
-        evidence: "Historical note",
-        createdAt: now - 390 * 24 * 60 * 60 * 1000,
-      })
-      .run();
-
-    insertMemoryItem({
-      id: "item-apollo-secret",
-      kind: "identity",
-      subject: "apollo secret",
-      statement: "Current-turn secret: Apollo code is 123XYZ.",
-      confidence: 0.99,
-      importance: 0.99,
-      firstSeenAt: now + 100_000,
-      lastSeenAt: now + 100_000,
-    });
-    db.insert(memoryItemSources)
-      .values({
-        memoryItemId: "item-apollo-secret",
-        messageId: currentMessageId,
-        evidence: "Current turn only",
-        createdAt: now + 100_000,
-      })
-      .run();
-
-    const summaryCounter = { calls: 0 };
-    const provider = makeSummaryProvider(summaryCounter);
-    const systemPrompt = "System prompt for context + memory e2e";
-    const history = makeLongHistory(60, currentUserText);
-    const manager = new ContextWindowManager({
-      provider,
-      systemPrompt,
-      config: {
-        ...DEFAULT_CONFIG.contextWindow,
-        maxInputTokens: 5200,
-        targetBudgetRatio: 0.55,
-        compactThreshold: 0.55,
-        summaryBudgetRatio: 0.05,
-      },
-    });
-
-    const compacted = await manager.maybeCompact(history);
-    expect(compacted.compacted).toBe(true);
-    expect(compacted.summaryCalls).toBe(summaryCounter.calls);
-    expect(compacted.estimatedInputTokens).toBeLessThan(
-      compacted.previousEstimatedInputTokens,
-    );
-
-    const recallConfig = {
-      ...DEFAULT_CONFIG,
-      contextWindow: { ...DEFAULT_CONFIG.contextWindow, maxInputTokens: 5200 },
-      memory: {
-        ...DEFAULT_CONFIG.memory,
-        embeddings: {
-          ...DEFAULT_CONFIG.memory.embeddings,
-          provider: "openai" as const,
-          required: false,
-        },
-        retrieval: {
-          ...DEFAULT_CONFIG.memory.retrieval,
-          maxInjectTokens: 900,
-          dynamicBudget: {
-            enabled: true,
-            minInjectTokens: 180,
-            maxInjectTokens: 320,
-            targetHeadroomTokens: 700,
-          },
-        },
-      },
-    };
-
-    const estimatedPromptTokens = estimatePromptTokens(
-      compacted.messages,
-      systemPrompt,
-      { providerName: provider.name },
-    );
-
-    const recallBudget = computeRecallBudget({
-      estimatedPromptTokens,
-      maxInputTokens: recallConfig.contextWindow.maxInputTokens,
-      targetHeadroomTokens:
-        recallConfig.memory.retrieval.dynamicBudget.targetHeadroomTokens,
-      minInjectTokens:
-        recallConfig.memory.retrieval.dynamicBudget.minInjectTokens,
-      maxInjectTokens:
-        recallConfig.memory.retrieval.dynamicBudget.maxInjectTokens,
-    });
-
-    const recallQuery = buildMemoryQuery(currentUserText, compacted.messages);
-    const recall = await buildMemoryRecall(
-      recallQuery,
-      conversationId,
-      recallConfig,
-      {
-        excludeMessageIds: [currentMessageId],
-        maxInjectTokensOverride: recallBudget,
-      },
-    );
-
-    expect(recall.injectedTokens).toBeLessThanOrEqual(recallBudget);
-
-    // With Qdrant mocked empty, no candidates are found and injectedText is
-    // empty — the pipeline requires at least tier-2 quality to inject.
-    // Verify current-turn secrets never leak regardless.
-    expect(recall.injectedText).not.toContain("123XYZ");
-  });
-});

package/src/__tests__/journal-context.test.ts

@@ -1,268 +0,0 @@
-import { mkdirSync, rmSync, statSync, writeFileSync } from "node:fs";
-import { join } from "node:path";
-import { afterEach, beforeEach, describe, expect, test } from "bun:test";
-
-const TEST_DIR = process.env.VELLUM_WORKSPACE_DIR!;
-const { buildJournalContext, formatJournalAbsoluteTime } = await import(
-  "../prompts/journal-context.js"
-);
-
-/** Small delay to ensure distinct file birthtimes on APFS. */
-const tick = () => Bun.sleep(5);
-
-describe("formatJournalAbsoluteTime", () => {
-  test("formats a timestamp as MM/DD/YY HH:MM", () => {
-    // 2025-03-15 14:30:00
-    const ts = new Date(2025, 2, 15, 14, 30, 0).getTime();
-    expect(formatJournalAbsoluteTime(ts)).toBe("03/15/25 14:30");
-  });
-
-  test("zero-pads single-digit months, days, hours, and minutes", () => {
-    // 2025-01-05 09:05:00
-    const ts = new Date(2025, 0, 5, 9, 5, 0).getTime();
-    expect(formatJournalAbsoluteTime(ts)).toBe("01/05/25 09:05");
-  });
-
-  test("handles midnight", () => {
-    const ts = new Date(2025, 5, 20, 0, 0, 0).getTime();
-    expect(formatJournalAbsoluteTime(ts)).toBe("06/20/25 00:00");
-  });
-});
-
-describe("buildJournalContext", () => {
-  const journalDir = join(TEST_DIR, "journal");
-
-  beforeEach(() => {
-    mkdirSync(journalDir, { recursive: true });
-  });
-
-  afterEach(() => {
-    rmSync(TEST_DIR, { recursive: true, force: true });
-  });
-
-  test("returns null when maxEntries is 0", () => {
-    const userDir = join(journalDir, "testuser");
-    mkdirSync(userDir, { recursive: true });
-    writeFileSync(join(userDir, "entry.md"), "content");
-    expect(buildJournalContext(0, "testuser")).toBeNull();
-  });
-
-  test("returns null when maxEntries is negative", () => {
-    const userDir = join(journalDir, "testuser");
-    mkdirSync(userDir, { recursive: true });
-    writeFileSync(join(userDir, "entry.md"), "content");
-    expect(buildJournalContext(-1, "testuser")).toBeNull();
-  });
-
-  test("returns null when journal directory does not exist", () => {
-    rmSync(journalDir, { recursive: true, force: true });
-    expect(buildJournalContext(10, "testuser")).toBeNull();
-  });
-
-  test("returns null when journal directory has no .md files", () => {
-    const userDir = join(journalDir, "testuser");
-    mkdirSync(userDir, { recursive: true });
-    writeFileSync(join(userDir, "notes.txt"), "not markdown");
-    expect(buildJournalContext(10, "testuser")).toBeNull();
-  });
-
-  test("excludes README.md (case-insensitive)", () => {
-    const userDir = join(journalDir, "testuser");
-    mkdirSync(userDir, { recursive: true });
-    writeFileSync(join(userDir, "README.md"), "readme content");
-    writeFileSync(join(userDir, "readme.md"), "readme content lower");
-    expect(buildJournalContext(10, "testuser")).toBeNull();
-  });
-
-  test("returns formatted journal context with single entry", () => {
-    const userDir = join(journalDir, "testuser");
-    mkdirSync(userDir, { recursive: true });
-    writeFileSync(join(userDir, "goals.md"), "My goals for this week.");
-    const result = buildJournalContext(10, "testuser");
-    expect(result).not.toBeNull();
-    expect(result).toContain("# Journal");
-    expect(result).toContain(
-      "Your journal entries, most recent first. These are YOUR words from past conversations.",
-    );
-    expect(result).toContain("## goals.md — MOST RECENT");
-    expect(result).toContain("My goals for this week.");
-    // Single entry, window not full — should NOT have LEAVING CONTEXT
-    expect(result).not.toContain("LEAVING CONTEXT");
-  });
-
-  test("sorts entries by creation time, newest first", async () => {
-    const userDir = join(journalDir, "testuser");
-    mkdirSync(userDir, { recursive: true });
-    // Small delays between writes ensure distinct birthtimes.
-    writeFileSync(join(userDir, "old.md"), "old entry");
-    await tick();
-    writeFileSync(join(userDir, "mid.md"), "mid entry");
-    await tick();
-    writeFileSync(join(userDir, "new.md"), "new entry");
-
-    const result = buildJournalContext(10, "testuser")!;
-    const newIdx = result.indexOf("new.md");
-    const midIdx = result.indexOf("mid.md");
-    const oldIdx = result.indexOf("old.md");
-    expect(newIdx).toBeLessThan(midIdx);
-    expect(midIdx).toBeLessThan(oldIdx);
-  });
-
-  test("marks most recent entry with MOST RECENT", async () => {
-    const userDir = join(journalDir, "testuser");
-    mkdirSync(userDir, { recursive: true });
-    writeFileSync(join(userDir, "older.md"), "older");
-    await tick();
-    writeFileSync(join(userDir, "newest.md"), "newest");
-
-    const result = buildJournalContext(10, "testuser")!;
-    expect(result).toContain("## newest.md — MOST RECENT");
-    expect(result).not.toContain("## older.md — MOST RECENT");
-  });
-
-  test("marks oldest entry with LEAVING CONTEXT when window is full", async () => {
-    const userDir = join(journalDir, "testuser");
-    mkdirSync(userDir, { recursive: true });
-    // Create in chronological order: c (oldest), b, a (newest)
-    writeFileSync(join(userDir, "c.md"), "entry c");
-    await tick();
-    writeFileSync(join(userDir, "b.md"), "entry b");
-    await tick();
-    writeFileSync(join(userDir, "a.md"), "entry a");
-
-    // maxEntries = 3 matches the number of files, so window is full
-    const result = buildJournalContext(3, "testuser")!;
-    expect(result).toContain("## a.md — MOST RECENT");
-    expect(result).toContain("## c.md — LEAVING CONTEXT");
-    expect(result).toContain(
-      "NOTE: This is the oldest entry in your active context.",
-    );
-    expect(result).toContain(
-      "carry forward anything from here that still matters to you",
-    );
-  });
-
-  test("does NOT mark oldest entry with LEAVING CONTEXT when window is not full", async () => {
-    const userDir = join(journalDir, "testuser");
-    mkdirSync(userDir, { recursive: true });
-    writeFileSync(join(userDir, "b.md"), "entry b");
-    await tick();
-    writeFileSync(join(userDir, "a.md"), "entry a");
-
-    // maxEntries = 5, only 2 files — window is NOT full
-    const result = buildJournalContext(5, "testuser")!;
-    expect(result).toContain("## a.md — MOST RECENT");
-    expect(result).not.toContain("LEAVING CONTEXT");
-  });
-
-  test("limits entries to maxEntries", async () => {
-    const userDir = join(journalDir, "testuser");
-    mkdirSync(userDir, { recursive: true });
-    // Create files 0-4 sequentially; file 4 is newest
-    for (let i = 0; i < 5; i++) {
-      writeFileSync(join(userDir, `entry-${i}.md`), `content ${i}`);
-      if (i < 4) await tick();
-    }
-
-    const result = buildJournalContext(3, "testuser")!;
-    // Should contain only 3 newest entries (entry-4, entry-3, entry-2)
-    expect(result).toContain("entry-4.md");
-    expect(result).toContain("entry-3.md");
-    expect(result).toContain("entry-2.md");
-    expect(result).not.toContain("entry-1.md");
-    expect(result).not.toContain("entry-0.md");
-  });
-
-  test("maxEntries=1 with exactly one entry marks it MOST RECENT, not LEAVING CONTEXT", () => {
-    const userDir = join(journalDir, "testuser");
-    mkdirSync(userDir, { recursive: true });
-    writeFileSync(join(userDir, "solo.md"), "only entry");
-    const result = buildJournalContext(1, "testuser")!;
-    expect(result).toContain("## solo.md — MOST RECENT");
-    expect(result).not.toContain("LEAVING CONTEXT");
-  });
-
-  test("includes absolute timestamps in headers", () => {
-    const userDir = join(journalDir, "testuser");
-    mkdirSync(userDir, { recursive: true });
-    writeFileSync(join(userDir, "recent.md"), "recent content");
-
-    const result = buildJournalContext(10, "testuser")!;
-    const birthtime = statSync(join(userDir, "recent.md")).birthtimeMs;
-    const expected = formatJournalAbsoluteTime(birthtime);
-    expect(result).toContain(expected);
-  });
-
-  test("middle entries have plain headers with timestamps", async () => {
-    const userDir = join(journalDir, "testuser");
-    mkdirSync(userDir, { recursive: true });
-    // Create in chronological order: last (oldest), middle, first (newest)
-    writeFileSync(join(userDir, "last.md"), "last");
-    await tick();
-    writeFileSync(join(userDir, "middle.md"), "middle");
-    await tick();
-    writeFileSync(join(userDir, "first.md"), "first");
-
-    const result = buildJournalContext(3, "testuser")!;
-    // Middle entry should have plain header format (no MOST RECENT, no LEAVING CONTEXT)
-    // Format: ## middle.md (MM/DD/YY HH:MM)
-    expect(result).toMatch(
-      /## middle\.md \(\d{2}\/\d{2}\/\d{2} \d{2}:\d{2}\)/,
-    );
-  });
-
-  // --- Per-user scoping tests ---
-
-  test("reads from per-user directory when userSlug is provided", () => {
-    const aliceDir = join(journalDir, "alice");
-    mkdirSync(aliceDir, { recursive: true });
-    writeFileSync(join(aliceDir, "thoughts.md"), "Alice's thoughts");
-    writeFileSync(join(aliceDir, "plans.md"), "Alice's plans");
-
-    const result = buildJournalContext(10, "alice");
-    expect(result).not.toBeNull();
-    expect(result).toContain("Alice's thoughts");
-    expect(result).toContain("Alice's plans");
-  });
-
-  test("returns null when userSlug is provided but directory does not exist", () => {
-    // journal/bob/ does not exist
-    const result = buildJournalContext(10, "bob");
-    expect(result).toBeNull();
-  });
-
-  test("returns null when no userSlug is provided", () => {
-    // Even if root journal/ has entries, no slug means null
-    writeFileSync(join(journalDir, "orphan.md"), "orphan entry");
-    const result = buildJournalContext(10);
-    expect(result).toBeNull();
-  });
-
-  test("returns null when userSlug is null", () => {
-    writeFileSync(join(journalDir, "orphan.md"), "orphan entry");
-    const result = buildJournalContext(10, null);
-    expect(result).toBeNull();
-  });
-
-  test("includes write-path directive in header when userSlug is provided", () => {
-    const aliceDir = join(journalDir, "alice");
-    mkdirSync(aliceDir, { recursive: true });
-    writeFileSync(join(aliceDir, "entry.md"), "some content");
-
-    const result = buildJournalContext(10, "alice")!;
-    expect(result).toContain("**Write new entries to:** `journal/alice/`");
-  });
-
-  test("sanitizes path-traversal in userSlug", () => {
-    // basename("../etc") => "etc", so it should read from journal/etc/
-    const etcDir = join(journalDir, "etc");
-    mkdirSync(etcDir, { recursive: true });
-    writeFileSync(join(etcDir, "safe.md"), "safe content");
-
-    const result = buildJournalContext(10, "../etc");
-    expect(result).not.toBeNull();
-    expect(result).toContain("safe content");
-    // Should reference the sanitized path, not the traversal attempt
-    expect(result).toContain("`journal/etc/`");
-  });
-});