@vellumai/assistant 0.5.7 → 0.5.9

package/src/__tests__/gateway-client-managed-outbound.test.ts

@@ -1,6 +1,9 @@
 import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
 
-import { deliverChannelReply } from "../runtime/gateway-client.js";
+import {
+  ChannelDeliveryError,
+  deliverChannelReply,
+} from "../runtime/gateway-client.js";
 
 type FetchCall = {
   url: string;
@@ -163,4 +166,79 @@ describe("gateway-client managed outbound lane", () => {
       text: "standard gateway callback",
     });
   });
+
+  test("throws ChannelDeliveryError with userMessage when gateway returns JSON error with userMessage", async () => {
+    globalThis.fetch = mock(async () => {
+      return new Response(
+        JSON.stringify({
+          error: "Permission denied",
+          userMessage:
+            "The bot is not a member of this channel. Please invite it first.",
+        }),
+        { status: 403 },
+      );
+    }) as unknown as typeof globalThis.fetch;
+
+    let caught: unknown;
+    try {
+      await deliverChannelReply("https://gateway.test/deliver/slack", {
+        chatId: "C123",
+        text: "hello",
+      });
+    } catch (err) {
+      caught = err;
+    }
+
+    expect(caught).toBeInstanceOf(ChannelDeliveryError);
+    const deliveryError = caught as ChannelDeliveryError;
+    expect(deliveryError.statusCode).toBe(403);
+    expect(deliveryError.userMessage).toBe(
+      "The bot is not a member of this channel. Please invite it first.",
+    );
+    expect(deliveryError.message).toContain("403");
+  });
+
+  test("throws ChannelDeliveryError without userMessage when gateway returns JSON error without userMessage", async () => {
+    globalThis.fetch = mock(async () => {
+      return new Response(JSON.stringify({ error: "Delivery failed" }), {
+        status: 502,
+      });
+    }) as unknown as typeof globalThis.fetch;
+
+    let caught: unknown;
+    try {
+      await deliverChannelReply("https://gateway.test/deliver/slack", {
+        chatId: "C123",
+        text: "hello",
+      });
+    } catch (err) {
+      caught = err;
+    }
+
+    expect(caught).toBeInstanceOf(ChannelDeliveryError);
+    const deliveryError = caught as ChannelDeliveryError;
+    expect(deliveryError.statusCode).toBe(502);
+    expect(deliveryError.userMessage).toBeUndefined();
+  });
+
+  test("throws ChannelDeliveryError without userMessage when gateway returns non-JSON error", async () => {
+    globalThis.fetch = mock(async () => {
+      return new Response("Internal Server Error", { status: 500 });
+    }) as unknown as typeof globalThis.fetch;
+
+    let caught: unknown;
+    try {
+      await deliverChannelReply("https://gateway.test/deliver/slack", {
+        chatId: "C123",
+        text: "hello",
+      });
+    } catch (err) {
+      caught = err;
+    }
+
+    expect(caught).toBeInstanceOf(ChannelDeliveryError);
+    const deliveryError = caught as ChannelDeliveryError;
+    expect(deliveryError.statusCode).toBe(500);
+    expect(deliveryError.userMessage).toBeUndefined();
+  });
 });

package/src/__tests__/journal-context.test.ts

@@ -0,0 +1,335 @@
+import { mkdirSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
+
+const TEST_DIR = join(tmpdir(), `vellum-journal-test-${crypto.randomUUID()}`);
+
+// eslint-disable-next-line @typescript-eslint/no-require-imports
+const realPlatform = require("../util/platform.js");
+mock.module("../util/platform.js", () => ({
+  ...realPlatform,
+  getWorkspaceDir: () => TEST_DIR,
+}));
+
+const {
+  buildJournalContext,
+  formatJournalRelativeTime,
+  formatJournalAbsoluteTime,
+} = await import("../prompts/journal-context.js");
+
+/** Small delay to ensure distinct file birthtimes on APFS. */
+const tick = () => Bun.sleep(5);
+
+describe("formatJournalRelativeTime", () => {
+  test("returns 'just now' for times less than 60 seconds ago", () => {
+    const now = Date.now();
+    expect(formatJournalRelativeTime(now - 30_000)).toBe("just now");
+    expect(formatJournalRelativeTime(now - 1_000)).toBe("just now");
+    expect(formatJournalRelativeTime(now)).toBe("just now");
+  });
+
+  test("returns minutes for times between 1-59 minutes ago", () => {
+    const now = Date.now();
+    expect(formatJournalRelativeTime(now - 60_000)).toBe("1 minute ago");
+    expect(formatJournalRelativeTime(now - 5 * 60_000)).toBe("5 minutes ago");
+    expect(formatJournalRelativeTime(now - 59 * 60_000)).toBe("59 minutes ago");
+  });
+
+  test("returns hours for times between 1-23 hours ago", () => {
+    const now = Date.now();
+    expect(formatJournalRelativeTime(now - 60 * 60_000)).toBe("1 hour ago");
+    expect(formatJournalRelativeTime(now - 3 * 60 * 60_000)).toBe(
+      "3 hours ago",
+    );
+    expect(formatJournalRelativeTime(now - 23 * 60 * 60_000)).toBe(
+      "23 hours ago",
+    );
+  });
+
+  test("returns days for times between 1-6 days ago", () => {
+    const now = Date.now();
+    expect(formatJournalRelativeTime(now - 24 * 60 * 60_000)).toBe(
+      "1 day ago",
+    );
+    expect(formatJournalRelativeTime(now - 3 * 24 * 60 * 60_000)).toBe(
+      "3 days ago",
+    );
+    expect(formatJournalRelativeTime(now - 6 * 24 * 60 * 60_000)).toBe(
+      "6 days ago",
+    );
+  });
+
+  test("returns weeks for times 7 or more days ago", () => {
+    const now = Date.now();
+    expect(formatJournalRelativeTime(now - 7 * 24 * 60 * 60_000)).toBe(
+      "1 week ago",
+    );
+    expect(formatJournalRelativeTime(now - 14 * 24 * 60 * 60_000)).toBe(
+      "2 weeks ago",
+    );
+    expect(formatJournalRelativeTime(now - 30 * 24 * 60 * 60_000)).toBe(
+      "4 weeks ago",
+    );
+  });
+});
+
+describe("formatJournalAbsoluteTime", () => {
+  test("formats a timestamp as MM/DD/YY HH:MM", () => {
+    // 2025-03-15 14:30:00
+    const ts = new Date(2025, 2, 15, 14, 30, 0).getTime();
+    expect(formatJournalAbsoluteTime(ts)).toBe("03/15/25 14:30");
+  });
+
+  test("zero-pads single-digit months, days, hours, and minutes", () => {
+    // 2025-01-05 09:05:00
+    const ts = new Date(2025, 0, 5, 9, 5, 0).getTime();
+    expect(formatJournalAbsoluteTime(ts)).toBe("01/05/25 09:05");
+  });
+
+  test("handles midnight", () => {
+    const ts = new Date(2025, 5, 20, 0, 0, 0).getTime();
+    expect(formatJournalAbsoluteTime(ts)).toBe("06/20/25 00:00");
+  });
+});
+
+describe("buildJournalContext", () => {
+  const journalDir = join(TEST_DIR, "journal");
+
+  beforeEach(() => {
+    mkdirSync(journalDir, { recursive: true });
+  });
+
+  afterEach(() => {
+    rmSync(TEST_DIR, { recursive: true, force: true });
+  });
+
+  test("returns null when maxEntries is 0", () => {
+    const userDir = join(journalDir, "testuser");
+    mkdirSync(userDir, { recursive: true });
+    writeFileSync(join(userDir, "entry.md"), "content");
+    expect(buildJournalContext(0, "testuser")).toBeNull();
+  });
+
+  test("returns null when maxEntries is negative", () => {
+    const userDir = join(journalDir, "testuser");
+    mkdirSync(userDir, { recursive: true });
+    writeFileSync(join(userDir, "entry.md"), "content");
+    expect(buildJournalContext(-1, "testuser")).toBeNull();
+  });
+
+  test("returns null when journal directory does not exist", () => {
+    rmSync(journalDir, { recursive: true, force: true });
+    expect(buildJournalContext(10, "testuser")).toBeNull();
+  });
+
+  test("returns null when journal directory has no .md files", () => {
+    const userDir = join(journalDir, "testuser");
+    mkdirSync(userDir, { recursive: true });
+    writeFileSync(join(userDir, "notes.txt"), "not markdown");
+    expect(buildJournalContext(10, "testuser")).toBeNull();
+  });
+
+  test("excludes README.md (case-insensitive)", () => {
+    const userDir = join(journalDir, "testuser");
+    mkdirSync(userDir, { recursive: true });
+    writeFileSync(join(userDir, "README.md"), "readme content");
+    writeFileSync(join(userDir, "readme.md"), "readme content lower");
+    expect(buildJournalContext(10, "testuser")).toBeNull();
+  });
+
+  test("returns formatted journal context with single entry", () => {
+    const userDir = join(journalDir, "testuser");
+    mkdirSync(userDir, { recursive: true });
+    writeFileSync(join(userDir, "goals.md"), "My goals for this week.");
+    const result = buildJournalContext(10, "testuser");
+    expect(result).not.toBeNull();
+    expect(result).toContain("# Journal");
+    expect(result).toContain(
+      "Your journal entries, most recent first. These are YOUR words from past conversations.",
+    );
+    expect(result).toContain("## goals.md — MOST RECENT");
+    expect(result).toContain("My goals for this week.");
+    // Single entry, window not full — should NOT have LEAVING CONTEXT
+    expect(result).not.toContain("LEAVING CONTEXT");
+  });
+
+  test("sorts entries by creation time, newest first", async () => {
+    const userDir = join(journalDir, "testuser");
+    mkdirSync(userDir, { recursive: true });
+    // Small delays between writes ensure distinct birthtimes.
+    writeFileSync(join(userDir, "old.md"), "old entry");
+    await tick();
+    writeFileSync(join(userDir, "mid.md"), "mid entry");
+    await tick();
+    writeFileSync(join(userDir, "new.md"), "new entry");
+
+    const result = buildJournalContext(10, "testuser")!;
+    const newIdx = result.indexOf("new.md");
+    const midIdx = result.indexOf("mid.md");
+    const oldIdx = result.indexOf("old.md");
+    expect(newIdx).toBeLessThan(midIdx);
+    expect(midIdx).toBeLessThan(oldIdx);
+  });
+
+  test("marks most recent entry with MOST RECENT", async () => {
+    const userDir = join(journalDir, "testuser");
+    mkdirSync(userDir, { recursive: true });
+    writeFileSync(join(userDir, "older.md"), "older");
+    await tick();
+    writeFileSync(join(userDir, "newest.md"), "newest");
+
+    const result = buildJournalContext(10, "testuser")!;
+    expect(result).toContain("## newest.md — MOST RECENT");
+    expect(result).not.toContain("## older.md — MOST RECENT");
+  });
+
+  test("marks oldest entry with LEAVING CONTEXT when window is full", async () => {
+    const userDir = join(journalDir, "testuser");
+    mkdirSync(userDir, { recursive: true });
+    // Create in chronological order: c (oldest), b, a (newest)
+    writeFileSync(join(userDir, "c.md"), "entry c");
+    await tick();
+    writeFileSync(join(userDir, "b.md"), "entry b");
+    await tick();
+    writeFileSync(join(userDir, "a.md"), "entry a");
+
+    // maxEntries = 3 matches the number of files, so window is full
+    const result = buildJournalContext(3, "testuser")!;
+    expect(result).toContain("## a.md — MOST RECENT");
+    expect(result).toContain("## c.md — LEAVING CONTEXT");
+    expect(result).toContain(
+      "NOTE: This is the oldest entry in your active context.",
+    );
+    expect(result).toContain(
+      "carry forward anything from here that still matters to you",
+    );
+  });
+
+  test("does NOT mark oldest entry with LEAVING CONTEXT when window is not full", async () => {
+    const userDir = join(journalDir, "testuser");
+    mkdirSync(userDir, { recursive: true });
+    writeFileSync(join(userDir, "b.md"), "entry b");
+    await tick();
+    writeFileSync(join(userDir, "a.md"), "entry a");
+
+    // maxEntries = 5, only 2 files — window is NOT full
+    const result = buildJournalContext(5, "testuser")!;
+    expect(result).toContain("## a.md — MOST RECENT");
+    expect(result).not.toContain("LEAVING CONTEXT");
+  });
+
+  test("limits entries to maxEntries", async () => {
+    const userDir = join(journalDir, "testuser");
+    mkdirSync(userDir, { recursive: true });
+    // Create files 0-4 sequentially; file 4 is newest
+    for (let i = 0; i < 5; i++) {
+      writeFileSync(join(userDir, `entry-${i}.md`), `content ${i}`);
+      if (i < 4) await tick();
+    }
+
+    const result = buildJournalContext(3, "testuser")!;
+    // Should contain only 3 newest entries (entry-4, entry-3, entry-2)
+    expect(result).toContain("entry-4.md");
+    expect(result).toContain("entry-3.md");
+    expect(result).toContain("entry-2.md");
+    expect(result).not.toContain("entry-1.md");
+    expect(result).not.toContain("entry-0.md");
+  });
+
+  test("maxEntries=1 with exactly one entry marks it MOST RECENT, not LEAVING CONTEXT", () => {
+    const userDir = join(journalDir, "testuser");
+    mkdirSync(userDir, { recursive: true });
+    writeFileSync(join(userDir, "solo.md"), "only entry");
+    const result = buildJournalContext(1, "testuser")!;
+    expect(result).toContain("## solo.md — MOST RECENT");
+    expect(result).not.toContain("LEAVING CONTEXT");
+  });
+
+  test("includes both absolute and relative timestamps in headers", () => {
+    const userDir = join(journalDir, "testuser");
+    mkdirSync(userDir, { recursive: true });
+    writeFileSync(join(userDir, "recent.md"), "recent content");
+
+    const result = buildJournalContext(10, "testuser")!;
+    // File was just created, so relative time should be "just now"
+    expect(result).toContain("just now");
+    // Absolute time should match the file's birthtime
+    const birthtime = statSync(join(userDir, "recent.md")).birthtimeMs;
+    const expected = formatJournalAbsoluteTime(birthtime);
+    expect(result).toContain(expected);
+  });
+
+  test("middle entries have plain headers with timestamps", async () => {
+    const userDir = join(journalDir, "testuser");
+    mkdirSync(userDir, { recursive: true });
+    // Create in chronological order: last (oldest), middle, first (newest)
+    writeFileSync(join(userDir, "last.md"), "last");
+    await tick();
+    writeFileSync(join(userDir, "middle.md"), "middle");
+    await tick();
+    writeFileSync(join(userDir, "first.md"), "first");
+
+    const result = buildJournalContext(3, "testuser")!;
+    // Middle entry should have plain header format (no MOST RECENT, no LEAVING CONTEXT)
+    // Format: ## middle.md (MM/DD/YY HH:MM, <relative time>)
+    expect(result).toMatch(
+      /## middle\.md \(\d{2}\/\d{2}\/\d{2} \d{2}:\d{2}, .+\)/,
+    );
+  });
+
+  // --- Per-user scoping tests ---
+
+  test("reads from per-user directory when userSlug is provided", () => {
+    const aliceDir = join(journalDir, "alice");
+    mkdirSync(aliceDir, { recursive: true });
+    writeFileSync(join(aliceDir, "thoughts.md"), "Alice's thoughts");
+    writeFileSync(join(aliceDir, "plans.md"), "Alice's plans");
+
+    const result = buildJournalContext(10, "alice");
+    expect(result).not.toBeNull();
+    expect(result).toContain("Alice's thoughts");
+    expect(result).toContain("Alice's plans");
+  });
+
+  test("returns null when userSlug is provided but directory does not exist", () => {
+    // journal/bob/ does not exist
+    const result = buildJournalContext(10, "bob");
+    expect(result).toBeNull();
+  });
+
+  test("returns null when no userSlug is provided", () => {
+    // Even if root journal/ has entries, no slug means null
+    writeFileSync(join(journalDir, "orphan.md"), "orphan entry");
+    const result = buildJournalContext(10);
+    expect(result).toBeNull();
+  });
+
+  test("returns null when userSlug is null", () => {
+    writeFileSync(join(journalDir, "orphan.md"), "orphan entry");
+    const result = buildJournalContext(10, null);
+    expect(result).toBeNull();
+  });
+
+  test("includes write-path directive in header when userSlug is provided", () => {
+    const aliceDir = join(journalDir, "alice");
+    mkdirSync(aliceDir, { recursive: true });
+    writeFileSync(join(aliceDir, "entry.md"), "some content");
+
+    const result = buildJournalContext(10, "alice")!;
+    expect(result).toContain("**Write new entries to:** `journal/alice/`");
+  });
+
+  test("sanitizes path-traversal in userSlug", () => {
+    // basename("../etc") => "etc", so it should read from journal/etc/
+    const etcDir = join(journalDir, "etc");
+    mkdirSync(etcDir, { recursive: true });
+    writeFileSync(join(etcDir, "safe.md"), "safe content");
+
+    const result = buildJournalContext(10, "../etc");
+    expect(result).not.toBeNull();
+    expect(result).toContain("safe content");
+    // Should reference the sanitized path, not the traversal attempt
+    expect(result).toContain("`journal/etc/`");
+  });
+});

package/src/__tests__/memory-context-benchmark.benchmark.test.ts

@@ -5,7 +5,6 @@
  * - compaction.summaryCalls: 2-6
  * - compaction.estimatedInputTokens: < previousEstimatedInputTokens
  * - recall.injectedTokens: <= computed dynamic budget
- * - recall.recencyHits: > 0
  * - recall.enabled: true
  */
 import { mkdtempSync, rmSync } from "node:fs";
@@ -303,8 +302,6 @@ describe("Memory context benchmark", () => {
       { maxInjectTokensOverride: recallBudget },
     );
 
-    // Recency search finds conversation-scoped segments.
-    expect(recall.recencyHits).toBeGreaterThan(0);
     expect(recall.enabled).toBe(true);
     // With Qdrant mock returning a high-scoring result, content should be injected.
     expect(recall.selectedCount).toBeGreaterThan(0);

package/src/__tests__/memory-lifecycle-e2e.test.ts

@@ -2,7 +2,7 @@
  * Memory lifecycle E2E regression test.
  *
  * Verifies the new memory pipeline end-to-end:
- * - 6-kind enum items (identity, preference, project, decision, constraint, event)
+ * - Standard-kind enum items (identity, preference, project, decision, constraint, event, journal, capability, ...)
  * - Supersession chains (supersedes/supersededBy fields)
  * - Hybrid search retrieval
  * - Two-layer XML injection format (<memory_context> with sections)
@@ -44,10 +44,31 @@ mock.module("../util/logger.js", () => ({
     }),
 }));
 
+// Stub the local embedding backend so the real ONNX model never loads
+mock.module("../memory/embedding-local.js", () => ({
+  LocalEmbeddingBackend: class {
+    readonly provider = "local" as const;
+    readonly model: string;
+    constructor(model: string) {
+      this.model = model;
+    }
+    async embed(texts: string[]): Promise<number[][]> {
+      return texts.map(() => new Array(384).fill(0));
+    }
+  },
+}));
+
+// Dynamic Qdrant mock: tests can push results to be returned by searchWithFilter/hybridSearch
+let mockQdrantResults: Array<{
+  id: string;
+  score: number;
+  payload: Record<string, unknown>;
+}> = [];
+
 mock.module("../memory/qdrant-client.js", () => ({
   getQdrantClient: () => ({
-    searchWithFilter: async () => [],
-    hybridSearch: async () => [],
+    searchWithFilter: async () => mockQdrantResults,
+    hybridSearch: async () => mockQdrantResults,
     upsertPoints: async () => {},
     deletePoints: async () => {},
   }),
@@ -60,7 +81,6 @@ const TEST_CONFIG = {
     ...DEFAULT_CONFIG.memory,
     embeddings: {
       ...DEFAULT_CONFIG.memory.embeddings,
-      provider: "openai" as const,
       required: false,
     },
     extraction: {
@@ -115,6 +135,7 @@ describe("Memory lifecycle E2E regression", () => {
     db.run("DELETE FROM conversations");
     db.run("DELETE FROM memory_jobs");
     db.run("DELETE FROM memory_checkpoints");
+    mockQdrantResults = [];
     resetCleanupScheduleThrottle();
     resetStaleSweepThrottle();
   });
@@ -128,7 +149,7 @@ describe("Memory lifecycle E2E regression", () => {
     }
   });
 
-  test("extraction produces items with 6-kind enum and supersession chains form correctly", async () => {
+  test("extraction produces items with standard-kind enum and supersession chains form correctly", async () => {
     const db = getDb();
     const now = 1_701_100_000_000;
     const conversationId = "conv-memory-lifecycle";
@@ -165,7 +186,7 @@ describe("Memory lifecycle E2E regression", () => {
       ])
       .run();
 
-    // Seed items using the 6-kind enum
+    // Seed items using the standard-kind enum
     const kinds = [
       "identity",
       "preference",
@@ -299,12 +320,8 @@ describe("Memory lifecycle E2E regression", () => {
       TEST_CONFIG,
     );
 
-    // Recency search finds segments but their finalScore (semantic*0.7 +
-    // recency*0.2 + confidence*0.1) is too low to pass tier classification
-    // (threshold > 0.6) because semantic=0 with Qdrant mocked empty.
-    // Verify recency search ran successfully.
-    expect(recall.recencyHits).toBeGreaterThan(0);
-    // Candidates exist but don't pass tier classification, so injectedText is empty
+    // Without semantic search (Qdrant mocked empty), no candidates pass
+    // tier classification (threshold > 0.6).
     expect(recall.enabled).toBe(true);
   });
 
@@ -343,15 +360,47 @@ describe("Memory lifecycle E2E regression", () => {
       })
       .run();
 
-    db.run(`
-      INSERT INTO memory_segments (
-        id, message_id, conversation_id, role, segment_index, text, token_estimate, scope_id, created_at, updated_at
-      ) VALUES (
-        'seg-injection-seed', 'msg-injection-seed', '${conversationId}', 'user', 0,
-        'My preferred timezone is America/Los_Angeles.', 10, 'default',
-        ${now + 10}, ${now + 10}
-      )
-    `);
+    // Seed a memory item so the semantic search path can find it
+    db.insert(memoryItems)
+      .values({
+        id: "item-timezone-pref",
+        kind: "preference",
+        subject: "timezone preference",
+        statement: "My preferred timezone is America/Los_Angeles.",
+        status: "active",
+        confidence: 0.9,
+        importance: 0.8,
+        fingerprint: "fp-item-timezone-pref",
+        firstSeenAt: now + 10,
+        lastSeenAt: now + 10,
+      })
+      .run();
+
+    db.insert(memoryItemSources)
+      .values({
+        memoryItemId: "item-timezone-pref",
+        messageId: "msg-injection-seed",
+        evidence: "timezone preference evidence",
+        createdAt: now + 10,
+      })
+      .run();
+
+    // Mock Qdrant to return the timezone preference item
+    mockQdrantResults = [
+      {
+        id: "emb-timezone-pref",
+        score: 0.92,
+        payload: {
+          target_type: "item",
+          target_id: "item-timezone-pref",
+          text: "My preferred timezone is America/Los_Angeles.",
+          kind: "preference",
+          status: "active",
+          created_at: now + 10,
+          last_seen_at: now + 10,
+        },
+      },
+    ];
 
     const recall = await buildMemoryRecall(
       "timezone",
@@ -359,10 +408,6 @@ describe("Memory lifecycle E2E regression", () => {
       TEST_CONFIG,
     );
 
-    // The recency-only promotion path (Step 6 in retriever) ensures the
-    // seeded segment reaches tier 2 and is injected even without semantic
-    // search. Verify structure of the two-layer XML format.
-    expect(recall.recencyHits).toBeGreaterThan(0);
     expect(recall.enabled).toBe(true);
     expect(recall.injectedText.length).toBeGreaterThan(0);
     expect(recall.injectedTokens).toBeGreaterThan(0);