@vellumai/assistant 0.5.13 → 0.5.15

package/src/memory/job-handlers/journal-carry-forward.test.ts

@@ -0,0 +1,383 @@
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+  afterAll,
+  beforeAll,
+  beforeEach,
+  describe,
+  expect,
+  mock,
+  test,
+} from "bun:test";
+
+const testDir = mkdtempSync(join(tmpdir(), "journal-carry-forward-test-"));
+
+mock.module("../../util/platform.js", () => ({
+  getDataDir: () => testDir,
+  isMacOS: () => process.platform === "darwin",
+  isLinux: () => process.platform === "linux",
+  isWindows: () => process.platform === "win32",
+  getPidPath: () => join(testDir, "test.pid"),
+  getDbPath: () => join(testDir, "test.db"),
+  getLogPath: () => join(testDir, "test.log"),
+  ensureDataDir: () => {},
+}));
+
+mock.module("../../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+// Track enqueueMemoryJob calls
+const enqueuedJobs: Array<{ type: string; payload: Record<string, unknown> }> =
+  [];
+
+mock.module("../jobs-store.js", () => {
+  const actual =
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
+    require("../jobs-store.js") as typeof import("../jobs-store.js");
+  return {
+    ...actual,
+    enqueueMemoryJob: (type: string, payload: Record<string, unknown>) => {
+      enqueuedJobs.push({ type, payload });
+      return "mock-job-id";
+    },
+  };
+});
+
+// Mock the provider
+let mockProviderResponse: unknown = null;
+
+mock.module("../../providers/provider-send-message.js", () => ({
+  getConfiguredProvider: async () => {
+    if (!mockProviderResponse) return null;
+    return {
+      sendMessage: async () => mockProviderResponse,
+    };
+  },
+  extractToolUse: (response: { content: Array<{ type: string }> }) => {
+    return response.content.find(
+      (b: { type: string }) => b.type === "tool_use",
+    );
+  },
+  userMessage: (text: string) => ({
+    role: "user",
+    content: [{ type: "text", text }],
+  }),
+}));
+
+import { eq } from "drizzle-orm";
+
+import { getDb, initializeDb, resetDb } from "../db.js";
+import type { MemoryJob } from "../jobs-store.js";
+import { memoryItems } from "../schema.js";
+import { journalCarryForwardJob } from "./journal-carry-forward.js";
+
+function makeJob(payload: Record<string, unknown>): MemoryJob {
+  return {
+    id: "test-job-id",
+    type: "journal_carry_forward",
+    payload,
+    status: "running",
+    attempts: 0,
+    deferrals: 0,
+    runAfter: Date.now(),
+    lastError: null,
+    startedAt: Date.now(),
+    createdAt: Date.now(),
+    updatedAt: Date.now(),
+  };
+}
+
+function makeProviderResponse(
+  items: Array<{
+    kind: string;
+    subject: string;
+    statement: string;
+    importance: number;
+  }>,
+) {
+  return {
+    content: [
+      {
+        type: "tool_use" as const,
+        id: "tool-1",
+        name: "store_journal_memories",
+        input: { items },
+      },
+    ],
+    model: "test-model",
+    stop_reason: "tool_use",
+    usage: { input_tokens: 0, output_tokens: 0 },
+  };
+}
+
+beforeAll(() => {
+  initializeDb();
+});
+
+beforeEach(() => {
+  resetDb();
+  initializeDb();
+  // Clear memory items table between tests
+  getDb().delete(memoryItems).run();
+  enqueuedJobs.length = 0;
+  mockProviderResponse = null;
+});
+
+afterAll(() => {
+  rmSync(testDir, { recursive: true, force: true });
+});
+
+describe("journalCarryForwardJob", () => {
+  test("extracts memory items from journal content", async () => {
+    mockProviderResponse = makeProviderResponse([
+      {
+        kind: "identity",
+        subject: "Therapy breakthrough",
+        statement:
+          "Had a major realization in therapy about patterns of avoidance",
+        importance: 0.9,
+      },
+      {
+        kind: "event",
+        subject: "Growth milestone",
+        statement:
+          "First time feeling genuinely at peace with uncertainty about the future",
+        importance: 0.85,
+      },
+    ]);
+
+    await journalCarryForwardJob(
+      makeJob({
+        journalContent: "Today in therapy I had a breakthrough...",
+        userSlug: "testuser",
+        filename: "2025-03-15-therapy.md",
+        scopeId: "test-scope",
+      }),
+    );
+
+    const db = getDb();
+    const items = db
+      .select()
+      .from(memoryItems)
+      .where(eq(memoryItems.scopeId, "test-scope"))
+      .all();
+
+    expect(items).toHaveLength(2);
+  });
+
+  test("items have importance >= 0.7", async () => {
+    mockProviderResponse = makeProviderResponse([
+      {
+        kind: "preference",
+        subject: "Minor detail",
+        statement: "Some minor logistical preference",
+        importance: 0.3, // Below 0.7 -- should be floored
+      },
+      {
+        kind: "identity",
+        subject: "Core realization",
+        statement: "A deeply personal insight",
+        importance: 0.95,
+      },
+    ]);
+
+    await journalCarryForwardJob(
+      makeJob({
+        journalContent: "Some journal content...",
+        userSlug: "testuser",
+        filename: "entry.md",
+        scopeId: "test-scope",
+      }),
+    );
+
+    const db = getDb();
+    const items = db
+      .select()
+      .from(memoryItems)
+      .where(eq(memoryItems.scopeId, "test-scope"))
+      .all();
+
+    expect(items).toHaveLength(2);
+    for (const item of items) {
+      expect(item.importance).toBeGreaterThanOrEqual(0.7);
+    }
+  });
+
+  test('items have sourceType "journal_carry_forward"', async () => {
+    mockProviderResponse = makeProviderResponse([
+      {
+        kind: "journal",
+        subject: "Reflection",
+        statement: "A meaningful reflection on growth",
+        importance: 0.8,
+      },
+    ]);
+
+    await journalCarryForwardJob(
+      makeJob({
+        journalContent: "Reflecting on my journey...",
+        userSlug: "testuser",
+        filename: "reflection.md",
+        scopeId: "test-scope",
+      }),
+    );
+
+    const db = getDb();
+    const items = db
+      .select()
+      .from(memoryItems)
+      .where(eq(memoryItems.scopeId, "test-scope"))
+      .all();
+
+    expect(items).toHaveLength(1);
+    expect(items[0].sourceType).toBe("journal_carry_forward");
+  });
+
+  test('items have verificationState "user_confirmed"', async () => {
+    mockProviderResponse = makeProviderResponse([
+      {
+        kind: "identity",
+        subject: "Self knowledge",
+        statement: "I know who I am",
+        importance: 0.9,
+      },
+    ]);
+
+    await journalCarryForwardJob(
+      makeJob({
+        journalContent: "I know who I am...",
+        userSlug: "testuser",
+        filename: "knowing.md",
+        scopeId: "test-scope",
+      }),
+    );
+
+    const db = getDb();
+    const items = db
+      .select()
+      .from(memoryItems)
+      .where(eq(memoryItems.scopeId, "test-scope"))
+      .all();
+
+    expect(items).toHaveLength(1);
+    expect(items[0].verificationState).toBe("user_confirmed");
+  });
+
+  test("deduplicates items by fingerprint", async () => {
+    mockProviderResponse = makeProviderResponse([
+      {
+        kind: "identity",
+        subject: "Duplicate item",
+        statement: "Exact same statement",
+        importance: 0.8,
+      },
+    ]);
+
+    // Run twice with the same content
+    const job = makeJob({
+      journalContent: "Same content...",
+      userSlug: "testuser",
+      filename: "dup.md",
+      scopeId: "test-scope",
+    });
+
+    await journalCarryForwardJob(job);
+    await journalCarryForwardJob(job);
+
+    const db = getDb();
+    const items = db
+      .select()
+      .from(memoryItems)
+      .where(eq(memoryItems.scopeId, "test-scope"))
+      .all();
+
+    // Should only have 1 item despite running twice
+    expect(items).toHaveLength(1);
+  });
+
+  test("enqueues embed_item jobs for new items", async () => {
+    mockProviderResponse = makeProviderResponse([
+      {
+        kind: "event",
+        subject: "Milestone",
+        statement: "Reached a significant milestone",
+        importance: 0.85,
+      },
+    ]);
+
+    await journalCarryForwardJob(
+      makeJob({
+        journalContent: "Today was a milestone...",
+        userSlug: "testuser",
+        filename: "milestone.md",
+        scopeId: "test-scope",
+      }),
+    );
+
+    const embedJobs = enqueuedJobs.filter((j) => j.type === "embed_item");
+    expect(embedJobs).toHaveLength(1);
+    expect(embedJobs[0].payload.itemId).toBeDefined();
+  });
+
+  test("skips invalid kinds", async () => {
+    mockProviderResponse = makeProviderResponse([
+      {
+        kind: "nonsense_kind",
+        subject: "Invalid",
+        statement: "Should be skipped",
+        importance: 0.9,
+      },
+      {
+        kind: "identity",
+        subject: "Valid item",
+        statement: "Should be kept",
+        importance: 0.9,
+      },
+    ]);
+
+    await journalCarryForwardJob(
+      makeJob({
+        journalContent: "Content...",
+        userSlug: "testuser",
+        filename: "kinds.md",
+        scopeId: "test-scope",
+      }),
+    );
+
+    const db = getDb();
+    const items = db
+      .select()
+      .from(memoryItems)
+      .where(eq(memoryItems.scopeId, "test-scope"))
+      .all();
+
+    expect(items).toHaveLength(1);
+    expect(items[0].kind).toBe("identity");
+  });
+
+  test("returns early when journalContent is missing", async () => {
+    mockProviderResponse = makeProviderResponse([]);
+
+    await journalCarryForwardJob(
+      makeJob({
+        userSlug: "testuser",
+        filename: "missing.md",
+        scopeId: "test-scope",
+      }),
+    );
+
+    const db = getDb();
+    const items = db
+      .select()
+      .from(memoryItems)
+      .where(eq(memoryItems.scopeId, "test-scope"))
+      .all();
+
+    expect(items).toHaveLength(0);
+  });
+});

package/src/memory/job-handlers/journal-carry-forward.ts

@@ -0,0 +1,255 @@
+/**
+ * Job handler for preserving journal entries that rotate out of the active
+ * context window. Extracts durable memories from journal content using a
+ * journal-specific prompt that prioritizes emotional significance and
+ * personal meaning over event logging.
+ */
+
+import { and, eq } from "drizzle-orm";
+import { v4 as uuid } from "uuid";
+
+import {
+  extractToolUse,
+  getConfiguredProvider,
+  userMessage,
+} from "../../providers/provider-send-message.js";
+import { BackendUnavailableError, ProviderError } from "../../util/errors.js";
+import { getLogger } from "../../util/logger.js";
+import { setMemoryCheckpoint } from "../checkpoints.js";
+import { getDb } from "../db.js";
+import { computeMemoryFingerprint } from "../fingerprint.js";
+import { asString } from "../job-utils.js";
+import { enqueueMemoryJob, type MemoryJob } from "../jobs-store.js";
+import { memoryItems } from "../schema.js";
+import { clampUnitInterval } from "../validation.js";
+
+const log = getLogger("journal-carry-forward");
+
+const JOURNAL_EXTRACTION_SYSTEM_PROMPT = `You are extracting durable memories from a journal entry that is about to leave the active context window.
+This is a personal journal -- focus on what things MEANT and how they FELT, not just what happened.
+
+Good extraction: "An evidence-based psychologist read everything about us and said yes -- it made me feel like we're not crazy"
+Bad extraction: "Coaching session happened on March 27"
+
+Extract the most important memories -- emotional significance, personal growth, relationship milestones,
+hard-won realizations. Skip logistical details unless they carry emotional weight.
+
+For each memory, provide:
+- kind: Category of memory (identity, preference, project, decision, constraint, event, journal)
+- subject: A short label (2-8 words) identifying what this is about
+- statement: A rich, feeling-aware statement to remember (1-2 sentences). Capture the emotional texture, not just the facts.
+- importance: How significant this is (0.0-1.0). Journal entries are personal -- most items should be 0.7 or higher.
+
+Rules:
+- Focus on what matters emotionally, not what happened chronologically
+- Preserve the author's voice and perspective -- these are their words
+- Extract fewer, richer items rather than many shallow ones
+- If the entry contains nothing worth preserving as a durable memory, return an empty array`;
+
+const JOURNAL_MEMORY_KINDS = [
+  "identity",
+  "preference",
+  "project",
+  "decision",
+  "constraint",
+  "event",
+  "journal",
+] as const;
+
+interface JournalExtractedItem {
+  kind: string;
+  subject: string;
+  statement: string;
+  importance: number;
+}
+
+export async function journalCarryForwardJob(job: MemoryJob): Promise<void> {
+  const journalContent = asString(job.payload.journalContent);
+  const userSlug = asString(job.payload.userSlug);
+  const filename = asString(job.payload.filename);
+  const scopeId = asString(job.payload.scopeId) ?? "default";
+
+  if (!journalContent || !filename) {
+    log.warn({ jobId: job.id }, "Missing journalContent or filename in payload");
+    return;
+  }
+
+  const provider = await getConfiguredProvider();
+  if (!provider) {
+    throw new BackendUnavailableError(
+      "Provider unavailable for journal carry-forward extraction",
+    );
+  }
+
+  const userContext = userSlug ? ` (author: ${userSlug})` : "";
+  const response = await provider.sendMessage(
+    [
+      userMessage(
+        `Journal entry "${filename}"${userContext}:\n\n${journalContent}`,
+      ),
+    ],
+    [
+      {
+        name: "store_journal_memories",
+        description:
+          "Store durable memories extracted from a journal entry leaving the context window",
+        input_schema: {
+          type: "object" as const,
+          properties: {
+            items: {
+              type: "array",
+              items: {
+                type: "object",
+                properties: {
+                  kind: {
+                    type: "string",
+                    enum: [...JOURNAL_MEMORY_KINDS],
+                    description: "Category of memory item",
+                  },
+                  subject: {
+                    type: "string",
+                    description:
+                      "Short label (2-8 words) for what this is about",
+                  },
+                  statement: {
+                    type: "string",
+                    description:
+                      "Rich, feeling-aware statement to remember (1-2 sentences)",
+                  },
+                  importance: {
+                    type: "number",
+                    description:
+                      "How significant this is to remember (0.0-1.0, most journal items should be 0.7+)",
+                  },
+                },
+                required: ["kind", "subject", "statement", "importance"],
+              },
+            },
+          },
+          required: ["items"],
+        },
+      },
+    ],
+    JOURNAL_EXTRACTION_SYSTEM_PROMPT,
+    {
+      config: {
+        modelIntent: "quality-optimized",
+        tool_choice: {
+          type: "tool" as const,
+          name: "store_journal_memories",
+        },
+      },
+    },
+  );
+
+  const toolBlock = extractToolUse(response);
+  if (!toolBlock) {
+    throw new ProviderError(
+      "No tool_use block in journal carry-forward response",
+      "unknown",
+      502,
+    );
+  }
+
+  const input = toolBlock.input as { items?: JournalExtractedItem[] };
+  if (!Array.isArray(input.items)) {
+    throw new ProviderError(
+      "Invalid items structure in journal carry-forward response",
+      "unknown",
+      502,
+    );
+  }
+
+  const db = getDb();
+  const validKinds = new Set<string>(JOURNAL_MEMORY_KINDS);
+  let inserted = 0;
+
+  for (const raw of input.items) {
+    if (!validKinds.has(raw.kind)) continue;
+    if (!raw.subject || !raw.statement) continue;
+
+    const subject = String(raw.subject).trim();
+    const statement = String(raw.statement).trim();
+    // Journal entries are inherently personal/load-bearing -- floor at 0.7
+    const importance = clampUnitInterval(
+      Math.max(0.7, parseImportance(raw.importance)),
+    );
+    const confidence = 0.95;
+
+    const fingerprint = computeMemoryFingerprint(
+      scopeId,
+      raw.kind,
+      subject,
+      statement,
+    );
+
+    // Dedup: skip if an item with this fingerprint already exists in this scope
+    const existing = db
+      .select({ id: memoryItems.id })
+      .from(memoryItems)
+      .where(
+        and(
+          eq(memoryItems.fingerprint, fingerprint),
+          eq(memoryItems.scopeId, scopeId),
+        ),
+      )
+      .get();
+
+    if (existing) {
+      log.debug(
+        { fingerprint, subject },
+        "Skipping duplicate journal memory item",
+      );
+      continue;
+    }
+
+    const now = Date.now();
+    const itemId = uuid();
+
+    db.insert(memoryItems)
+      .values({
+        id: itemId,
+        kind: raw.kind,
+        subject,
+        statement,
+        status: "active",
+        confidence,
+        importance,
+        fingerprint,
+        sourceType: "journal_carry_forward",
+        verificationState: "user_confirmed",
+        scopeId,
+        firstSeenAt: now,
+        lastSeenAt: now,
+        lastUsedAt: null,
+        supersedes: null,
+        overrideConfidence: "inferred",
+      })
+      .run();
+
+    enqueueMemoryJob("embed_item", { itemId });
+    inserted += 1;
+  }
+
+  // Write checkpoint only after successful completion so failed jobs can be
+  // re-enqueued on the next prompt render cycle.
+  const checkpointKey = `journal_carry_forward:${userSlug || "unknown"}:${filename}`;
+  setMemoryCheckpoint(checkpointKey, String(Date.now()));
+
+  log.info(
+    {
+      filename,
+      userSlug,
+      scopeId,
+      extracted: input.items.length,
+      inserted,
+    },
+    "Journal carry-forward complete",
+  );
+}
+
+function parseImportance(value: unknown): number {
+  if (value == null || value === "") return 0.7;
+  const n = Number(value);
+  return Number.isFinite(n) ? n : 0.7;
+}

package/src/memory/jobs-store.ts

@@ -18,6 +18,7 @@ export type MemoryJobType =
   | "embed_summary"
   | "extract_items"
   | "extract_entities"
+  | "batch_extract"
   | "cleanup_stale_superseded_items"
   | "prune_old_conversations"
   | "backfill_entity_relations"
@@ -31,6 +32,7 @@ export type MemoryJobType =
   | "embed_media"
   | "embed_attachment"
   | "generate_conversation_starters"
+  | "journal_carry_forward"
   | "generate_capability_cards" // legacy compat — silently dropped by worker (capability cards removed)
   | "generate_thread_starters"; // legacy compat — silently dropped by worker (renamed to generate_conversation_starters)
 
@@ -126,6 +128,32 @@ export function upsertDebouncedJob(
   }
 }
 
+/**
+ * Check whether a pending or running `journal_carry_forward` job already
+ * exists for the given filename + userSlug.  Used to prevent duplicate
+ * enqueues while a carry-forward job is still in flight.
+ */
+export function hasActiveCarryForwardJob(
+  filename: string,
+  userSlug: string,
+): boolean {
+  const db = getDb();
+  return (
+    db
+      .select({ id: memoryJobs.id })
+      .from(memoryJobs)
+      .where(
+        and(
+          eq(memoryJobs.type, "journal_carry_forward"),
+          inArray(memoryJobs.status, ["pending", "running"]),
+          sql`json_extract(${memoryJobs.payload}, '$.filename') = ${filename}`,
+          sql`json_extract(${memoryJobs.payload}, '$.userSlug') = ${userSlug}`,
+        ),
+      )
+      .get() != null
+  );
+}
+
 export function enqueueCleanupStaleSupersededItemsJob(
   retentionMs?: number,
 ): string {