@vellumai/assistant 0.5.7 → 0.5.9

package/src/memory/job-handlers/summarization.ts

@@ -19,18 +19,25 @@ import { memorySegments, memorySummaries } from "../schema.js";
 const log = getLogger("memory-jobs-worker");
 
 const SUMMARY_LLM_TIMEOUT_MS = 20_000;
-const SUMMARY_MAX_TOKENS = 800;
+const SUMMARY_MAX_TOKENS = 500;
 
 const CONVERSATION_SUMMARY_SYSTEM_PROMPT = [
-  "You are a memory summarization system. Your job is to produce a compact, information-dense summary of a conversation.",
+  "You compress conversation transcripts into compact summaries for semantic search and memory retrieval.",
+  "Focus on durable facts, not transient discussion.",
+  "Preserve: goals, decisions, constraints, preferences, names, technical details, actions taken.",
+  "Remove: filler, pleasantries, tool invocation details, transient status updates.",
   "",
-  "Guidelines:",
-  "- Focus on key facts, decisions, user preferences, and actionable information.",
-  "- Preserve concrete details: names, file paths, tool choices, technical decisions, constraints.",
-  "- Remove filler, pleasantries, and transient discussion that has no lasting value.",
-  "- Use concise bullet points grouped by topic.",
-  "- Target 400-600 tokens. Be dense but readable.",
-  "- If updating an existing summary with new data, merge new information and remove anything that was superseded.",
+  "Return concise markdown:",
+  "## Topic",
+  "One-line description of what the conversation is about.",
+  "## Key Facts",
+  "Bullet points of concrete facts, names, decisions, preferences.",
+  "## Outcomes",
+  "What was decided, resolved, or accomplished.",
+  "## Open Items",
+  "Unresolved questions, pending tasks, or follow-ups (omit section if none).",
+  "",
+  "Target 200-400 tokens. Be dense.",
 ].join("\n");
 
 export async function buildConversationSummaryJob(
@@ -62,9 +69,8 @@ export async function buildConversationSummaryJob(
 
   // Build segment text for LLM input (chronological order)
   const segmentTexts = rows
-    .slice(0, 30)
     .reverse()
-    .map((row) => `[${row.role}] ${truncate(row.text, 400)}`)
+    .map((row) => `[${row.role}] ${truncate(row.text, 600)}`)
     .join("\n\n");
 
   const summaryText = await summarizeWithLLM(
@@ -208,14 +214,18 @@ async function summarizeWithLLM(
 }
 
 function buildFallbackSummary(
-  _existingSummary: string | null,
+  existingSummary: string | null,
   newContent: string,
   label: string,
 ): string {
   const lines = newContent.split("\n").filter((l) => l.trim().length > 0);
-  const snippets = lines
-    .slice(0, 20)
-    .map((l) => `- ${truncate(l.trim(), 180)}`);
-  const parts: string[] = [`${label} summary`, "", ...snippets];
+  if (lines.length === 0) return existingSummary ?? `${label} (no content)`;
+  const head = lines.slice(0, 3).map((l) => `- ${truncate(l.trim(), 200)}`);
+  const tail =
+    lines.length > 6
+      ? lines.slice(-3).map((l) => `- ${truncate(l.trim(), 200)}`)
+      : [];
+  const parts = [`${label} summary`, "", ...head];
+  if (tail.length > 0) parts.push("", "...", "", ...tail);
   return parts.join("\n");
 }

package/src/memory/jobs-store.ts

@@ -1,4 +1,4 @@
-import { and, asc, eq, inArray, lte, notInArray } from "drizzle-orm";
+import { and, asc, eq, inArray, lte, notInArray, sql } from "drizzle-orm";
 import { v4 as uuid } from "uuid";
 
 import { getLogger } from "../util/logger.js";
@@ -86,6 +86,38 @@ export function enqueueMemoryJob(
   return id;
 }
 
+/**
+ * Upsert a debounced job: if a pending job of the same type and conversation
+ * already exists, push its `runAfter` forward instead of creating a duplicate.
+ * This prevents rapid message indexing from spawning redundant jobs.
+ */
+export function upsertDebouncedJob(
+  type: MemoryJobType,
+  payload: { conversationId: string },
+  runAfter: number,
+): void {
+  const db = getDb();
+  const existing = db
+    .select()
+    .from(memoryJobs)
+    .where(
+      and(
+        eq(memoryJobs.type, type),
+        eq(memoryJobs.status, "pending"),
+        sql`json_extract(${memoryJobs.payload}, '$.conversationId') = ${payload.conversationId}`,
+      ),
+    )
+    .get();
+  if (existing) {
+    db.update(memoryJobs)
+      .set({ runAfter, updatedAt: Date.now() })
+      .where(eq(memoryJobs.id, existing.id))
+      .run();
+  } else {
+    enqueueMemoryJob(type, payload, runAfter);
+  }
+}
+
 export function enqueueCleanupStaleSupersededItemsJob(
   retentionMs?: number,
 ): string {

package/src/memory/journal-memory.ts

@@ -0,0 +1,214 @@
+import { readdirSync, readFileSync, statSync } from "node:fs";
+import { join } from "node:path";
+
+import { and, eq } from "drizzle-orm";
+import { v4 as uuid } from "uuid";
+
+import { getLogger } from "../util/logger.js";
+import { getWorkspaceDir } from "../util/platform.js";
+import { type DrizzleDb, getDb } from "./db.js";
+import { computeMemoryFingerprint } from "./fingerprint.js";
+import { enqueueMemoryJob } from "./jobs-store.js";
+import { memoryItems, memoryItemSources } from "./schema.js";
+
+const log = getLogger("memory-journal");
+
+/**
+ * Process a single journal `.md` file: read content, derive subject, compute
+ * fingerprint, upsert to DB, and enqueue an embed job.
+ *
+ * Returns `true` if a new memory item was inserted, `false` if it already
+ * existed (or was skipped).
+ */
+function upsertSingleJournalFile(
+  filepath: string,
+  filename: string,
+  messageCreatedAt: number,
+  scopeId: string,
+  messageId: string,
+  db: DrizzleDb,
+): boolean {
+  const content = readFileSync(filepath, "utf-8");
+
+  // Derive subject from filename:
+  // strip .md extension, strip leading date prefix, replace hyphens with spaces, capitalize first letter
+  const basename = filename.replace(/\.md$/, "");
+  const withoutDate = basename.replace(/^\d{4}-\d{2}-\d{2}-?/, "");
+  const withSpaces = withoutDate.replace(/-/g, " ");
+  const subject =
+    withSpaces.length > 0
+      ? withSpaces.charAt(0).toUpperCase() + withSpaces.slice(1)
+      : basename;
+
+  const fingerprint = computeMemoryFingerprint(
+    scopeId,
+    "journal",
+    subject,
+    content,
+  );
+
+  const existing = db
+    .select()
+    .from(memoryItems)
+    .where(
+      and(
+        eq(memoryItems.fingerprint, fingerprint),
+        eq(memoryItems.scopeId, scopeId),
+      ),
+    )
+    .get();
+
+  let memoryItemId: string;
+  let inserted = false;
+
+  if (existing) {
+    memoryItemId = existing.id;
+    db.update(memoryItems)
+      .set({
+        lastSeenAt: messageCreatedAt,
+        status: "active",
+      })
+      .where(eq(memoryItems.id, existing.id))
+      .run();
+  } else {
+    memoryItemId = uuid();
+    db.insert(memoryItems)
+      .values({
+        id: memoryItemId,
+        kind: "journal",
+        subject,
+        statement: content,
+        status: "active",
+        confidence: 0.95,
+        importance: 0.8,
+        fingerprint,
+        sourceType: "extraction",
+        sourceMessageRole: "assistant",
+        verificationState: "assistant_inferred",
+        scopeId,
+        firstSeenAt: messageCreatedAt,
+        lastSeenAt: messageCreatedAt,
+        lastUsedAt: null,
+        supersedes: null,
+        overrideConfidence: null,
+      })
+      .run();
+    inserted = true;
+  }
+
+  db.insert(memoryItemSources)
+    .values({
+      memoryItemId,
+      messageId,
+      evidence: content,
+      createdAt: Date.now(),
+    })
+    .onConflictDoNothing()
+    .run();
+
+  enqueueMemoryJob("embed_item", { itemId: memoryItemId });
+
+  return inserted;
+}
+
+/**
+ * Scan the journal directory for `.md` files created during (or after) the
+ * given message timestamp and upsert them as journal memory items with the
+ * raw, unedited file content as the `statement`.
+ *
+ * Also scans immediate subdirectories (e.g. per-user folders like
+ * `journal/sidd/`) so that user-scoped journal entries are indexed alongside
+ * root-level files.
+ *
+ * This bypasses the LLM extraction layer entirely — journal memories are
+ * stored verbatim so they are never summarised or rewritten.
+ *
+ * Returns the number of newly inserted items.
+ */
+export function upsertJournalMemoriesFromDisk(
+  messageCreatedAt: number,
+  scopeId: string,
+  messageId: string,
+): number {
+  try {
+    const journalDir = join(getWorkspaceDir(), "journal");
+
+    let files: string[];
+    try {
+      files = readdirSync(journalDir);
+    } catch {
+      // Directory doesn't exist — no journal entries
+      return 0;
+    }
+
+    // Filter for .md files, excluding readme.md (case-insensitive)
+    const mdFiles = files.filter(
+      (f) => f.endsWith(".md") && f.toLowerCase() !== "readme.md",
+    );
+
+    let upserted = 0;
+    const db = getDb();
+
+    for (const filename of mdFiles) {
+      try {
+        const filepath = join(journalDir, filename);
+        const stat = statSync(filepath);
+        if (!stat.isFile()) continue;
+
+        // Only process files created during or after this message
+        if (stat.birthtimeMs < messageCreatedAt) continue;
+
+        if (upsertSingleJournalFile(filepath, filename, messageCreatedAt, scopeId, messageId, db)) {
+          upserted += 1;
+        }
+      } catch (err) {
+        log.warn(
+          { filename, err: err instanceof Error ? err.message : String(err) },
+          "Failed to process journal file for memory — skipping",
+        );
+      }
+    }
+
+    // Scan per-user journal subdirectories
+    for (const entry of files) {
+      try {
+        const subdirPath = join(journalDir, entry);
+        if (!statSync(subdirPath).isDirectory()) continue;
+
+        const subFiles = readdirSync(subdirPath).filter(
+          (f) => f.endsWith(".md") && f.toLowerCase() !== "readme.md",
+        );
+
+        for (const filename of subFiles) {
+          try {
+            const filepath = join(subdirPath, filename);
+            const stat = statSync(filepath);
+            if (!stat.isFile()) continue;
+
+            // Only process files created during or after this message
+            if (stat.birthtimeMs < messageCreatedAt) continue;
+
+            if (upsertSingleJournalFile(filepath, filename, messageCreatedAt, scopeId, messageId, db)) {
+              upserted += 1;
+            }
+          } catch (err) {
+            log.warn(
+              { filename, err: err instanceof Error ? err.message : String(err) },
+              "Failed to process journal file for memory — skipping",
+            );
+          }
+        }
+      } catch {
+        // Skip unreadable subdirectories
+      }
+    }
+
+    return upserted;
+  } catch (err) {
+    log.warn(
+      { err: err instanceof Error ? err.message : String(err) },
+      "Failed to scan journal directory for memories",
+    );
+    return 0;
+  }
+}

package/src/memory/migrations/193-add-source-type-columns.ts

@@ -0,0 +1,81 @@
+import type { DrizzleDb } from "../db-connection.js";
+import { getSqliteFrom } from "../db-connection.js";
+import { tableHasColumn } from "./schema-introspection.js";
+import { withCrashRecovery } from "./validate-migration-state.js";
+
+/**
+ * Add source_type and source_message_role columns to memory_items.
+ *
+ * - source_type: "extraction" (default) or "tool" — distinguishes how the
+ *   memory was created (LLM/pattern extraction vs explicit tool/API save).
+ * - source_message_role: the role of the source message (e.g. "user",
+ *   "assistant") when the item was created via extraction.
+ *
+ * Backfills:
+ * 1. Items with verification_state = "user_confirmed" → source_type = "tool"
+ * 2. source_message_role from the earliest source message's role via subquery
+ */
+export function migrateAddSourceTypeColumns(database: DrizzleDb): void {
+  withCrashRecovery(database, "migration_add_source_type_columns_v1", () => {
+    const raw = getSqliteFrom(database);
+
+    // Add source_type column if it doesn't exist
+    if (!tableHasColumn(database, "memory_items", "source_type")) {
+      raw.exec(
+        /*sql*/ `ALTER TABLE memory_items ADD COLUMN source_type TEXT NOT NULL DEFAULT 'extraction'`,
+      );
+    }
+
+    // Add source_message_role column if it doesn't exist
+    if (!tableHasColumn(database, "memory_items", "source_message_role")) {
+      raw.exec(
+        /*sql*/ `ALTER TABLE memory_items ADD COLUMN source_message_role TEXT`,
+      );
+    }
+
+    // Backfill source_type = 'tool' for items that were explicitly saved
+    raw.exec(
+      /*sql*/ `UPDATE memory_items SET source_type = 'tool' WHERE verification_state = 'user_confirmed'`,
+    );
+
+    // Backfill source_message_role from the earliest source message's role.
+    // Only backfill where source_message_role is currently NULL and a source
+    // message exists.
+    raw.exec(/*sql*/ `
+        UPDATE memory_items
+        SET source_message_role = (
+          SELECT m.role
+          FROM memory_item_sources mis
+          JOIN messages m ON m.id = mis.message_id
+          WHERE mis.memory_item_id = memory_items.id
+          ORDER BY mis.created_at ASC
+          LIMIT 1
+        )
+        WHERE source_message_role IS NULL
+          AND EXISTS (
+            SELECT 1
+            FROM memory_item_sources mis
+            WHERE mis.memory_item_id = memory_items.id
+          )
+      `);
+  });
+}
+
+/**
+ * Reverse: drop source_type and source_message_role columns.
+ *
+ * SQLite doesn't support DROP COLUMN on older versions, but modern SQLite
+ * (3.35.0+) does. Since Bun bundles a modern SQLite, this is safe.
+ */
+export function migrateAddSourceTypeColumnsDown(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+
+  if (tableHasColumn(database, "memory_items", "source_type")) {
+    raw.exec(/*sql*/ `ALTER TABLE memory_items DROP COLUMN source_type`);
+  }
+  if (tableHasColumn(database, "memory_items", "source_message_role")) {
+    raw.exec(
+      /*sql*/ `ALTER TABLE memory_items DROP COLUMN source_message_role`,
+    );
+  }
+}

package/src/memory/migrations/index.ts

@@ -131,6 +131,7 @@ export { migrateDropSimplifiedMemory } from "./189-drop-simplified-memory.js";
 export { migrateCallSessionSkipDisclosure } from "./190-call-session-skip-disclosure.js";
 export { migrateBackfillAudioAttachmentMimeTypes } from "./191-backfill-audio-attachment-mime-types.js";
 export { migrateContactsUserFileColumn } from "./192-contacts-user-file-column.js";
+export { migrateAddSourceTypeColumns } from "./193-add-source-type-columns.js";
 export {
   MIGRATION_REGISTRY,
   type MigrationRegistryEntry,

package/src/memory/migrations/registry.ts

@@ -37,6 +37,7 @@ import { migrateDropCapabilityCardStateDown } from "./176-drop-capability-card-s
 import { migrateBackfillInlineAttachmentsToDiskDown } from "./180-backfill-inline-attachments-to-disk.js";
 import { migrateRenameThreadStartersCheckpointsDown } from "./181-rename-thread-starters-checkpoints.js";
 import { migrateBackfillAudioAttachmentMimeTypesDown } from "./191-backfill-audio-attachment-mime-types.js";
+import { migrateAddSourceTypeColumnsDown } from "./193-add-source-type-columns.js";
 
 export interface MigrationRegistryEntry {
   /** The checkpoint key written to memory_checkpoints on completion. */
@@ -325,6 +326,13 @@ export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
       "Backfill correct MIME types for audio attachments stored as application/octet-stream due to missing extension map entries",
     down: migrateBackfillAudioAttachmentMimeTypesDown,
   },
+  {
+    key: "migration_add_source_type_columns_v1",
+    version: 37,
+    description:
+      "Add source_type and source_message_role columns to memory_items with backfill from verification_state and source messages",
+    down: migrateAddSourceTypeColumnsDown,
+  },
 ];
 
 export function getMaxMigrationVersion(): number {

package/src/memory/retriever.test.ts

@@ -337,10 +337,7 @@ describe("Memory Retriever Pipeline", () => {
     expect(result.tier1Count).toBeDefined();
     expect(result.tier2Count).toBeDefined();
     expect(result.hybridSearchMs).toBeDefined();
-    // Recency search finds raw candidates from this conversation…
-    expect(result.recencyHits).toBeGreaterThan(0);
-    // …but they are filtered out because they belong to the active
-    // conversation and are already present in the conversation history.
+    // Without semantic search, no candidates are found.
     expect(result.mergedCount).toBe(0);
   });
 
@@ -403,11 +400,7 @@ describe("Memory Retriever Pipeline", () => {
     );
 
     expect(result.enabled).toBe(true);
-    // Recency search finds segments from the active conversation
-    expect(result.recencyHits).toBeGreaterThan(0);
-    // But they are filtered out of merged results; only other-conversation
-    // segments would survive (none in this case since recency is scoped to
-    // the active conversation).
+    // Without semantic search, no candidates are found.
     expect(result.mergedCount).toBe(0);
   });
 
@@ -506,10 +499,6 @@ describe("Memory Retriever Pipeline", () => {
     );
 
     expect(result.enabled).toBe(true);
-    // Recency search finds segments from this conversation
-    expect(result.recencyHits).toBeGreaterThan(0);
-    // Compacted segments survive filtering — they are no longer in context
-    expect(result.mergedCount).toBeGreaterThan(0);
   });
 
   // -----------------------------------------------------------------------
@@ -716,9 +705,7 @@ describe("Memory Retriever Pipeline", () => {
     expect(result.enabled).toBe(true);
     // Semantic/hybrid search should be skipped
     expect(result.semanticHits).toBe(0);
-    // Recency search finds raw candidates…
-    expect(result.recencyHits).toBeGreaterThan(0);
-    // …but current-conversation segments are filtered out
+    // Without semantic search, no candidates are found.
     expect(result.mergedCount).toBe(0);
   });
 
@@ -752,7 +739,7 @@ describe("Memory Retriever Pipeline", () => {
     expect(result.degradation).toBeDefined();
     expect(result.degradation!.semanticUnavailable).toBe(true);
     expect(result.degradation!.reason).toBe("embedding_provider_down");
-    expect(result.degradation!.fallbackSources).toContain("recency");
+    expect(result.degradation!.fallbackSources).toEqual([]);
   });
 
   // -----------------------------------------------------------------------
@@ -864,9 +851,7 @@ describe("Memory Retriever Pipeline", () => {
     // pipeline proceeds non-degraded end-to-end.
     expect(result.enabled).toBe(true);
     expect(result.degraded).toBe(false);
-    // Recency search finds raw candidates; hybrid search returns empty from mock
-    expect(result.recencyHits).toBeGreaterThan(0);
-    // Current-conversation segments are filtered out of merged results
+    // Without semantic search, no candidates are found.
     expect(result.mergedCount).toBe(0);
   });
 
@@ -1142,8 +1127,7 @@ describe("Memory Retriever Pipeline", () => {
       );
 
       // Simulate Qdrant returning the parent-conversation segment as a
-      // semantic hit so it enters the candidate map (recency search is scoped
-      // to forkConv and would never find it).
+      // semantic hit so it enters the candidate map.
       mockQdrantResults.push({
         id: "qdrant-fork-1",
         score: 0.9,
@@ -1264,6 +1248,35 @@ describe("Memory Retriever Pipeline", () => {
         now - 50_000,
       );
 
+      // Simulate Qdrant returning both segments as semantic hits so they
+      // enter the candidate map (recency search was removed).
+      mockQdrantResults.push(
+        {
+          id: "qdrant-compact-fork-1",
+          score: 0.9,
+          payload: {
+            target_type: "segment",
+            target_id: "seg-compact-fork",
+            text: "compacted parent topic detail",
+            created_at: now - 100_000,
+            message_id: "fork-compact-msg-1",
+            conversation_id: forkConv,
+          },
+        },
+        {
+          id: "qdrant-compact-fork-2",
+          score: 0.85,
+          payload: {
+            target_type: "segment",
+            target_id: "seg-in-context-fork",
+            text: "recent fork topic detail",
+            created_at: now - 50_000,
+            message_id: "fork-compact-msg-3",
+            conversation_id: forkConv,
+          },
+        },
+      );
+
       const result = await buildMemoryRecall(
         "compacted parent topic",
         forkConv,
@@ -1273,7 +1286,7 @@ describe("Memory Retriever Pipeline", () => {
       expect(result.enabled).toBe(true);
       // The segment from the compacted fork message survives filtering
       // (its source message is no longer in context). The in-context segment
-      // is filtered out. Recency search returns both, but only the compacted
+      // is filtered out. Semantic search returns both, but only the compacted
       // one survives step 5b.
       expect(result.mergedCount).toBeGreaterThan(0);
     });
@@ -1342,8 +1355,7 @@ describe("Memory Retriever Pipeline", () => {
       );
 
       // Simulate Qdrant returning the grandparent segment as a semantic hit
-      // so it enters the candidate map (recency search is scoped to childConv
-      // and would never find it).
+      // so it enters the candidate map.
       mockQdrantResults.push({
         id: "qdrant-gp-1",
         score: 0.9,