@vellumai/assistant 0.6.4 → 0.6.6

package/src/memory/graph/bootstrap.test.ts

@@ -0,0 +1,282 @@
+import { beforeAll, beforeEach, describe, expect, test } from "bun:test";
+
+import { setMemoryCheckpoint } from "../checkpoints.js";
+import {
+  initializeDb,
+  rawAll,
+  rawGet,
+  rawRun,
+  resetTestTables,
+} from "../db.js";
+import { migrateToolCreatedItems } from "./bootstrap.js";
+
+// ---------------------------------------------------------------------------
+// The checkpoint key used by migrateToolCreatedItems (not exported, so we
+// inline the literal value).
+// ---------------------------------------------------------------------------
+const MIGRATE_ITEMS_CHECKPOINT = "graph_bootstrap:migrated_tool_items";
+
+// ---------------------------------------------------------------------------
+// Setup
+// ---------------------------------------------------------------------------
+
+beforeAll(() => {
+  initializeDb();
+});
+
+beforeEach(() => {
+  // Clear graph nodes and checkpoints between tests so each test starts clean.
+  resetTestTables("memory_graph_nodes", "memory_checkpoints", "memory_jobs");
+});
+
+// ---------------------------------------------------------------------------
+// migrateToolCreatedItems
+// ---------------------------------------------------------------------------
+
+describe("migrateToolCreatedItems", () => {
+  test("migrates legacy memory_items to graph nodes", () => {
+    // The memory_items table has been dropped by migration 203, so we need to
+    // recreate it for this test. We create a minimal version with just the
+    // columns the migration reads.
+    rawRun(
+      `CREATE TABLE IF NOT EXISTS memory_items (
+        id TEXT PRIMARY KEY,
+        kind TEXT NOT NULL,
+        subject TEXT NOT NULL,
+        statement TEXT NOT NULL,
+        status TEXT NOT NULL,
+        confidence REAL NOT NULL,
+        importance REAL,
+        scope_id TEXT NOT NULL DEFAULT 'default',
+        first_seen_at INTEGER NOT NULL,
+        fingerprint TEXT NOT NULL DEFAULT ''
+      )`,
+    );
+
+    try {
+      // Insert a legacy playbook item
+      rawRun(
+        `INSERT INTO memory_items (id, kind, subject, statement, status, confidence, importance, scope_id, first_seen_at)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+        "legacy-item-1",
+        "playbook",
+        "Test Playbook",
+        "Do the thing correctly",
+        "active",
+        0.9,
+        0.8,
+        "default",
+        1700000000000,
+      );
+
+      // Clear the checkpoint so the migration re-runs
+      rawRun(
+        "DELETE FROM memory_checkpoints WHERE key = ?",
+        MIGRATE_ITEMS_CHECKPOINT,
+      );
+
+      // Run migration
+      migrateToolCreatedItems();
+
+      // Assert a corresponding graph node was created
+      const node = rawGet<{
+        id: string;
+        content: string;
+        type: string;
+        source_conversations: string;
+        image_refs: string | null;
+      }>(
+        `SELECT id, content, type, source_conversations, image_refs
+         FROM memory_graph_nodes
+         WHERE source_conversations LIKE ?`,
+        "%playbook:legacy-item-1%",
+      );
+
+      expect(node).not.toBeNull();
+      expect(node!.content).toBe("Test Playbook\nDo the thing correctly");
+      expect(node!.type).toBe("semantic");
+      expect(JSON.parse(node!.source_conversations)).toContain(
+        "playbook:legacy-item-1",
+      );
+      expect(node!.image_refs).toBeNull();
+    } finally {
+      // Clean up the recreated table so it does not interfere with other tests
+      rawRun("DROP TABLE IF EXISTS memory_items");
+    }
+  });
+
+  test("succeeds when image_refs column does not exist", () => {
+    // This is the regression test for the v0.6.0 bug: when upgrading from
+    // v0.5.x, migrateToolCreatedItems ran before the image_refs column was
+    // added by migration 205, causing a crash.
+
+    // Recreate memory_items for legacy data
+    rawRun(
+      `CREATE TABLE IF NOT EXISTS memory_items (
+        id TEXT PRIMARY KEY,
+        kind TEXT NOT NULL,
+        subject TEXT NOT NULL,
+        statement TEXT NOT NULL,
+        status TEXT NOT NULL,
+        confidence REAL NOT NULL,
+        importance REAL,
+        scope_id TEXT NOT NULL DEFAULT 'default',
+        first_seen_at INTEGER NOT NULL,
+        fingerprint TEXT NOT NULL DEFAULT ''
+      )`,
+    );
+
+    try {
+      // Rebuild memory_graph_nodes WITHOUT the image_refs column to simulate
+      // the pre-205 schema. SQLite doesn't support DROP COLUMN on all
+      // versions, so we use the standard CREATE-new/INSERT-SELECT/DROP-old/
+      // RENAME pattern.
+      rawRun(`CREATE TABLE memory_graph_nodes_backup AS
+        SELECT
+          id, content, type, created, last_accessed, last_consolidated,
+          event_date, emotional_charge, fidelity, confidence, significance,
+          stability, reinforcement_count, last_reinforced,
+          source_conversations, source_type, narrative_role, part_of_story,
+          scope_id
+        FROM memory_graph_nodes`);
+      rawRun("DROP TABLE memory_graph_nodes");
+      rawRun(
+        `CREATE TABLE memory_graph_nodes (
+          id                    TEXT PRIMARY KEY,
+          content               TEXT NOT NULL,
+          type                  TEXT NOT NULL,
+          created               INTEGER NOT NULL,
+          last_accessed         INTEGER NOT NULL,
+          last_consolidated     INTEGER NOT NULL,
+          event_date            INTEGER,
+          emotional_charge      TEXT NOT NULL,
+          fidelity              TEXT NOT NULL DEFAULT 'vivid',
+          confidence            REAL NOT NULL,
+          significance          REAL NOT NULL,
+          stability             REAL NOT NULL DEFAULT 14,
+          reinforcement_count   INTEGER NOT NULL DEFAULT 0,
+          last_reinforced       INTEGER NOT NULL,
+          source_conversations  TEXT NOT NULL DEFAULT '[]',
+          source_type           TEXT NOT NULL DEFAULT 'inferred',
+          narrative_role        TEXT,
+          part_of_story         TEXT,
+          scope_id              TEXT NOT NULL DEFAULT 'default'
+        )`,
+      );
+      rawRun(
+        `INSERT INTO memory_graph_nodes
+         SELECT * FROM memory_graph_nodes_backup`,
+      );
+      rawRun("DROP TABLE memory_graph_nodes_backup");
+
+      // Clear checkpoint
+      rawRun(
+        "DELETE FROM memory_checkpoints WHERE key = ?",
+        MIGRATE_ITEMS_CHECKPOINT,
+      );
+
+      // Insert a legacy item
+      rawRun(
+        `INSERT INTO memory_items (id, kind, subject, statement, status, confidence, importance, scope_id, first_seen_at)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+        "legacy-item-2",
+        "style",
+        "Formal tone",
+        "Always use formal language",
+        "active",
+        0.85,
+        0.7,
+        "default",
+        1700000000000,
+      );
+
+      // This should NOT throw — the migration should succeed without image_refs
+      expect(() => migrateToolCreatedItems()).not.toThrow();
+
+      // Verify the row was migrated
+      const node = rawGet<{ id: string; content: string }>(
+        `SELECT id, content FROM memory_graph_nodes
+         WHERE source_conversations LIKE ?`,
+        "%style:legacy-item-2%",
+      );
+      expect(node).not.toBeNull();
+      expect(node!.content).toBe("Formal tone\nAlways use formal language");
+    } finally {
+      rawRun("DROP TABLE IF EXISTS memory_items");
+      // Restore the full schema for subsequent tests by re-adding image_refs
+      try {
+        rawRun("ALTER TABLE memory_graph_nodes ADD COLUMN image_refs TEXT");
+      } catch {
+        // Column may already exist
+      }
+    }
+  });
+
+  test("skips migration when checkpoint is already set", () => {
+    // Recreate memory_items for legacy data
+    rawRun(
+      `CREATE TABLE IF NOT EXISTS memory_items (
+        id TEXT PRIMARY KEY,
+        kind TEXT NOT NULL,
+        subject TEXT NOT NULL,
+        statement TEXT NOT NULL,
+        status TEXT NOT NULL,
+        confidence REAL NOT NULL,
+        importance REAL,
+        scope_id TEXT NOT NULL DEFAULT 'default',
+        first_seen_at INTEGER NOT NULL,
+        fingerprint TEXT NOT NULL DEFAULT ''
+      )`,
+    );
+
+    try {
+      // Set the checkpoint BEFORE inserting data — the migration should skip
+      setMemoryCheckpoint(MIGRATE_ITEMS_CHECKPOINT, "done");
+
+      rawRun(
+        `INSERT INTO memory_items (id, kind, subject, statement, status, confidence, importance, scope_id, first_seen_at)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+        "legacy-item-3",
+        "relationship",
+        "Colleague",
+        "Works with user on project X",
+        "active",
+        0.75,
+        0.6,
+        "default",
+        1700000000000,
+      );
+
+      // Run migration — should be a no-op because checkpoint is set
+      migrateToolCreatedItems();
+
+      // Assert no rows were inserted into memory_graph_nodes
+      const rows = rawAll<{ id: string }>("SELECT id FROM memory_graph_nodes");
+      expect(rows).toHaveLength(0);
+    } finally {
+      rawRun("DROP TABLE IF EXISTS memory_items");
+    }
+  });
+
+  test("handles missing memory_items table gracefully", () => {
+    // Ensure memory_items table does not exist
+    rawRun("DROP TABLE IF EXISTS memory_items");
+
+    // Clear the checkpoint so migration attempts to run
+    rawRun(
+      "DELETE FROM memory_checkpoints WHERE key = ?",
+      MIGRATE_ITEMS_CHECKPOINT,
+    );
+
+    // Should not throw even though the table doesn't exist
+    expect(() => migrateToolCreatedItems()).not.toThrow();
+
+    // The checkpoint should be set to "done" (migration handled the missing table)
+    const checkpoint = rawGet<{ value: string }>(
+      "SELECT value FROM memory_checkpoints WHERE key = ?",
+      MIGRATE_ITEMS_CHECKPOINT,
+    );
+    expect(checkpoint).not.toBeNull();
+    expect(checkpoint!.value).toBe("done");
+  });
+});

package/src/memory/graph/bootstrap.ts

@@ -21,7 +21,7 @@ import { getWorkspaceDir } from "../../util/platform.js";
 import { getMemoryCheckpoint, setMemoryCheckpoint } from "../checkpoints.js";
 import { getDb, rawAll, rawGet, rawRun } from "../db.js";
 import { enqueueMemoryJob, hasActiveJobOfType } from "../jobs-store.js";
-import { initQdrantClient } from "../qdrant-client.js";
+import { initQdrantClient, resolveQdrantUrl } from "../qdrant-client.js";
 import { conversations, memoryGraphNodes, memorySegments } from "../schema.js";
 import { runGraphExtraction } from "./extraction.js";
 import { countNodes } from "./store.js";
@@ -71,7 +71,7 @@ export async function bootstrapFromHistory(
   // Initialize Qdrant client for inline embedding
   try {
     initQdrantClient({
-      url: config.memory.qdrant.url ?? "http://127.0.0.1:6333",
+      url: resolveQdrantUrl(config),
       collection: config.memory.qdrant.collection,
       vectorSize: config.memory.qdrant.vectorSize,
       onDisk: config.memory.qdrant.onDisk ?? true,
@@ -384,6 +384,10 @@ const KIND_TO_PREFIX: Record<string, string> = {
  * schema. ORM-based inserts include every column in the schema definition,
  * so adding a column in a later migration would cause this migration to
  * fail with "table has no column named …" on upgrade paths.
+ *
+ * The INSERT intentionally omits columns added by later migrations (e.g.
+ * `image_refs` from migration 205) since they default to NULL and
+ * including them would couple this migration to those later schema changes.
  */
 export function migrateToolCreatedItems(): void {
   if (getMemoryCheckpoint(MIGRATE_ITEMS_CHECKPOINT)) return;
@@ -446,8 +450,8 @@ export function migrateToolCreatedItems(): void {
         event_date, emotional_charge, fidelity, confidence, significance,
         stability, reinforcement_count, last_reinforced,
         source_conversations, source_type, narrative_role, part_of_story,
-        image_refs, scope_id
-      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+        scope_id
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
       id,
       content,
       "semantic",
@@ -466,7 +470,6 @@ export function migrateToolCreatedItems(): void {
       "direct",
       null,
       null,
-      null,
       row.scope_id || "default",
     );
 

package/src/memory/graph/compaction.ts

@@ -0,0 +1,299 @@
+// ---------------------------------------------------------------------------
+// Memory Graph — One-off backfill to compact over-long node content
+//
+// Scans memory_graph_nodes for entries whose `content` exceeds a length
+// threshold and rewrites them via an LLM call constrained to the same
+// "1-3 sentences / ~300 chars" rule the extraction prompt now enforces.
+// Preserves all other node fields (significance, emotionalCharge, edges,
+// triggers, image_refs). Each rewrite is logged in memory_graph_node_edits
+// with source="manual" so it is reversible and auditable.
+// ---------------------------------------------------------------------------
+
+import { and, sql } from "drizzle-orm";
+
+import {
+  extractToolUse,
+  getConfiguredProvider,
+  userMessage,
+} from "../../providers/provider-send-message.js";
+import { BackendUnavailableError } from "../../util/errors.js";
+import { getLogger } from "../../util/logger.js";
+import { getDb } from "../db.js";
+import { memoryGraphNodes } from "../schema.js";
+import { recordNodeEdit, updateNode } from "./store.js";
+
+const log = getLogger("graph-compaction");
+
+const COMPACTION_TOOL_SCHEMA = {
+  name: "compact_memory",
+  description: "Rewrite a memory node's content to fit the length cap",
+  input_schema: {
+    type: "object" as const,
+    properties: {
+      compacted_content: {
+        type: "string" as const,
+        description:
+          "The rewritten content — 1-3 sentences, first-person, ~300 characters or fewer",
+      },
+    },
+    required: ["compacted_content"] as const,
+  },
+};
+
+const COMPACTION_SYSTEM_PROMPT = `You are compacting an over-long memory node in an AI assistant's memory graph. The original content exceeds the extraction prompt's 1-3 sentence / ~300 character cap because it was written before the cap existed. Rewrite it to fit.
+
+## Rules
+
+**LENGTH: 1-3 sentences. Target ~300 characters. Hard cap ~400.** No exceptions.
+
+**Preserve:**
+- The core fact, event, or moment being remembered
+- First-person prose style if the original is first-person
+- The essential emotional tone of the memory
+
+**Drop:**
+- Scene-setting and surrounding context
+- Dialogue preservation (unless one short quote IS the memory)
+- Narrative "what it meant" commentary
+- Cataloging of every emotional nuance
+- Image descriptions packed into prose (images live in image_refs, not content)
+
+**Never:**
+- Invent facts not present in the original
+- Change the core subject or emotional valence
+- Pad to hit a target length — shorter than 300 chars is fine if the essence is there
+
+The node's \`emotionalCharge\` and \`significance\` fields (not shown) already carry the weight. Content stays lean.
+
+Call the \`compact_memory\` tool with the rewritten content. No preamble.`;
+
+export interface CompactionCandidate {
+  id: string;
+  beforeLen: number;
+}
+
+export interface CompactionProgress {
+  nodeId: string;
+  beforeLen: number;
+  afterLen: number;
+  action: "compacted" | "skipped" | "failed";
+  newContent?: string;
+  reason?: string;
+}
+
+export interface CompactionResult {
+  /** Nodes whose content exceeded the threshold (before --limit is applied). */
+  scanned: number;
+  /** Nodes actually processed (scanned ∩ limit). */
+  processed: number;
+  compacted: number;
+  skipped: number;
+  failures: number;
+  beforeChars: number;
+  afterChars: number;
+}
+
+export interface CompactionOptions {
+  /** Content length threshold — nodes above this get compacted. Default 400. */
+  threshold?: number;
+  /** If true, write changes. If false, only list candidates (no LLM calls). Default false. */
+  apply?: boolean;
+  /** Max nodes to process. Default: no limit. */
+  limit?: number;
+  /** Called for each node as it is processed in apply mode. */
+  onProgress?: (evt: CompactionProgress) => void;
+  /** Called once with the full candidate list (both preview and apply modes). */
+  onCandidates?: (candidates: CompactionCandidate[]) => void;
+}
+
+/**
+ * Find over-length nodes in the memory graph and (optionally) rewrite their
+ * content to fit the length cap. In preview mode (apply=false) no LLM calls
+ * are made — only the candidate list is returned.
+ */
+export async function compactLongMemories(
+  opts: CompactionOptions = {},
+): Promise<CompactionResult> {
+  const threshold = opts.threshold ?? 400;
+  const apply = opts.apply ?? false;
+
+  const db = getDb();
+  const rows = db
+    .select({
+      id: memoryGraphNodes.id,
+      content: memoryGraphNodes.content,
+    })
+    .from(memoryGraphNodes)
+    .where(
+      and(
+        // Skip already-dead nodes — rewriting content on "gone" fidelity is wasted work
+        sql`${memoryGraphNodes.fidelity} != 'gone'`,
+        sql`LENGTH(${memoryGraphNodes.content}) > ${threshold}`,
+      ),
+    )
+    .all();
+
+  const candidates = opts.limit != null ? rows.slice(0, opts.limit) : rows;
+
+  opts.onCandidates?.(
+    candidates.map((r) => ({ id: r.id, beforeLen: r.content.length })),
+  );
+
+  const result: CompactionResult = {
+    scanned: rows.length,
+    processed: candidates.length,
+    compacted: 0,
+    skipped: 0,
+    failures: 0,
+    beforeChars: candidates.reduce((sum, r) => sum + r.content.length, 0),
+    afterChars: 0,
+  };
+
+  if (!apply) {
+    // Preview mode: record beforeChars as afterChars (no rewrites attempted)
+    result.afterChars = result.beforeChars;
+    return result;
+  }
+
+  if (candidates.length === 0) return result;
+
+  const provider = await getConfiguredProvider("memoryConsolidation");
+  if (!provider) {
+    throw new BackendUnavailableError(
+      "Provider unavailable for memory compaction",
+    );
+  }
+
+  for (const row of candidates) {
+    const beforeLen = row.content.length;
+
+    try {
+      const response = await provider.sendMessage(
+        [
+          userMessage(
+            `Original memory content (length: ${beforeLen} chars):\n\n${row.content}`,
+          ),
+        ],
+        [COMPACTION_TOOL_SCHEMA],
+        COMPACTION_SYSTEM_PROMPT,
+        {
+          config: {
+            callSite: "memoryConsolidation" as const,
+            tool_choice: {
+              type: "tool" as const,
+              name: "compact_memory",
+            },
+          },
+        },
+      );
+
+      const toolBlock = extractToolUse(response);
+      if (!toolBlock) {
+        result.failures += 1;
+        result.afterChars += beforeLen;
+        opts.onProgress?.({
+          nodeId: row.id,
+          beforeLen,
+          afterLen: beforeLen,
+          action: "failed",
+          reason: "no tool_use block in response",
+        });
+        continue;
+      }
+
+      const input = toolBlock.input as { compacted_content?: string };
+      const newContent =
+        typeof input.compacted_content === "string"
+          ? input.compacted_content.trim()
+          : "";
+
+      if (!newContent) {
+        result.failures += 1;
+        result.afterChars += beforeLen;
+        opts.onProgress?.({
+          nodeId: row.id,
+          beforeLen,
+          afterLen: beforeLen,
+          action: "failed",
+          reason: "empty compacted_content",
+        });
+        continue;
+      }
+
+      // If the model didn't actually shrink it, skip — never overwrite with
+      // something equivalent or longer. Preserves the original when the LLM
+      // fails to compress.
+      if (newContent.length >= beforeLen) {
+        result.skipped += 1;
+        result.afterChars += beforeLen;
+        opts.onProgress?.({
+          nodeId: row.id,
+          beforeLen,
+          afterLen: newContent.length,
+          action: "skipped",
+          reason: "rewrite was not shorter than original",
+          newContent,
+        });
+        continue;
+      }
+
+      // Reject rewrites that are shorter than the original but still exceed
+      // the threshold cap — the whole point of the compaction is to land
+      // under the cap, not merely to reduce length.
+      if (newContent.length > threshold) {
+        result.skipped += 1;
+        result.afterChars += beforeLen;
+        opts.onProgress?.({
+          nodeId: row.id,
+          beforeLen,
+          afterLen: newContent.length,
+          action: "skipped",
+          reason: `rewrite still exceeds threshold (${newContent.length} > ${threshold})`,
+          newContent,
+        });
+        continue;
+      }
+
+      // Wrap edit recording + node update in a transaction so they are
+      // atomic: if updateNode fails, the edit record is rolled back. Without
+      // this, a mid-operation throw leaves an orphaned audit row pointing at
+      // a change that never committed.
+      getDb().transaction(() => {
+        recordNodeEdit({
+          nodeId: row.id,
+          previousContent: row.content,
+          newContent,
+          source: "manual",
+        });
+        updateNode(row.id, {
+          content: newContent,
+          lastConsolidated: Date.now(),
+        });
+      });
+
+      result.compacted += 1;
+      result.afterChars += newContent.length;
+      opts.onProgress?.({
+        nodeId: row.id,
+        beforeLen,
+        afterLen: newContent.length,
+        action: "compacted",
+        newContent,
+      });
+    } catch (err) {
+      result.failures += 1;
+      result.afterChars += beforeLen;
+      const reason = err instanceof Error ? err.message : String(err);
+      log.warn({ nodeId: row.id, err: reason }, "Compaction failed for node");
+      opts.onProgress?.({
+        nodeId: row.id,
+        beforeLen,
+        afterLen: beforeLen,
+        action: "failed",
+        reason,
+      });
+    }
+  }
+
+  return result;
+}

package/src/memory/graph/consolidation.ts

@@ -262,7 +262,7 @@ async function identifyDuplicateGroups(
 ): Promise<MemoryNode[][]> {
   if (nodes.length < 2) return [];
 
-  const provider = await getConfiguredProvider();
+  const provider = await getConfiguredProvider("memoryConsolidation");
   if (!provider) return [];
 
   // Compact listing: ID + first 100 chars of content
@@ -282,7 +282,7 @@ async function identifyDuplicateGroups(
     systemPrompt,
     {
       config: {
-        modelIntent: "quality-optimized" as const,
+        callSite: "memoryConsolidation" as const,
         tool_choice: { type: "tool" as const, name: "report_duplicate_groups" },
       },
     },
@@ -428,7 +428,7 @@ async function consolidateChunk(
     return true;
   });
 
-  const provider = await getConfiguredProvider();
+  const provider = await getConfiguredProvider("memoryConsolidation");
   if (!provider) {
     throw new BackendUnavailableError("Provider unavailable for consolidation");
   }
@@ -459,7 +459,7 @@ async function consolidateChunk(
     systemPrompt,
     {
       config: {
-        modelIntent: "quality-optimized" as const,
+        callSite: "memoryConsolidation" as const,
         tool_choice: { type: "tool" as const, name: "consolidate_diff" },
       },
     },