@vellumai/assistant 0.3.5 → 0.3.7

package/src/memory/migrations/002-tool-invocations-fk.ts

@@ -0,0 +1,43 @@
+import { getSqliteFrom, type DrizzleDb } from '../db-connection.js';
+
+/**
+ * Migrate existing tool_invocations table to add FK constraint with ON DELETE CASCADE.
+ * SQLite doesn't support ALTER TABLE ADD CONSTRAINT, so we rebuild the table.
+ * This is idempotent: it checks whether the FK already exists before migrating.
+ */
+export function migrateToolInvocationsFk(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+  const row = raw.query(`SELECT sql FROM sqlite_master WHERE type='table' AND name='tool_invocations'`).get() as { sql: string } | null;
+  if (!row) return; // table doesn't exist yet (will be created above)
+
+  // If the DDL already contains REFERENCES, the FK is in place
+  if (row.sql.includes('REFERENCES')) return;
+
+  raw.exec('PRAGMA foreign_keys = OFF');
+  try {
+    raw.exec(/*sql*/ `
+      BEGIN;
+      CREATE TABLE tool_invocations_new (
+        id TEXT PRIMARY KEY,
+        conversation_id TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE,
+        tool_name TEXT NOT NULL,
+        input TEXT NOT NULL,
+        result TEXT NOT NULL,
+        decision TEXT NOT NULL,
+        risk_level TEXT NOT NULL,
+        duration_ms INTEGER NOT NULL,
+        created_at INTEGER NOT NULL
+      );
+      INSERT INTO tool_invocations_new SELECT t.* FROM tool_invocations t
+        WHERE EXISTS (SELECT 1 FROM conversations c WHERE c.id = t.conversation_id);
+      DROP TABLE tool_invocations;
+      ALTER TABLE tool_invocations_new RENAME TO tool_invocations;
+      COMMIT;
+    `);
+  } catch (e) {
+    try { raw.exec('ROLLBACK'); } catch { /* no active transaction */ }
+    throw e;
+  } finally {
+    raw.exec('PRAGMA foreign_keys = ON');
+  }
+}

package/src/memory/migrations/003-memory-fts-backfill.ts

@@ -0,0 +1,24 @@
+import { getSqliteFrom, type DrizzleDb } from '../db-connection.js';
+
+/**
+ * Backfill FTS rows for existing memory_segments records when upgrading from a
+ * version that may not have had trigger-managed FTS.
+ */
+export function migrateMemoryFtsBackfill(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+  const ftsCountRow = raw.query(`SELECT COUNT(*) AS c FROM memory_segment_fts`).get() as { c: number } | null;
+  const ftsCount = ftsCountRow?.c ?? 0;
+  if (ftsCount > 0) return;
+
+  try {
+    raw.exec('BEGIN');
+    raw.exec(/*sql*/ `
+      INSERT INTO memory_segment_fts(segment_id, text)
+      SELECT id, text FROM memory_segments
+    `);
+    raw.exec('COMMIT');
+  } catch (e) {
+    try { raw.exec('ROLLBACK'); } catch { /* no active transaction */ }
+    throw e;
+  }
+}

package/src/memory/migrations/004-entity-relation-dedup.ts

@@ -0,0 +1,87 @@
+import { getSqliteFrom, type DrizzleDb } from '../db-connection.js';
+
+/**
+ * One-shot migration: merge duplicate relation edges so uniqueness can be
+ * enforced on (source_entity_id, target_entity_id, relation).
+ */
+export function migrateMemoryEntityRelationDedup(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+  const checkpointKey = 'migration_memory_entity_relations_dedup_v1';
+  const checkpoint = raw.query(
+    `SELECT 1 FROM memory_checkpoints WHERE key = ?`,
+  ).get(checkpointKey);
+  if (checkpoint) return;
+
+  // Drop the staging temp table if it was left behind by a previous failed
+  // attempt in the same connection.  TEMP tables survive ROLLBACK (they live
+  // in a separate SQLite schema) so a mid-migration exception can leave the
+  // table present even after the transaction rolls back.  Clearing it here
+  // makes re-entry safe without needing IF NOT EXISTS semantics on the full
+  // CREATE ... AS SELECT.
+  raw.exec(/*sql*/ `DROP TABLE IF EXISTS temp.memory_entity_relation_merge`);
+
+  try {
+    raw.exec('BEGIN');
+
+    raw.exec(/*sql*/ `
+      CREATE TEMP TABLE memory_entity_relation_merge AS
+      WITH ranked AS (
+        SELECT
+          source_entity_id,
+          target_entity_id,
+          relation,
+          first_seen_at,
+          last_seen_at,
+          evidence,
+          ROW_NUMBER() OVER (
+            PARTITION BY source_entity_id, target_entity_id, relation
+            ORDER BY last_seen_at DESC, first_seen_at DESC, id DESC
+          ) AS rank_latest
+        FROM memory_entity_relations
+      )
+      SELECT
+        source_entity_id,
+        target_entity_id,
+        relation,
+        MIN(first_seen_at) AS merged_first_seen_at,
+        MAX(last_seen_at) AS merged_last_seen_at,
+        MAX(CASE WHEN rank_latest = 1 THEN evidence ELSE NULL END) AS merged_evidence
+      FROM ranked
+      GROUP BY source_entity_id, target_entity_id, relation
+    `);
+
+    raw.exec(/*sql*/ `DELETE FROM memory_entity_relations`);
+
+    raw.exec(/*sql*/ `
+      INSERT INTO memory_entity_relations (
+        id,
+        source_entity_id,
+        target_entity_id,
+        relation,
+        evidence,
+        first_seen_at,
+        last_seen_at
+      )
+      SELECT
+        lower(hex(randomblob(16))),
+        source_entity_id,
+        target_entity_id,
+        relation,
+        merged_evidence,
+        merged_first_seen_at,
+        merged_last_seen_at
+      FROM memory_entity_relation_merge
+    `);
+
+    raw.exec(/*sql*/ `DROP TABLE temp.memory_entity_relation_merge`);
+
+    raw.query(
+      `INSERT OR IGNORE INTO memory_checkpoints (key, value, updated_at) VALUES (?, '1', ?)`,
+    ).run(checkpointKey, Date.now());
+
+    raw.exec('COMMIT');
+  } catch (e) {
+    try { raw.exec('ROLLBACK'); } catch { /* no active transaction */ }
+    throw e;
+  }
+}

package/src/memory/migrations/005-fingerprint-scope-unique.ts

@@ -0,0 +1,80 @@
+import { getSqliteFrom, type DrizzleDb } from '../db-connection.js';
+
+/**
+ * Migrate from a column-level UNIQUE on fingerprint to a compound unique
+ * index on (fingerprint, scope_id) so that the same item can exist in
+ * different scopes independently.
+ */
+export function migrateMemoryItemsFingerprintScopeUnique(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+  const checkpointKey = 'migration_memory_items_fingerprint_scope_unique_v1';
+  const checkpoint = raw.query(
+    `SELECT 1 FROM memory_checkpoints WHERE key = ?`,
+  ).get(checkpointKey);
+  if (checkpoint) return;
+
+  // Check if the old column-level UNIQUE constraint still exists by inspecting
+  // the CREATE TABLE DDL for the word UNIQUE (the PK also creates an autoindex,
+  // so we cannot rely on sqlite_autoindex_* presence alone).
+  const tableDdl = raw.query(
+    `SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'memory_items'`,
+  ).get() as { sql: string } | null;
+  if (!tableDdl || !tableDdl.sql.match(/fingerprint\s+TEXT\s+NOT\s+NULL\s+UNIQUE/i)) {
+    // No column-level UNIQUE on fingerprint — either fresh DB or already migrated.
+    raw.query(
+      `INSERT OR IGNORE INTO memory_checkpoints (key, value, updated_at) VALUES (?, '1', ?)`,
+    ).run(checkpointKey, Date.now());
+    return;
+  }
+
+  // Rebuild the table without the column-level UNIQUE constraint.
+  raw.exec('PRAGMA foreign_keys = OFF');
+  try {
+    raw.exec('BEGIN');
+
+    // Create new table without UNIQUE on fingerprint — all other columns
+    // match the latest schema (including migration-added columns).
+    raw.exec(/*sql*/ `
+      CREATE TABLE memory_items_new (
+        id TEXT PRIMARY KEY,
+        kind TEXT NOT NULL,
+        subject TEXT NOT NULL,
+        statement TEXT NOT NULL,
+        status TEXT NOT NULL,
+        confidence REAL NOT NULL,
+        fingerprint TEXT NOT NULL,
+        first_seen_at INTEGER NOT NULL,
+        last_seen_at INTEGER NOT NULL,
+        last_used_at INTEGER,
+        importance REAL,
+        access_count INTEGER NOT NULL DEFAULT 0,
+        valid_from INTEGER,
+        invalid_at INTEGER,
+        verification_state TEXT NOT NULL DEFAULT 'assistant_inferred',
+        scope_id TEXT NOT NULL DEFAULT 'default'
+      )
+    `);
+
+    raw.exec(/*sql*/ `
+      INSERT INTO memory_items_new
+      SELECT id, kind, subject, statement, status, confidence, fingerprint,
+             first_seen_at, last_seen_at, last_used_at, importance, access_count,
+             valid_from, invalid_at, verification_state, scope_id
+      FROM memory_items
+    `);
+
+    raw.exec(/*sql*/ `DROP TABLE memory_items`);
+    raw.exec(/*sql*/ `ALTER TABLE memory_items_new RENAME TO memory_items`);
+
+    raw.query(
+      `INSERT OR IGNORE INTO memory_checkpoints (key, value, updated_at) VALUES (?, '1', ?)`,
+    ).run(checkpointKey, Date.now());
+
+    raw.exec('COMMIT');
+  } catch (e) {
+    try { raw.exec('ROLLBACK'); } catch { /* no active transaction */ }
+    throw e;
+  } finally {
+    raw.exec('PRAGMA foreign_keys = ON');
+  }
+}

package/src/memory/migrations/006-scope-salted-fingerprints.ts

@@ -0,0 +1,62 @@
+import { computeMemoryFingerprint } from '../fingerprint.js';
+import { getSqliteFrom, type DrizzleDb } from '../db-connection.js';
+
+/**
+ * One-shot migration: recompute fingerprints for existing memory items to
+ * include the scope_id prefix introduced in the scope-salted fingerprint PR.
+ *
+ * Old format: sha256(`${kind}|${subject.toLowerCase()}|${statement.toLowerCase()}`)
+ * New format: sha256(`${scopeId}|${kind}|${subject.toLowerCase()}|${statement.toLowerCase()}`)
+ *
+ * Without this migration, pre-upgrade items would never match on re-extraction,
+ * causing duplicates and broken deduplication.
+ */
+export function migrateMemoryItemsScopeSaltedFingerprints(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+  const checkpointKey = 'migration_memory_items_scope_salted_fingerprints_v1';
+  const checkpoint = raw.query(
+    `SELECT 1 FROM memory_checkpoints WHERE key = ?`,
+  ).get(checkpointKey);
+  if (checkpoint) return;
+
+  interface ItemRow {
+    id: string;
+    kind: string;
+    subject: string;
+    statement: string;
+    scope_id: string;
+  }
+
+  const items = raw.query(
+    `SELECT id, kind, subject, statement, scope_id FROM memory_items`,
+  ).all() as ItemRow[];
+
+  if (items.length === 0) {
+    raw.query(
+      `INSERT OR IGNORE INTO memory_checkpoints (key, value, updated_at) VALUES (?, '1', ?)`,
+    ).run(checkpointKey, Date.now());
+    return;
+  }
+
+  try {
+    raw.exec('BEGIN');
+
+    const updateStmt = raw.prepare(
+      `UPDATE memory_items SET fingerprint = ? WHERE id = ?`,
+    );
+
+    for (const item of items) {
+      const fingerprint = computeMemoryFingerprint(item.scope_id, item.kind, item.subject, item.statement);
+      updateStmt.run(fingerprint, item.id);
+    }
+
+    raw.query(
+      `INSERT OR IGNORE INTO memory_checkpoints (key, value, updated_at) VALUES (?, '1', ?)`,
+    ).run(checkpointKey, Date.now());
+
+    raw.exec('COMMIT');
+  } catch (e) {
+    try { raw.exec('ROLLBACK'); } catch { /* no active transaction */ }
+    throw e;
+  }
+}

package/src/memory/migrations/007-assistant-id-to-self.ts

@@ -0,0 +1,254 @@
+import { getSqliteFrom, type DrizzleDb } from '../db-connection.js';
+
+/**
+ * One-shot migration: normalize all assistant_id values in assistant-scoped tables
+ * to "self" so they are visible after the daemon switched to the implicit single-tenant
+ * identity.
+ *
+ * Before this change, rows were keyed by the real assistantId string passed via the
+ * HTTP route. After the route change, all lookups use the constant "self". Without this
+ * migration an upgraded daemon would see empty history / attachment lists for existing
+ * data that was stored under the old assistantId.
+ *
+ * Affected tables:
+ *   - conversation_keys   UNIQUE (assistant_id, conversation_key)
+ *   - attachments         UNIQUE (assistant_id, content_hash) WHERE content_hash IS NOT NULL
+ *   - channel_inbound_events  UNIQUE (assistant_id, source_channel, external_chat_id, external_message_id)
+ *   - message_runs        no unique constraint on assistant_id
+ *
+ * Data-safety guarantees:
+ *   - conversation_keys: when a key exists under both 'self' and a real assistantId, the
+ *     'self' row is updated to point to the real-assistantId conversation (which holds the
+ *     historical message thread). The 'self' conversation may be orphaned but is not deleted.
+ *   - attachments: message_attachments links are remapped to the surviving attachment before
+ *     any duplicate row is deleted, so no message loses its attachment metadata.
+ *   - channel_inbound_events: only delivery-tracking metadata, not user content; dedup
+ *     keeps one row per unique (channel, chat, message) tuple.
+ *   - All conversations and messages remain untouched — only assistant_id index columns
+ *     and key-lookup rows are modified.
+ */
+export function migrateAssistantIdToSelf(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+  const checkpointKey = 'migration_normalize_assistant_id_to_self_v1';
+  const checkpoint = raw.query(
+    `SELECT 1 FROM memory_checkpoints WHERE key = ?`,
+  ).get(checkpointKey);
+  if (checkpoint) return;
+
+  // On fresh installs the tables are created without assistant_id (PR 7+). Skip the
+  // migration if NONE of the four affected tables have the column — pre-seed the
+  // checkpoint so subsequent startups are also skipped. Checking all four (not just
+  // conversation_keys) avoids a false negative on very old installs where
+  // conversation_keys may not exist yet but other tables still carry assistant_id data.
+  const affectedTables = ['conversation_keys', 'attachments', 'channel_inbound_events', 'message_runs'];
+  const anyHasAssistantId = affectedTables.some((tbl) => {
+    const ddl = raw.query(
+      `SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?`,
+    ).get(tbl) as { sql: string } | null;
+    return ddl?.sql.includes('assistant_id') ?? false;
+  });
+  if (!anyHasAssistantId) {
+    raw.query(
+      `INSERT OR IGNORE INTO memory_checkpoints (key, value, updated_at) VALUES (?, '1', ?)`,
+    ).run(checkpointKey, Date.now());
+    return;
+  }
+
+  // Helper: returns true if the given table's current DDL contains 'assistant_id'.
+  const tableHasAssistantId = (tbl: string): boolean => {
+    const ddl = raw.query(
+      `SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?`,
+    ).get(tbl) as { sql: string } | null;
+    return ddl?.sql.includes('assistant_id') ?? false;
+  };
+
+  try {
+    raw.exec('BEGIN');
+
+    // Each section is guarded so that SQL referencing assistant_id is only executed
+    // when the column still exists in that table. This handles mixed-schema states
+    // (e.g., very old installs where some tables may already lack the column).
+
+    // conversation_keys: UNIQUE (assistant_id, conversation_key)
+    if (tableHasAssistantId('conversation_keys')) {
+      // Step 1: Among non-self rows, keep only one per conversation_key so the
+      //         bulk UPDATE cannot hit a (non-self-A, key) + (non-self-B, key) collision.
+      raw.exec(/*sql*/ `
+        DELETE FROM conversation_keys
+        WHERE assistant_id != 'self'
+          AND rowid NOT IN (
+            SELECT MIN(rowid) FROM conversation_keys
+            WHERE assistant_id != 'self'
+            GROUP BY conversation_key
+          )
+      `);
+      // Step 2: For 'self' rows that have a non-self counterpart with the same
+      //         conversation_key, update the 'self' row to use the non-self row's
+      //         conversation_id. This preserves the historical conversation (which
+      //         has the message history from before the route change) rather than
+      //         discarding it in favour of a potentially-empty 'self' conversation.
+      raw.exec(/*sql*/ `
+        UPDATE conversation_keys
+        SET conversation_id = (
+          SELECT ck_ns.conversation_id
+          FROM conversation_keys ck_ns
+          WHERE ck_ns.assistant_id != 'self'
+            AND ck_ns.conversation_key = conversation_keys.conversation_key
+          ORDER BY ck_ns.rowid
+          LIMIT 1
+        )
+        WHERE assistant_id = 'self'
+          AND EXISTS (
+            SELECT 1 FROM conversation_keys ck_ns
+            WHERE ck_ns.assistant_id != 'self'
+              AND ck_ns.conversation_key = conversation_keys.conversation_key
+          )
+      `);
+      // Step 3: Delete the now-redundant non-self rows (their conversation_ids
+      //         have been preserved in the 'self' rows above).
+      raw.exec(/*sql*/ `
+        DELETE FROM conversation_keys
+        WHERE assistant_id != 'self'
+          AND EXISTS (
+            SELECT 1 FROM conversation_keys ck2
+            WHERE ck2.assistant_id = 'self'
+              AND ck2.conversation_key = conversation_keys.conversation_key
+          )
+      `);
+      // Step 4: Remaining non-self rows have no 'self' counterpart — safe to bulk-update.
+      raw.exec(/*sql*/ `
+        UPDATE conversation_keys SET assistant_id = 'self' WHERE assistant_id != 'self'
+      `);
+    }
+
+    // attachments: UNIQUE (assistant_id, content_hash) WHERE content_hash IS NOT NULL
+    //
+    // message_attachments rows reference attachment IDs with ON DELETE CASCADE, so we
+    // must remap links to the surviving row BEFORE deleting duplicates to avoid
+    // silently dropping attachment metadata from messages.
+    if (tableHasAssistantId('attachments')) {
+      // Step 1: Remap message_attachments from non-self duplicates to their survivor
+      //         (MIN rowid per content_hash group), then delete the duplicates.
+      raw.exec(/*sql*/ `
+        UPDATE message_attachments
+        SET attachment_id = (
+          SELECT a_survivor.id
+          FROM attachments a_survivor
+          WHERE a_survivor.assistant_id != 'self'
+            AND a_survivor.content_hash = (
+              SELECT a_dup.content_hash FROM attachments a_dup
+              WHERE a_dup.id = message_attachments.attachment_id
+            )
+          ORDER BY a_survivor.rowid
+          LIMIT 1
+        )
+        WHERE attachment_id IN (
+          SELECT id FROM attachments
+          WHERE assistant_id != 'self'
+            AND content_hash IS NOT NULL
+            AND rowid NOT IN (
+              SELECT MIN(rowid) FROM attachments
+              WHERE assistant_id != 'self' AND content_hash IS NOT NULL
+              GROUP BY content_hash
+            )
+        )
+      `);
+      raw.exec(/*sql*/ `
+        DELETE FROM attachments
+        WHERE assistant_id != 'self'
+          AND content_hash IS NOT NULL
+          AND rowid NOT IN (
+            SELECT MIN(rowid) FROM attachments
+            WHERE assistant_id != 'self'
+              AND content_hash IS NOT NULL
+            GROUP BY content_hash
+          )
+      `);
+      // Step 2: Remap message_attachments from non-self rows conflicting with a 'self'
+      //         row to the 'self' row, then delete the now-unlinked non-self rows.
+      raw.exec(/*sql*/ `
+        UPDATE message_attachments
+        SET attachment_id = (
+          SELECT a_self.id
+          FROM attachments a_self
+          WHERE a_self.assistant_id = 'self'
+            AND a_self.content_hash = (
+              SELECT a_ns.content_hash FROM attachments a_ns
+              WHERE a_ns.id = message_attachments.attachment_id
+            )
+          LIMIT 1
+        )
+        WHERE attachment_id IN (
+          SELECT id FROM attachments
+          WHERE assistant_id != 'self'
+            AND content_hash IS NOT NULL
+            AND EXISTS (
+              SELECT 1 FROM attachments a2
+              WHERE a2.assistant_id = 'self'
+                AND a2.content_hash = attachments.content_hash
+            )
+        )
+      `);
+      raw.exec(/*sql*/ `
+        DELETE FROM attachments
+        WHERE assistant_id != 'self'
+          AND content_hash IS NOT NULL
+          AND EXISTS (
+            SELECT 1 FROM attachments a2
+            WHERE a2.assistant_id = 'self'
+              AND a2.content_hash = attachments.content_hash
+          )
+      `);
+      // Step 3: Bulk-update remaining non-self rows.
+      raw.exec(/*sql*/ `
+        UPDATE attachments SET assistant_id = 'self' WHERE assistant_id != 'self'
+      `);
+    }
+
+    // channel_inbound_events: UNIQUE (assistant_id, source_channel, external_chat_id, external_message_id)
+    if (tableHasAssistantId('channel_inbound_events')) {
+      // Step 1: Dedup non-self rows sharing the same (source_channel, external_chat_id, external_message_id).
+      raw.exec(/*sql*/ `
+        DELETE FROM channel_inbound_events
+        WHERE assistant_id != 'self'
+          AND rowid NOT IN (
+            SELECT MIN(rowid) FROM channel_inbound_events
+            WHERE assistant_id != 'self'
+            GROUP BY source_channel, external_chat_id, external_message_id
+          )
+      `);
+      // Step 2: Delete non-self rows conflicting with existing 'self' rows.
+      raw.exec(/*sql*/ `
+        DELETE FROM channel_inbound_events
+        WHERE assistant_id != 'self'
+          AND EXISTS (
+            SELECT 1 FROM channel_inbound_events e2
+            WHERE e2.assistant_id = 'self'
+              AND e2.source_channel = channel_inbound_events.source_channel
+              AND e2.external_chat_id = channel_inbound_events.external_chat_id
+              AND e2.external_message_id = channel_inbound_events.external_message_id
+          )
+      `);
+      // Step 3: Bulk-update remaining non-self rows.
+      raw.exec(/*sql*/ `
+        UPDATE channel_inbound_events SET assistant_id = 'self' WHERE assistant_id != 'self'
+      `);
+    }
+
+    // message_runs: no unique constraint on assistant_id — simple bulk update
+    if (tableHasAssistantId('message_runs')) {
+      raw.exec(/*sql*/ `
+        UPDATE message_runs SET assistant_id = 'self' WHERE assistant_id != 'self'
+      `);
+    }
+
+    raw.query(
+      `INSERT OR IGNORE INTO memory_checkpoints (key, value, updated_at) VALUES (?, '1', ?)`,
+    ).run(checkpointKey, Date.now());
+
+    raw.exec('COMMIT');
+  } catch (e) {
+    try { raw.exec('ROLLBACK'); } catch { /* no active transaction */ }
+    throw e;
+  }
+}