@vellumai/assistant 0.3.5 → 0.3.7

package/src/memory/migrations/registry.ts

@@ -0,0 +1,79 @@
+export interface MigrationRegistryEntry {
+  /** The checkpoint key written to memory_checkpoints on completion. */
+  key: string;
+  /** Monotonic version number used for ordering assertions. */
+  version: number;
+  /** Keys of other migrations that must complete before this one runs. */
+  dependsOn?: string[];
+  /** Human-readable description for diagnostics and future authorship guidance. */
+  description: string;
+}
+
+// ---------------------------------------------------------------------------
+// Central registry of all checkpoint-based one-shot migrations.  Each entry
+// carries a monotonic version number (for documentation / ordering assertions)
+// and an optional list of prerequisite checkpoint keys that must already be
+// completed before this migration runs.
+//
+// Migrations that use pure DDL guards (CREATE TABLE IF NOT EXISTS, index
+// presence checks, ALTER TABLE ADD COLUMN try/catch) are inherently idempotent
+// and do not need entries here — they are safe to re-run on every startup.
+// ---------------------------------------------------------------------------
+
+export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
+  {
+    key: 'migration_job_deferrals',
+    version: 1,
+    description: 'Reconcile legacy deferral history from attempts column into deferrals column',
+  },
+  {
+    key: 'migration_memory_entity_relations_dedup_v1',
+    version: 2,
+    description: 'Deduplicate entity relation edges before enforcing the (source, target, relation) unique index',
+  },
+  {
+    key: 'migration_memory_items_fingerprint_scope_unique_v1',
+    version: 3,
+    description: 'Replace column-level UNIQUE on fingerprint with compound (fingerprint, scope_id) unique index',
+  },
+  {
+    key: 'migration_memory_items_scope_salted_fingerprints_v1',
+    version: 4,
+    dependsOn: ['migration_memory_items_fingerprint_scope_unique_v1'],
+    description: 'Recompute memory item fingerprints to include scope_id prefix after schema change',
+  },
+  {
+    key: 'migration_normalize_assistant_id_to_self_v1',
+    version: 5,
+    description: 'Normalize all assistant_id values in scoped tables to the implicit "self" single-tenant identity',
+  },
+  {
+    key: 'migration_remove_assistant_id_columns_v1',
+    version: 6,
+    dependsOn: ['migration_normalize_assistant_id_to_self_v1'],
+    description: 'Rebuild four tables to drop the assistant_id column after normalization',
+  },
+  {
+    key: 'migration_remove_assistant_id_lue_v1',
+    version: 7,
+    dependsOn: ['migration_normalize_assistant_id_to_self_v1'],
+    description: 'Remove assistant_id column from llm_usage_events (separate checkpoint from the four-table migration)',
+  },
+  {
+    key: 'backfill_inbox_thread_state_from_bindings',
+    version: 8,
+    description: 'Seed assistant_inbox_thread_state from external_conversation_bindings',
+  },
+  {
+    key: 'drop_active_search_index_v1',
+    version: 9,
+    description: 'Drop old idx_memory_items_active_search so it can be recreated with updated covering columns',
+  },
+];
+
+export interface MigrationValidationResult {
+  /** Keys of migrations whose checkpoint has value 'started' — started but never completed. */
+  crashed: string[];
+  /** Pairs where a completed migration's declared prerequisite is missing from checkpoints. */
+  dependencyViolations: Array<{ migration: string; missingDependency: string }>;
+}

package/src/memory/migrations/validate-migration-state.ts

@@ -0,0 +1,69 @@
+import { getSqliteFrom, type DrizzleDb } from '../db-connection.js';
+import { getLogger } from '../../util/logger.js';
+import { MIGRATION_REGISTRY, type MigrationValidationResult } from './registry.js';
+
+const log = getLogger('memory-db');
+
+/**
+ * Validate the applied migration state against the registry at startup.
+ *
+ * Logs warnings when a migration started but never completed (crash detected),
+ * and logs errors when a migration was applied but a declared prerequisite is
+ * missing from the checkpoints table (dependency ordering violation).
+ *
+ * Returns structured diagnostic data so callers (e.g. tests) can assert on the
+ * specific issues detected without having to re-query the DB or inspect logs.
+ *
+ * Call this AFTER all DDL and migration functions have run so that the final
+ * state is inspected.
+ */
+export function validateMigrationState(database: DrizzleDb): MigrationValidationResult {
+  const raw = getSqliteFrom(database);
+
+  let rows: Array<{ key: string; value: string }>;
+  try {
+    rows = raw.query(`SELECT key, value FROM memory_checkpoints`).all() as Array<{ key: string; value: string }>;
+  } catch {
+    // memory_checkpoints may not exist on a very old database; skip.
+    return { crashed: [], dependencyViolations: [] };
+  }
+
+  // Detect crashed migrations: a checkpoint value of 'started' means the
+  // migration wrote its start marker but never reached the completion INSERT.
+  // The migration will re-run on the next startup (its own idempotency guard
+  // will determine safety), but we surface a warning for visibility.
+  const crashed = rows.filter((r) => r.value === 'started').map((r) => r.key);
+  if (crashed.length > 0) {
+    log.warn(
+      { crashed },
+      'Crashed migrations detected — these migrations started but never completed; they will re-run on next startup',
+    );
+  }
+
+  // Only rows whose value is NOT 'started' represent truly completed migrations.
+  // In-progress/crashed checkpoints (value = 'started') must not count as applied
+  // dependencies — the migration never finished, so its postconditions are unmet.
+  const completed = new Set(rows.filter((r) => r.value !== 'started').map((r) => r.key));
+
+  const dependencyViolations: Array<{ migration: string; missingDependency: string }> = [];
+
+  // Validate dependency ordering.
+  for (const entry of MIGRATION_REGISTRY) {
+    if (!entry.dependsOn || entry.dependsOn.length === 0) continue;
+    // Only check entries that have been completed — unapplied or in-progress
+    // migrations have not had a chance to violate their prerequisites yet.
+    if (!completed.has(entry.key)) continue;
+
+    for (const dep of entry.dependsOn) {
+      if (!completed.has(dep)) {
+        log.error(
+          { migration: entry.key, missingDependency: dep, version: entry.version },
+          'Migration dependency violation: this migration is marked complete but its declared prerequisite has no checkpoint — database schema may be inconsistent',
+        );
+        dependencyViolations.push({ migration: entry.key, missingDependency: dep });
+      }
+    }
+  }
+
+  return { crashed, dependencyViolations };
+}

package/src/memory/qdrant-manager.ts

@@ -1,9 +1,11 @@
 import { existsSync, readFileSync, writeFileSync, unlinkSync, mkdirSync, chmodSync } from 'node:fs';
 import { join, dirname } from 'node:path';
 import { arch, platform } from 'node:os';
+import { createHash } from 'node:crypto';
 import type { Subprocess } from 'bun';
 import { getLogger } from '../util/logger.js';
 import { getDataDir } from '../util/platform.js';
+import { getQdrantUrlEnv } from '../config/env.js';
 
 const log = getLogger('qdrant-manager');
 
@@ -15,6 +17,12 @@ const SHUTDOWN_GRACE_MS = 5_000;
 export interface QdrantManagerConfig {
   url: string;
   storagePath?: string;
+  /** Override readyz poll interval (ms). Default: 200 */
+  readyzPollIntervalMs?: number;
+  /** Override readyz timeout (ms). Default: 30 000 */
+  readyzTimeoutMs?: number;
+  /** Override SIGTERM→SIGKILL grace period (ms). Default: 5 000 */
+  shutdownGraceMs?: number;
 }
 
 /**
@@ -36,6 +44,9 @@ export class QdrantManager {
   private readonly storagePath: string;
   private readonly pidPath: string;
   private readonly isExternal: boolean;
+  private readonly readyzPollIntervalMs: number;
+  private readonly readyzTimeoutMs: number;
+  private readonly shutdownGraceMs: number;
 
   constructor(config: QdrantManagerConfig) {
     this.url = config.url;
@@ -45,8 +56,12 @@ export class QdrantManager {
     this.storagePath = config.storagePath ?? join(getDataDir(), 'qdrant');
     this.pidPath = join(getDataDir(), 'qdrant', 'qdrant.pid');
 
+    this.readyzPollIntervalMs = config.readyzPollIntervalMs ?? READYZ_POLL_INTERVAL_MS;
+    this.readyzTimeoutMs = config.readyzTimeoutMs ?? READYZ_TIMEOUT_MS;
+    this.shutdownGraceMs = config.shutdownGraceMs ?? SHUTDOWN_GRACE_MS;
+
     // External mode only if QDRANT_URL is explicitly set
-    this.isExternal = Boolean(process.env.QDRANT_URL?.trim());
+    this.isExternal = Boolean(getQdrantUrlEnv());
   }
 
   async start(): Promise<void> {
@@ -107,7 +122,7 @@ export class QdrantManager {
     // Wait for graceful shutdown
     const graceful = await Promise.race([
       this.process.exited.then(() => true),
-      new Promise<false>((resolve) => setTimeout(() => resolve(false), SHUTDOWN_GRACE_MS)),
+      new Promise<false>((resolve) => setTimeout(() => resolve(false), this.shutdownGraceMs)),
     ]);
 
     if (!graceful) {
@@ -146,16 +161,42 @@ export class QdrantManager {
     }
 
     const filename = `qdrant-${target}.tar.gz`;
-    const url = `https://github.com/qdrant/qdrant/releases/download/v${QDRANT_VERSION}/${filename}`;
+    const baseUrl = `https://github.com/qdrant/qdrant/releases/download/v${QDRANT_VERSION}`;
+    const url = `${baseUrl}/${filename}`;
+    const checksumUrl = `${baseUrl}/${filename}.sha256`;
+
     log.info({ url, binaryPath }, 'Downloading Qdrant binary');
 
-    const response = await fetch(url);
+    // Fetch the tarball and its SHA-256 checksum in parallel
+    const [response, checksumResponse] = await Promise.all([
+      fetch(url),
+      fetch(checksumUrl),
+    ]);
+
     if (!response.ok) {
       throw new Error(`Failed to download Qdrant: ${response.status} ${response.statusText} from ${url}`);
     }
 
     const tarball = await response.arrayBuffer();
 
+    // Verify SHA-256 integrity if the checksum file is available
+    if (checksumResponse.ok) {
+      const checksumText = (await checksumResponse.text()).trim();
+      // Checksum files contain "<hex>  <filename>" or just "<hex>"
+      const expectedHash = checksumText.split(/\s+/)[0].toLowerCase();
+      const actualHash = createHash('sha256').update(Buffer.from(tarball)).digest('hex');
+
+      if (actualHash !== expectedHash) {
+        throw new Error(
+          `Qdrant binary checksum mismatch! ` +
+          `expected=${expectedHash} actual=${actualHash} url=${url}`,
+        );
+      }
+      log.info({ hash: actualHash }, 'Qdrant binary checksum verified');
+    } else {
+      log.warn({ checksumUrl, status: checksumResponse.status }, 'Could not fetch Qdrant checksum — skipping integrity check');
+    }
+
     // Extract the qdrant binary from the tarball
     const binDir = dirname(binaryPath);
     mkdirSync(binDir, { recursive: true });
@@ -185,16 +226,16 @@ export class QdrantManager {
 
   private async waitForReady(): Promise<void> {
     const start = Date.now();
-    while (Date.now() - start < READYZ_TIMEOUT_MS) {
+    while (Date.now() - start < this.readyzTimeoutMs) {
       try {
         const res = await fetch(`${this.url}/readyz`);
         if (res.ok) return;
       } catch {
         // Not ready yet
       }
-      await Bun.sleep(READYZ_POLL_INTERVAL_MS);
+      await Bun.sleep(this.readyzPollIntervalMs);
     }
-    throw new Error(`Qdrant did not become ready within ${READYZ_TIMEOUT_MS}ms at ${this.url}`);
+    throw new Error(`Qdrant did not become ready within ${this.readyzTimeoutMs}ms at ${this.url}`);
   }
 
   private getBinaryPath(): string {
@@ -203,7 +244,7 @@ export class QdrantManager {
 
   private cleanupStaleProcess(): void {
     const pid = this.readPid();
-    if (pid === null) return;
+    if (pid == null) return;
 
     try {
       process.kill(pid, 0); // Check if process exists

package/src/memory/query-builder.ts

@@ -23,7 +23,7 @@ export function buildMemoryQuery(
   const requestText = clampSection(userRequest.trim(), maxUserRequestChars);
   const sessionSummary = messages
     .map((message) => getSummaryFromContextMessage(message))
-    .find((summary): summary is string => summary !== null);
+    .find((summary): summary is string => summary != null);
 
   const content = requestText.length > 0 ? requestText : '(empty)';
 

package/src/memory/raw-query.ts

@@ -0,0 +1,119 @@
+/**
+ * Type-safe wrappers for raw SQL queries against bun:sqlite.
+ *
+ * ## When to use raw SQL vs Drizzle ORM
+ *
+ * **Default to Drizzle** for all standard CRUD operations. Use raw SQL only when
+ * Drizzle cannot express the query or when a raw API is required:
+ *
+ * - **FTS5 operations**: MATCH operator, bm25() ranking, virtual table
+ *   INSERT/DELETE. Drizzle has no FTS5 support.
+ *
+ * - **Schema migrations**: DDL statements (CREATE TABLE, ALTER TABLE, DROP TABLE),
+ *   PRAGMA control, and transaction-wrapped table rebuilds. These are structural
+ *   operations outside Drizzle's query-building scope.
+ *
+ * - **Affected-row checks after Drizzle .run()**: Drizzle's bun:sqlite adapter
+ *   returns void from .run(), so checking changes() requires the raw client.
+ *   Use `rawChanges()` for this.
+ *
+ * - **INSERT OR IGNORE / ON CONFLICT**: SQLite-specific upsert syntax that
+ *   Drizzle's bun:sqlite adapter doesn't fully support.
+ *
+ * - **Atomic in-place updates**: Expressions like `SET count = count + 1` can
+ *   use Drizzle's `sql` template, but raw SQL is acceptable when simpler.
+ *
+ * - **Bulk deletes across virtual tables**: Operations like clearing
+ *   memory_segment_fts that reference virtual tables not modeled in Drizzle.
+ *
+ * For everything else — selects, inserts, updates, deletes, joins, aggregations,
+ * filtering, ordering, pagination — use Drizzle.
+ */
+
+import type { Database, SQLQueryBindings } from 'bun:sqlite';
+import { getSqlite, type DrizzleDb, getSqliteFrom } from './db-connection.js';
+
+type SqlParam = SQLQueryBindings;
+
+// ---------------------------------------------------------------------------
+// Typed query helpers (global Drizzle instance)
+// ---------------------------------------------------------------------------
+
+/** Execute a raw SQL query and return a single typed row, or null if no match. */
+export function rawGet<T>(sql: string, ...params: SqlParam[]): T | null {
+  return (getSqlite().query(sql).get(...params) as T) ?? null;
+}
+
+/** Execute a raw SQL query and return all matching rows with type safety. */
+export function rawAll<T>(sql: string, ...params: SqlParam[]): T[] {
+  return getSqlite().query(sql).all(...params) as T[];
+}
+
+/**
+ * Execute a raw SQL statement (INSERT/UPDATE/DELETE) and return the number
+ * of affected rows.
+ */
+export function rawRun(sql: string, ...params: SqlParam[]): number {
+  getSqlite().query(sql).run(...params);
+  return rawChanges();
+}
+
+/** Execute batch SQL (multiple statements, no bindings). */
+export function rawExec(sql: string): void {
+  getSqlite().exec(sql);
+}
+
+/**
+ * Return the number of rows affected by the most recent INSERT/UPDATE/DELETE.
+ *
+ * Useful after a Drizzle `.run()` call, since Drizzle's bun:sqlite adapter
+ * returns void and discards the changes count.
+ */
+export function rawChanges(): number {
+  return (getSqlite().query('SELECT changes() AS c').get() as { c: number }).c;
+}
+
+// ---------------------------------------------------------------------------
+// Typed query helpers (explicit Drizzle instance)
+//
+// Used by migrations and tests that receive the Drizzle instance as a
+// parameter rather than using the global singleton.
+// ---------------------------------------------------------------------------
+
+/** Execute a raw SQL query against a specific Drizzle instance and return a single typed row. */
+export function rawGetFrom<T>(db: DrizzleDb, sql: string, ...params: SqlParam[]): T | null {
+  return (getSqliteFrom(db).query(sql).get(...params) as T) ?? null;
+}
+
+/** Execute a raw SQL query against a specific Drizzle instance and return all matching rows. */
+export function rawAllFrom<T>(db: DrizzleDb, sql: string, ...params: SqlParam[]): T[] {
+  return getSqliteFrom(db).query(sql).all(...params) as T[];
+}
+
+/**
+ * Execute a raw SQL statement against a specific Drizzle instance and return
+ * affected row count.
+ */
+export function rawRunFrom(db: DrizzleDb, sql: string, ...params: SqlParam[]): number {
+  const sqlite = getSqliteFrom(db);
+  sqlite.query(sql).run(...params);
+  return (sqlite.query('SELECT changes() AS c').get() as { c: number }).c;
+}
+
+/** Execute batch SQL against a specific Drizzle instance. */
+export function rawExecFrom(db: DrizzleDb, sql: string): void {
+  getSqliteFrom(db).exec(sql);
+}
+
+/**
+ * Create a prepared statement from the raw SQLite client.
+ * Useful when you need to execute the same parameterized query in a loop.
+ */
+export function rawPrepare(sql: string): ReturnType<Database['prepare']> {
+  return getSqlite().prepare(sql);
+}
+
+/** Create a prepared statement from a specific Drizzle instance. */
+export function rawPrepareFrom(db: DrizzleDb, sql: string): ReturnType<Database['prepare']> {
+  return getSqliteFrom(db).prepare(sql);
+}

package/src/memory/recall-cache.ts

@@ -68,11 +68,14 @@ export function getCachedRecall(
     _cache.delete(key);
     return undefined;
   }
+  // Move to end of Map iteration order so it's treated as most-recently-used
+  _cache.delete(key);
+  _cache.set(key, entry);
   return entry.result;
 }
 
 /**
- * Store a recall result in the cache. Evicts oldest entries when full.
+ * Store a recall result in the cache. Evicts least-recently-used entries when full.
  *
  * When `snapshotVersion` is provided, the entry is only stored if the
  * snapshot still matches the current global version — this prevents a