memorix 1.2.3 → 1.2.5

package/src/store/bun-sqlite-compat.ts

@@ -1,21 +1,120 @@
 /**
  * Bun SQLite Compatibility Layer
  *
- * Provides a better-sqlite3 compatible API using bun:sqlite when running under Bun.
- * Falls back to better-sqlite3 when running under Node.js.
+ * Provides a small better-sqlite3 compatibility layer for the available local
+ * SQLite implementation. Node 22 ships node:sqlite, which keeps Memorix
+ * usable when an optional better-sqlite3 native binary is unavailable.
  */
 
 import { createRequire } from 'node:module';
 
 let Database: any;
+let driver: 'better-sqlite3' | 'node:sqlite' | 'bun:sqlite' | undefined;
 const requireFromHere = createRequire(import.meta.url);
 
+function loadNodeSqlite(): any {
+  const nodeSqlite = requireFromHere('node:sqlite');
+  return nodeSqlite.DatabaseSync;
+}
+
+function isNativeBindingFailure(error: unknown): boolean {
+  const message = error instanceof Error ? error.message : String(error);
+  return /could not locate the bindings file|better_sqlite3\.node|module did not self-register/i.test(message);
+}
+
+function addCompatibility(db: any): any {
+  // Bun and node:sqlite do not expose better-sqlite3's pragma helper.
+  if (!db.pragma) {
+    db.pragma = function (pragma: string, options?: { simple?: boolean }) {
+      const statement = db.prepare(`PRAGMA ${pragma}`);
+      if (options?.simple) {
+        const result = statement.get();
+        return result ? Object.values(result)[0] : undefined;
+      }
+      return statement.all();
+    };
+  }
+
+  // Stores use better-sqlite3's synchronous transaction factory. Keep nested
+  // calls correct with savepoints instead of silently committing inner work.
+  if (!db.transaction) {
+    let depth = 0;
+    let sequence = 0;
+    db.transaction = function <T>(fn: (...args: any[]) => T) {
+      return (...args: any[]): T => {
+        const outermost = depth === 0;
+        const savepoint = `memorix_tx_${++sequence}`;
+        if (outermost) {
+          db.exec('BEGIN IMMEDIATE');
+        } else {
+          db.exec(`SAVEPOINT ${savepoint}`);
+        }
+        depth += 1;
+        try {
+          const result = fn(...args);
+          if (result && typeof (result as any).then === 'function') {
+            throw new Error('[memorix] SQLite transactions must be synchronous');
+          }
+          depth -= 1;
+          if (outermost) {
+            db.exec('COMMIT');
+          } else {
+            db.exec(`RELEASE SAVEPOINT ${savepoint}`);
+          }
+          return result;
+        } catch (error) {
+          depth -= 1;
+          try {
+            if (outermost) {
+              db.exec('ROLLBACK');
+            } else {
+              db.exec(`ROLLBACK TO SAVEPOINT ${savepoint}`);
+              db.exec(`RELEASE SAVEPOINT ${savepoint}`);
+            }
+          } catch {
+            // Preserve the original application failure when rollback itself fails.
+          }
+          throw error;
+        }
+      };
+    };
+  }
+
+  return db;
+}
+
+function instantiateDatabase(Sqlite: any, filePath: string, options?: any): any {
+  // DatabaseSync validates that an explicitly supplied second argument is an
+  // object, unlike better-sqlite3 which accepts undefined.
+  return driver === 'node:sqlite' && options === undefined
+    ? new Sqlite(filePath)
+    : new Sqlite(filePath, options);
+}
+
 export function loadSqlite(): any {
   if (Database) return Database;
 
+  // Kept as an operational escape hatch and a regression-test seam. Node 22
+  // is the supported minimum runtime, so this does not expand the platform
+  // contract beyond package.json's existing engine requirement.
+  if (process.env.MEMORIX_SQLITE_DRIVER === 'node') {
+    Database = loadNodeSqlite();
+    driver = 'node:sqlite';
+    return Database;
+  }
+
   // Try better-sqlite3 first (Node.js)
   try {
     Database = requireFromHere('better-sqlite3');
+    driver = 'better-sqlite3';
+    return Database;
+  } catch {
+    // Fall through to node:sqlite, then Bun's runtime implementation.
+  }
+
+  try {
+    Database = loadNodeSqlite();
+    driver = 'node:sqlite';
     return Database;
   } catch {
     // Fall through to bun:sqlite
@@ -26,9 +125,10 @@ export function loadSqlite(): any {
     // bun:sqlite is a Bun built-in
     const bunSqlite = requireFromHere('bun:sqlite');
     Database = bunSqlite.Database;
+    driver = 'bun:sqlite';
     return Database;
   } catch {
-    throw new Error('[memorix] Neither better-sqlite3 nor bun:sqlite is available');
+    throw new Error('[memorix] SQLite is unavailable (better-sqlite3, node:sqlite, and bun:sqlite failed)');
   }
 }
 
@@ -38,18 +138,21 @@ export function loadSqlite(): any {
  */
 export function createDatabase(path: string, options?: any): any {
   const Sqlite = loadSqlite();
-  const db = new Sqlite(path, options);
-
-  // Bun's sqlite doesn't have .pragma() method, add compatibility
-  if (!db.pragma) {
-    db.pragma = function (pragma: string, options?: any) {
-      if (options && options.simple) {
-        const result = db.prepare(`PRAGMA ${pragma}`).get();
-        return result ? Object.values(result)[0] : undefined;
-      }
-      return db.prepare(`PRAGMA ${pragma}`).all();
-    };
+  try {
+    return addCompatibility(instantiateDatabase(Sqlite, path, options));
+  } catch (error) {
+    // require('better-sqlite3') can succeed even when npm had no prebuild and
+    // no local compiler. Fall back to Node's built-in SQLite for that one
+    // recoverable case; real open/corruption errors still surface unchanged.
+    if (driver !== 'better-sqlite3' || !isNativeBindingFailure(error)) throw error;
+    Database = loadNodeSqlite();
+    driver = 'node:sqlite';
+    return addCompatibility(instantiateDatabase(Database, path, options));
   }
+}
 
-  return db;
+/** Test-only reset for the driver cache used by the forced node:sqlite path. */
+export function resetSqliteDriverForTests(): void {
+  Database = undefined;
+  driver = undefined;
 }

package/src/store/orama-store.ts

@@ -17,6 +17,7 @@ import { getEmbeddingProvider, type EmbeddingProvider } from '../embedding/provi
 import { calculateProjectAffinity, extractProjectKeywords, type AffinityContext, type MemoryContent } from './project-affinity.js';
 import { detectQueryIntent, applyIntentBoost } from '../search/intent-detector.js';
 import { maybeExpandSearchQuery } from '../search/query-expansion.js';
+import { withTimeout } from '../timeout.js';
 
 let db: AnyOrama | null = null;
 let dbInitPromise: Promise<AnyOrama> | null = null;
@@ -501,8 +502,12 @@ export async function searchObservations(options: SearchOptions): Promise<IndexE
   const t0 = perf ? performance.now() : 0;
   const mark = (label: string) => { if (perf) { const now = performance.now(); process.stderr.write(`  [search-perf] ${label}: ${(now - t0).toFixed(0)}ms\n`); } };
   const modeKey = options.projectId ?? SEARCH_MODE_DEFAULT_KEY;
-  lastSearchModeByProject.set(modeKey, embeddingEnabled ? 'hybrid' : 'fulltext');
+  const quality = options.quality ?? 'balanced';
   const database = await getDb();
+  lastSearchModeByProject.set(
+    modeKey,
+    quality === 'fast' ? 'fulltext (fast profile)' : embeddingEnabled ? 'hybrid' : 'fulltext',
+  );
 
   // Resolve project aliases — safety net for observations not yet migrated to canonical ID.
   // After migration, this is typically a single-element array matching options.projectId.
@@ -531,17 +536,28 @@ export async function searchObservations(options: SearchOptions): Promise<IndexE
   // Determine search mode: hybrid (with vector) or fulltext (default)
   const hasQuery = options.query && options.query.trim().length > 0;
   const originalQuery = options.query;
-  const tier = hasQuery ? classifyQueryTier(originalQuery!) : 'fast' as QueryTier;
+  // The fast profile is a bounded local probe, not a lower-quality spelling
+  // corrector. Skip query classification and the expensive recall enrichments
+  // it would otherwise inherit from the balanced path.
+  const tier = quality === 'fast' ? 'fast' as QueryTier : hasQuery ? classifyQueryTier(originalQuery!) : 'fast' as QueryTier;
   mark(`tier=${tier}`);
 
-  // Query expansion: only for heavy-tier (CJK) queries
-  const expandedEmbeddingQuery = tier === 'heavy' ? await maybeExpandSearchQuery(options.query!) : options.query;
+  // LLM quality work is explicit. It never runs on the default search path.
+  if (quality === 'thorough' && hasQuery) {
+    const { initLLM, isLLMEnabled } = await import('../llm/provider.js');
+    if (!isLLMEnabled()) initLLM();
+  }
+
+  // Query expansion: only for thorough heavy-tier (CJK) searches.
+  const expandedEmbeddingQuery = quality === 'thorough' && tier === 'heavy'
+    ? await maybeExpandSearchQuery(options.query!)
+    : options.query;
   mark('queryExpansion');
 
   // ── Intent-Aware Recall ──────────────────────────────────────
   // Detect query intent (why/when/how/what/problem) and adjust
   // field weights and type boosting accordingly.
-  const intentResult = hasQuery ? detectQueryIntent(originalQuery!) : null;
+  const intentResult = quality === 'fast' || !hasQuery ? null : detectQueryIntent(originalQuery!);
 
   // Orama's vector/hybrid search can leak cross-project hits even when `where`
   // is present, so always keep enough headroom for a deterministic post-filter.
@@ -565,20 +581,24 @@ export async function searchObservations(options: SearchOptions): Promise<IndexE
   let searchParams: Record<string, unknown> = {
     term: originalQuery,
     limit: requestLimit,
-    includeVectors: true,
+    // Fast search never uses vectors, so returning them only adds work and
+    // payload pressure to an explicitly latency-sensitive path.
+    includeVectors: quality !== 'fast',
     ...(Object.keys(filters).length > 0 ? { where: filters } : {}),
     // Search specific fields (not tokens, accessCount, etc.)
     properties: ['title', 'entityName', 'narrative', 'facts', 'concepts', 'filesModified'],
     // Field boosting: intent-aware or default
     boost: fieldBoost,
     // Fuzzy tolerance: allow 1-char typos for short queries, 2 for longer
-    ...(hasQuery ? { tolerance: originalQuery!.length > 6 ? 2 : 1 } : {}),
+    // Fuzzy matching is useful for ordinary recall, but grows sharply with
+    // corpus size. The fast profile deliberately uses exact lexical matching.
+    ...(hasQuery && quality !== 'fast' ? { tolerance: originalQuery!.length > 6 ? 2 : 1 } : {}),
   };
 
   // If embedding provider is available and query tier warrants it, use hybrid search
   // Fast-tier queries skip embedding entirely (fulltext is sufficient)
   let queryVector: number[] | null = null;
-  if (embeddingEnabled && hasQuery && tier !== 'fast') {
+  if (quality !== 'fast' && embeddingEnabled && hasQuery && tier !== 'fast') {
     try {
       const provider = await getEmbeddingProvider();
       if (provider) {
@@ -594,11 +614,11 @@ export async function searchObservations(options: SearchOptions): Promise<IndexE
         } else {
           // Embedding timeout: 15 seconds
           const EMBEDDING_TIMEOUT_MS = 15000;
-          const embedPromise = provider.embed(expandedEmbeddingQuery!);
-          const timeoutPromise = new Promise<never>((_, reject) =>
-            setTimeout(() => reject(new Error(`Embedding timeout after ${EMBEDDING_TIMEOUT_MS}ms`)), EMBEDDING_TIMEOUT_MS)
+          queryVector = await withTimeout(
+            provider.embed(expandedEmbeddingQuery!),
+            EMBEDDING_TIMEOUT_MS,
+            'Embedding',
           );
-          queryVector = await Promise.race([embedPromise, timeoutPromise]);
           mark('embedding');
           // Detect CJK-heavy queries: BM25 can't tokenize Chinese/Japanese/Korean well
           const cjkRatio = (originalQuery!.match(/[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]/g) || []).length / originalQuery!.length;
@@ -946,9 +966,10 @@ export async function searchObservations(options: SearchOptions): Promise<IndexE
 
   // ── LLM Reranking (heavy-tier only) ────────────────────────────
   // Only triggered for heavy-tier queries with ambiguous top results.
-  // Fast and standard tiers skip entirely.
+  // Non-thorough profiles, fast, and standard tiers skip entirely.
   // Ambiguity check: top-2 scores within 30% → results are uncertain.
-  const shouldRerank = tier === 'heavy'
+  const shouldRerank = quality === 'thorough'
+    && tier === 'heavy'
     && hasQuery
     && intermediate.length > 2
     && (() => {
@@ -979,11 +1000,11 @@ export async function searchObservations(options: SearchOptions): Promise<IndexE
       // LLM rerank timeout: configurable via MEMORIX_RERANK_TIMEOUT_MS, default 5s
       const _parsedRerank = parseInt(process.env.MEMORIX_RERANK_TIMEOUT_MS || '', 10);
       const RERANK_TIMEOUT_MS = Number.isFinite(_parsedRerank) && _parsedRerank > 0 ? _parsedRerank : 5000;
-      const rerankPromise = rerankResults(originalQuery!, candidates);
-      const timeoutPromise = new Promise<never>((_, reject) =>
-        setTimeout(() => reject(new Error(`LLM rerank timeout after ${RERANK_TIMEOUT_MS}ms`)), RERANK_TIMEOUT_MS)
+      const { reranked, usedLLM } = await withTimeout(
+        rerankResults(originalQuery!, candidates),
+        RERANK_TIMEOUT_MS,
+        'LLM rerank',
       );
-      const { reranked, usedLLM } = await Promise.race([rerankPromise, timeoutPromise]);
       mark(`rerank(usedLLM=${usedLLM})`);
       
       if (usedLLM) {

package/src/store/sqlite-db.ts

@@ -17,7 +17,7 @@ import path from 'node:path';
 import fs from 'node:fs';
 import { createDatabase, loadSqlite } from './bun-sqlite-compat.js';
 
-// Dynamic require for SQLite (better-sqlite3 or bun:sqlite)
+// Dynamic require for SQLite (better-sqlite3, node:sqlite, or bun:sqlite)
 let BetterSqlite3: any;
 
 export function loadBetterSqlite3(): any {
@@ -26,7 +26,7 @@ export function loadBetterSqlite3(): any {
     BetterSqlite3 = loadSqlite();
     return BetterSqlite3;
   } catch {
-    throw new Error('[memorix] SQLite is not available (neither better-sqlite3 nor bun:sqlite)');
+    throw new Error('[memorix] SQLite is not available (better-sqlite3, node:sqlite, and bun:sqlite all failed)');
   }
 }
 
@@ -688,7 +688,7 @@ function applySchemaMigrations(db: any): void {
 const _dbCache = new Map<string, any>();
 
 /**
- * Get or create a shared better-sqlite3 database handle for the given data directory.
+ * Get or create a shared SQLite database handle for the given data directory.
  *
  * The handle is cached per normalized dataDir path. All stores (observations,
  * mini-skills, sessions) share the same connection and the same DB file.

package/src/timeout.ts

@@ -0,0 +1,23 @@
+/**
+ * Run an operation with a bounded wait without leaving its watchdog alive.
+ *
+ * The underlying operation is not force-cancelled because many existing
+ * callers do not expose an AbortSignal. Its settlement is still observed so a
+ * late rejection is handled, while the caller receives the timeout promptly.
+ */
+export function withTimeout<T>(promise: Promise<T>, ms: number, label: string): Promise<T> {
+  return new Promise<T>((resolve, reject) => {
+    const timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms);
+
+    promise.then(
+      (value) => {
+        clearTimeout(timer);
+        resolve(value);
+      },
+      (error) => {
+        clearTimeout(timer);
+        reject(error);
+      },
+    );
+  });
+}

package/src/types.ts

@@ -239,6 +239,9 @@ export interface TimelineContext {
   after: IndexEntry[];
 }
 
+/** Retrieval work allowed for a search request. */
+export type RetrievalQuality = 'fast' | 'balanced' | 'thorough';
+
 /** Search options for the compact engine */
 export interface SearchOptions {
   query: string;
@@ -257,6 +260,11 @@ export interface SearchOptions {
   trackAccess?: boolean;
   /** Internal reader context for agent-facing visibility filtering. */
   reader?: ObservationReader;
+  /**
+   * `fast` stays fully local, `balanced` (default) permits embeddings, and
+   * `thorough` explicitly permits optional LLM query rewrite and reranking.
+   */
+  quality?: RetrievalQuality;
 }
 
 /** Topic key family heuristics for suggesting stable topic keys */