@equationalapplications/core-llm-wiki 3.0.0 → 3.1.0

package/README.md

@@ -124,13 +124,171 @@ const memory = await wikiMemory.read('user-123', 'my preferences', {
 ```
 
 **Hybrid scoring blends:**
-- `hybridWeight: 1.0` → pure semantic scoring among the candidates being scored; if `preFilterLimit` is set, semantic scoring is still limited to the top-K MiniSearch matches
+- `hybridWeight: 1.0` → all-semantic blend with semantic scores clamped to non-negative range (no keyword component)
 - `hybridWeight: 0.5` → balanced semantic + keyword (50/50 blend)
 - `hybridWeight: 0.0` → pure keyword ranking, skips `embed()` entirely (no LLM API cost)
 
+True cosine-range pure semantic ranking (including negative cosine values) is used when `hybridWeight` is left `undefined`.
+
 **Pre-filtering optimization:**
 When `preFilterLimit: 50` is set with 1000 facts, cosine similarity is computed only for the top 50 MiniSearch keyword matches, reducing O(N) scoring to O(50).
 
+## Pluggable Vector Retrieval
+
+When your entity corpus grows, in-process cosine similarity scoring becomes a bottleneck. The optional **`VectorRanker`** interface lets you delegate semantic ranking to **sqlite-vec**, **sqlite-vss**, or an external vector database while `WikiMemory` handles embedding validation, hybrid scoring, and tier-2 row hydration.
+
+### `VectorRanker` purpose
+
+`VectorRanker` provides an optional injection point for approximate nearest-neighbor (ANN) ranking:
+
+```typescript
+export interface VectorRanker {
+  /**
+   * Return semantic scores for facts in scope, sorted by similarity.
+   * - `entityId`: restricts results to one entity
+   * - `queryVec`: the embedded query (Float32Array or number[])
+   * - `candidateIds` (optional): when set, rank only within this set (MiniSearch pre-filter mode)
+   * - `limit`: requested top-K count
+   */
+  rankBySimilarity(args: VectorRankerRankArgs): Promise<VectorRankerSemanticResult[]>;
+
+  /**
+   * Optional hook called after embedding persistence (upsert, reembed, delete).
+   * Implementations use this to keep external indexes (sqlite-vec, remote ANN) in sync.
+   */
+  onEmbeddingPersisted?(event: {
+    entityId: string;
+    factId: string;
+    vector: Float32Array | null; // null = embedding removed
+  }): void | Promise<void>;
+}
+```
+
+**When no ranker is configured**, `WikiMemory` uses built-in JS cosine similarity — the same behavior as today. When a ranker is supplied and embeddings preconditions are met (`embed` available, dimensions match, no mismatches), `WikiMemory` delegates scoring to the ranker and blends results with keyword scores.
+
+### Example: sqlite-vec adapter
+
+```typescript
+import { WikiMemory } from '@equationalapplications/core-llm-wiki';
+import type { VectorRanker, VectorRankerRankArgs, VectorRankerSemanticResult } from '@equationalapplications/core-llm-wiki';
+
+// Minimal sqlite-vec adapter (pseudo-code)
+const sqliteVecRanker: VectorRanker = {
+  async rankBySimilarity(args: VectorRankerRankArgs): Promise<VectorRankerSemanticResult[]> {
+    const { entityId, queryVec, candidateIds, limit } = args;
+
+    // Build KNN query using sqlite-vec's distance functions.
+    // sqlite-vec returns cosine distance (0 = identical, 2 = opposite) ascending.
+    // Invert to semanticScore: higher = more similar, matching VectorRanker contract.
+    let sql = `SELECT id, (1.0 - distance) AS semanticScore FROM vec_facts 
+              WHERE entity_id = ? AND deleted_at IS NULL`;
+    const params: any[] = [entityId];
+
+    // Apply pre-filter if provided
+    if (candidateIds) {
+      sql += ` AND id IN (${candidateIds.map(() => '?').join(',')})`;
+      params.push(...candidateIds);
+    }
+
+    // KNN search (example syntax; adjust for your sqlite-vec version)
+    sql += ` ORDER BY vec MATCH vec_neighbor(?) LIMIT ?`;
+    params.push(queryVec, limit);
+
+    const rows = await db.getAllAsync<{ id: string; semanticScore: number }>(sql, params);
+    return rows; // sorted descending by semanticScore (closest distance → highest similarity)
+  },
+
+  async onEmbeddingPersisted(event) {
+    const { entityId, factId, vector } = event;
+    if (vector) {
+      // Upsert into sqlite-vec table
+      await db.runAsync(
+        `INSERT OR REPLACE INTO vec_facts (id, entity_id, vec) VALUES (?, ?, ?)`,
+        [factId, entityId, vector]
+      );
+    } else {
+      // Delete when embedding is removed
+      await db.runAsync(`DELETE FROM vec_facts WHERE id = ?`, [factId]);
+    }
+  },
+};
+
+const wikiMemory = new WikiMemory(db, {
+  llmProvider: { /* ... */ },
+  vectorRanker: sqliteVecRanker,
+});
+
+// read() now uses sqlite-vec for scoring instead of JS cosine
+const memory = await wikiMemory.read('user-123', 'my preferences');
+```
+
+### Fallback policies
+
+When `rankBySimilarity` rejects (e.g., ANN service outage, misconfiguration), `WikiMemory` applies a recovery policy:
+
+```typescript
+export type VectorRankerFallback =
+  | 'js-cosine'  // (default) Score candidates in-process with JS cosine — same as no ranker
+  | 'keyword'    // Skip semantic ranking; return keyword-only results
+  | 'empty'      // Semantic facts list empty for this read; tasks/events still included
+  | 'throw';     // Reject read() with the ranker error
+
+const wikiMemory = new WikiMemory(db, {
+  llmProvider: { /* ... */ },
+  vectorRanker: sqliteVecRanker,
+  vectorRankerFallback: 'js-cosine', // default
+  onVectorRankerFallback: (info) => {
+    console.warn(
+      `Ranker failed (policy: ${info.policy}); error:`,
+      info.error
+    );
+  },
+});
+```
+
+- **`'js-cosine'` (default):** Seamless degradation; same behavior as if no ranker was configured.
+- **`'keyword'`:** Useful when semantic ranking is optional; keyword search proceeds normally.
+- **`'empty'`:** Return no facts for this query (but tasks/events still load); useful for strict consistency.
+- **`'throw'`:** Propagate the error and fail the read.
+
+### `onEmbeddingPersisted` eventual consistency
+
+If `vectorRanker.onEmbeddingPersisted` returns a pending Promise, the hook **may resolve asynchronously**. This supports ANN indexes that rebuild on a schedule (e.g., sqlite-vec triggers on transaction commit) or external services with eventual consistency.
+
+**Best practice:**
+- If your adapter has **synchronous guarantees** (in-process sqlite-vec, same transaction), await the promise.
+- If your adapter is **eventually consistent** (remote ANN, async rebuild), document the lag and document that queries may miss recently-added facts until the index refreshes.
+- The **SQLite blob remains the source of truth**; `WikiMemory` always writes embeddings to `embedding_blob` first before calling the hook.
+
+### Hybrid scoring with ranker
+
+When both `vectorRanker` and `hybridWeight` are configured, `WikiMemory` still applies hybrid blending after the ranker returns scores:
+
+```typescript
+const wikiMemory = new WikiMemory(db, {
+  config: {
+    hybridWeight: 0.7, // 70% semantic, 30% keyword
+  },
+  vectorRanker: sqliteVecRanker,
+});
+
+// ranker returns semanticScore; WikiMemory blends with MiniSearch keyword score
+const memory = await wikiMemory.read('user-123', 'my preferences', {
+  hybridWeight: 0.5, // per-call override to 50/50 blend
+});
+```
+
+Note on semantics:
+- Leave `hybridWeight` undefined for true pure-semantic cosine-range scoring.
+- Set `hybridWeight: 1` for an all-semantic variant that clamps negative semantic scores to 0.
+
+For details on hybrid scoring formulas and trade-offs, see [Retrieval Tuning](#retrieval-tuning) above.
+
+### Spec and issue reference
+
+- **Full spec:** [`docs/superpowers/specs/2026-05-07-pluggable-vector-retrieval.md`](https://github.com/equationalapplications/expo-llm-wiki/blob/main/docs/superpowers/specs/2026-05-07-pluggable-vector-retrieval.md)
+- **GitHub issue:** [#15](https://github.com/equationalapplications/expo-llm-wiki/issues/15)
+
 ## Vector Cache
 
 Parsed embedding vectors from full-scan `read()` calls are cached in memory, keyed by entity ID (max 16 entities, max 500 vectors per entity). This avoids redundant `Float32Array` parsing on repeated queries for the same entity. When the 16-entity limit is reached, the oldest-inserted entity is evicted to make room; if an entity exceeds 500 facts, its vectors are not cached at all for that read.

package/dist/index.d.mts

@@ -132,20 +132,100 @@ interface LLMProvider {
      */
     embed?: (text: string) => Promise<number[]>;
 }
+/**
+ * Result of semantic ranking for a single fact.
+ */
+interface VectorRankerSemanticResult {
+    id: string;
+    /** Cosine similarity in [-1, 1] when exact; implementations MAY document other monotonic scales. */
+    semanticScore: number;
+}
+/**
+ * Arguments passed to VectorRanker.rankBySimilarity.
+ */
+interface VectorRankerRankArgs {
+    entityId: string;
+    queryVec: Float32Array | number[];
+    /**
+     * When set (MiniSearch pre-filter path): ranker MUST only produce results for ids in this set.
+     * When omitted (full-entity semantic path): ranker scopes by entityId per its backing store contract.
+     */
+    candidateIds?: readonly string[];
+    /**
+     * Upper bound on how many distinct fact ids should receive a semanticScore in this call.
+     * WikiMemory derives this from maxResults / candidate cardinality / documented oversampling policy.
+     */
+    limit: number;
+}
+/**
+ * Optional backend for semantic candidate scoring / top-k retrieval.
+ * When omitted, WikiMemory scores rows with embedding_blob / embedding TEXT in JS (cosine).
+ */
+interface VectorRanker {
+    /**
+     * Return semantic scores for facts in scope, sorted descending by semanticScore (stable tie-breaking
+     * not required — WikiMemory reapplies existing tie-breakers after blending).
+     * Implementations SHOULD omit facts with no usable vector; callers treat missing ids like today's
+     * "no embedding" rows (pure semantic: -2; hybrid: keyword-only portion).
+     */
+    rankBySimilarity(args: VectorRankerRankArgs): Promise<VectorRankerSemanticResult[]>;
+    /**
+     * Called after a fact's embedding is successfully persisted to embedding_blob (or cleared).
+     * Hosts use this to keep sqlite-vec / external indexes consistent with SQLite as source of truth.
+     * Optional: if omitted, hosts MUST document "index rebuilt separately" and accept stale ANN until rebuild.
+     */
+    onEmbeddingPersisted?(event: {
+        entityId: string;
+        factId: string;
+        vector: Float32Array | null;
+    }): void | Promise<void>;
+}
+/**
+ * Fallback policy when rankBySimilarity rejects.
+ */
+type VectorRankerFallback = 'js-cosine' | 'keyword' | 'empty' | 'throw';
 interface WikiOptions {
     config?: WikiConfig;
     llmProvider: LLMProvider;
     /**
-     * Called when embedding-based retrieval is unavailable during `read()` and
-     * MiniSearch keyword search is used instead. This can happen when:
-     * - `embed()` throws (e.g. network error, model unavailable)
-     * - `embed()` returns a vector with non-finite values (NaN / Infinity)
+     * Called when embedding-based retrieval is degraded or unavailable during `read()`.
+     * This can happen when:
+     * - `embed()` throws (e.g. network error, model unavailable) → falls back to keyword search
+     * - `embed()` returns a vector with non-finite values (NaN / Infinity) → falls back to keyword search
      * - The query vector's dimension doesn't match stored embeddings (model switch;
-     *   resolve by calling `runReembed()`)
+     *   resolve by calling `runReembed()`) → falls back to keyword search
+     * - `vectorRanker` returns IDs that don't belong to the requested entity or don't exist
+     *   (ranker integrity issue; returned rows will be filtered out, reducing result count) →
+     *   may still use semantic ranking, but with degraded quality
      *
-     * `read()` still returns keyword-search results — this is a notification, not an error path.
+     * `read()` returns results (keyword fallback or degraded semantic) — this is a notification, not an error path.
      */
     onRetrievalFallback?: (error: Error) => void;
+    /**
+     * Optional backend for semantic candidate scoring / top-k retrieval.
+     * When omitted, WikiMemory scores rows with embedding_blob / embedding TEXT in JS (cosine).
+     */
+    vectorRanker?: VectorRanker;
+    /**
+     * When rankBySimilarity throws. Default `'js-cosine'`.
+     * Ignored when vectorRanker is undefined.
+     */
+    vectorRankerFallback?: VectorRankerFallback;
+    /**
+     * Called only when rankBySimilarity rejects (after embeddings path succeeded).
+     * Invoked before applying vectorRankerFallback when that policy recovers or before rejecting when policy is 'throw'.
+     */
+    onVectorRankerFallback?: (info: {
+        error: Error;
+        /** Effective policy core will apply for this read (same as WikiOptions.vectorRankerFallback, default js-cosine). */
+        policy: VectorRankerFallback;
+    }) => void;
+    /**
+     * When true: after rankBySimilarity failure, once the recoverable fallback has finished
+     * and read() will resolve, invoke onRetrievalFallback — after onVectorRankerFallback if set.
+     * Ignored when vectorRankerFallback is 'throw'. Default false.
+     */
+    propagateRankerFailureToRetrievalFallback?: boolean;
 }
 interface MemoryBundle {
     facts: WikiFact[];
@@ -234,6 +314,7 @@ declare class WikiMemory {
     private _librarianKey;
     private _healKey;
     private _warnCrossEntityCollision;
+    private _notifyEmbeddingPersisted;
     constructor(db: SQLiteAdapter, options: WikiOptions);
     setup(): Promise<void>;
     hasChanged(entityId: string, sourceRef: string, sourceHash: string): Promise<boolean>;
@@ -261,6 +342,26 @@ declare class WikiMemory {
         events: number;
     }>;
     read(entityId: string, query: string, options?: ReadOptions): Promise<MemoryBundle>;
+    /**
+     * Stable tie-break sort: score desc → access_count desc → updated_at desc → id asc.
+     */
+    private _tieBreakSort;
+    /**
+     * Comparator for score + deterministic tie-break fields.
+     * Negative return means "a ranks ahead of b" for descending score order.
+     */
+    private _compareScoredRows;
+    /**
+     * Score candidate rows using in-process JS cosine similarity.
+     * Applies hybrid blending (if weight set) and tie-break sorting before returning.
+     */
+    private _rankWithJsCosine;
+    /**
+     * Delegate semantic ranking to the injected VectorRanker.
+     * Caller should pass an oversampledLimit to preserve recall after re-ranking.
+     * Returns scored results ready for hybrid blending and tie-break sorting.
+     */
+    private _rankWithVectorRanker;
     getMemoryBundle(entityId: string): Promise<MemoryBundle>;
     write(entityId: string, event: Omit<WikiEvent, 'id' | 'entity_id' | 'created_at'>): Promise<void>;
     private runLibrarianThenMaybeHeal;
@@ -315,4 +416,4 @@ declare function formatMemoryDump(dump: MemoryDump): FormattedMemoryDump;
 
 declare function createWiki(db: SQLiteAdapter, options: WikiOptions): WikiMemory;
 
-export { type EntityStatus, type ExtractedFact, type ExtractedTask, type FormatContextOptions, type FormattedMemoryDump, type LLMProvider, type MemoryBundle, type MemoryDump, type ReadOptions, type SQLiteAdapter, WikiBusyError, type WikiBusyOperation, type WikiCheckpoint, type WikiConfig, type WikiEvent, type WikiFact, WikiMemory, type WikiOptions, type WikiTask, createWiki, formatContext, formatMemoryDump };
+export { type EntityStatus, type ExtractedFact, type ExtractedTask, type FormatContextOptions, type FormattedMemoryDump, type LLMProvider, type MemoryBundle, type MemoryDump, type ReadOptions, type SQLiteAdapter, type VectorRanker, type VectorRankerFallback, type VectorRankerRankArgs, type VectorRankerSemanticResult, WikiBusyError, type WikiBusyOperation, type WikiCheckpoint, type WikiConfig, type WikiEvent, type WikiFact, WikiMemory, type WikiOptions, type WikiTask, createWiki, formatContext, formatMemoryDump };

package/dist/index.d.ts

@@ -132,20 +132,100 @@ interface LLMProvider {
      */
     embed?: (text: string) => Promise<number[]>;
 }
+/**
+ * Result of semantic ranking for a single fact.
+ */
+interface VectorRankerSemanticResult {
+    id: string;
+    /** Cosine similarity in [-1, 1] when exact; implementations MAY document other monotonic scales. */
+    semanticScore: number;
+}
+/**
+ * Arguments passed to VectorRanker.rankBySimilarity.
+ */
+interface VectorRankerRankArgs {
+    entityId: string;
+    queryVec: Float32Array | number[];
+    /**
+     * When set (MiniSearch pre-filter path): ranker MUST only produce results for ids in this set.
+     * When omitted (full-entity semantic path): ranker scopes by entityId per its backing store contract.
+     */
+    candidateIds?: readonly string[];
+    /**
+     * Upper bound on how many distinct fact ids should receive a semanticScore in this call.
+     * WikiMemory derives this from maxResults / candidate cardinality / documented oversampling policy.
+     */
+    limit: number;
+}
+/**
+ * Optional backend for semantic candidate scoring / top-k retrieval.
+ * When omitted, WikiMemory scores rows with embedding_blob / embedding TEXT in JS (cosine).
+ */
+interface VectorRanker {
+    /**
+     * Return semantic scores for facts in scope, sorted descending by semanticScore (stable tie-breaking
+     * not required — WikiMemory reapplies existing tie-breakers after blending).
+     * Implementations SHOULD omit facts with no usable vector; callers treat missing ids like today's
+     * "no embedding" rows (pure semantic: -2; hybrid: keyword-only portion).
+     */
+    rankBySimilarity(args: VectorRankerRankArgs): Promise<VectorRankerSemanticResult[]>;
+    /**
+     * Called after a fact's embedding is successfully persisted to embedding_blob (or cleared).
+     * Hosts use this to keep sqlite-vec / external indexes consistent with SQLite as source of truth.
+     * Optional: if omitted, hosts MUST document "index rebuilt separately" and accept stale ANN until rebuild.
+     */
+    onEmbeddingPersisted?(event: {
+        entityId: string;
+        factId: string;
+        vector: Float32Array | null;
+    }): void | Promise<void>;
+}
+/**
+ * Fallback policy when rankBySimilarity rejects.
+ */
+type VectorRankerFallback = 'js-cosine' | 'keyword' | 'empty' | 'throw';
 interface WikiOptions {
     config?: WikiConfig;
     llmProvider: LLMProvider;
     /**
-     * Called when embedding-based retrieval is unavailable during `read()` and
-     * MiniSearch keyword search is used instead. This can happen when:
-     * - `embed()` throws (e.g. network error, model unavailable)
-     * - `embed()` returns a vector with non-finite values (NaN / Infinity)
+     * Called when embedding-based retrieval is degraded or unavailable during `read()`.
+     * This can happen when:
+     * - `embed()` throws (e.g. network error, model unavailable) → falls back to keyword search
+     * - `embed()` returns a vector with non-finite values (NaN / Infinity) → falls back to keyword search
      * - The query vector's dimension doesn't match stored embeddings (model switch;
-     *   resolve by calling `runReembed()`)
+     *   resolve by calling `runReembed()`) → falls back to keyword search
+     * - `vectorRanker` returns IDs that don't belong to the requested entity or don't exist
+     *   (ranker integrity issue; returned rows will be filtered out, reducing result count) →
+     *   may still use semantic ranking, but with degraded quality
      *
-     * `read()` still returns keyword-search results — this is a notification, not an error path.
+     * `read()` returns results (keyword fallback or degraded semantic) — this is a notification, not an error path.
      */
     onRetrievalFallback?: (error: Error) => void;
+    /**
+     * Optional backend for semantic candidate scoring / top-k retrieval.
+     * When omitted, WikiMemory scores rows with embedding_blob / embedding TEXT in JS (cosine).
+     */
+    vectorRanker?: VectorRanker;
+    /**
+     * When rankBySimilarity throws. Default `'js-cosine'`.
+     * Ignored when vectorRanker is undefined.
+     */
+    vectorRankerFallback?: VectorRankerFallback;
+    /**
+     * Called only when rankBySimilarity rejects (after embeddings path succeeded).
+     * Invoked before applying vectorRankerFallback when that policy recovers or before rejecting when policy is 'throw'.
+     */
+    onVectorRankerFallback?: (info: {
+        error: Error;
+        /** Effective policy core will apply for this read (same as WikiOptions.vectorRankerFallback, default js-cosine). */
+        policy: VectorRankerFallback;
+    }) => void;
+    /**
+     * When true: after rankBySimilarity failure, once the recoverable fallback has finished
+     * and read() will resolve, invoke onRetrievalFallback — after onVectorRankerFallback if set.
+     * Ignored when vectorRankerFallback is 'throw'. Default false.
+     */
+    propagateRankerFailureToRetrievalFallback?: boolean;
 }
 interface MemoryBundle {
     facts: WikiFact[];
@@ -234,6 +314,7 @@ declare class WikiMemory {
     private _librarianKey;
     private _healKey;
     private _warnCrossEntityCollision;
+    private _notifyEmbeddingPersisted;
     constructor(db: SQLiteAdapter, options: WikiOptions);
     setup(): Promise<void>;
     hasChanged(entityId: string, sourceRef: string, sourceHash: string): Promise<boolean>;
@@ -261,6 +342,26 @@ declare class WikiMemory {
         events: number;
     }>;
     read(entityId: string, query: string, options?: ReadOptions): Promise<MemoryBundle>;
+    /**
+     * Stable tie-break sort: score desc → access_count desc → updated_at desc → id asc.
+     */
+    private _tieBreakSort;
+    /**
+     * Comparator for score + deterministic tie-break fields.
+     * Negative return means "a ranks ahead of b" for descending score order.
+     */
+    private _compareScoredRows;
+    /**
+     * Score candidate rows using in-process JS cosine similarity.
+     * Applies hybrid blending (if weight set) and tie-break sorting before returning.
+     */
+    private _rankWithJsCosine;
+    /**
+     * Delegate semantic ranking to the injected VectorRanker.
+     * Caller should pass an oversampledLimit to preserve recall after re-ranking.
+     * Returns scored results ready for hybrid blending and tie-break sorting.
+     */
+    private _rankWithVectorRanker;
     getMemoryBundle(entityId: string): Promise<MemoryBundle>;
     write(entityId: string, event: Omit<WikiEvent, 'id' | 'entity_id' | 'created_at'>): Promise<void>;
     private runLibrarianThenMaybeHeal;
@@ -315,4 +416,4 @@ declare function formatMemoryDump(dump: MemoryDump): FormattedMemoryDump;
 
 declare function createWiki(db: SQLiteAdapter, options: WikiOptions): WikiMemory;
 
-export { type EntityStatus, type ExtractedFact, type ExtractedTask, type FormatContextOptions, type FormattedMemoryDump, type LLMProvider, type MemoryBundle, type MemoryDump, type ReadOptions, type SQLiteAdapter, WikiBusyError, type WikiBusyOperation, type WikiCheckpoint, type WikiConfig, type WikiEvent, type WikiFact, WikiMemory, type WikiOptions, type WikiTask, createWiki, formatContext, formatMemoryDump };
+export { type EntityStatus, type ExtractedFact, type ExtractedTask, type FormatContextOptions, type FormattedMemoryDump, type LLMProvider, type MemoryBundle, type MemoryDump, type ReadOptions, type SQLiteAdapter, type VectorRanker, type VectorRankerFallback, type VectorRankerRankArgs, type VectorRankerSemanticResult, WikiBusyError, type WikiBusyOperation, type WikiCheckpoint, type WikiConfig, type WikiEvent, type WikiFact, WikiMemory, type WikiOptions, type WikiTask, createWiki, formatContext, formatMemoryDump };