@equationalapplications/core-llm-wiki 2.6.0 → 3.0.0

package/README.md

@@ -2,10 +2,15 @@
 
 Pure TypeScript business logic for LLM Wiki Memory.
 
+> Inspired by [Andrej Karpathy's LLM Wiki memory spec](https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f).
+
 ## Features
 
 - **Platform-agnostic** — Zero runtime dependencies; works with any SQLite driver via the `SQLiteAdapter` interface
-- **Full-featured memory** — Facts, tasks, events, semantic search, maintenance jobs
+- **Semantic search** — Vector embeddings via your LLM's `embed` function, ranked by cosine similarity
+- **Keyword fallback** — MiniSearch in-memory index for offline/degraded scenarios when embeddings unavailable
+- **Retrieval tuning** — Per-call overrides for `maxResults`, `preFilterLimit`, and `hybridWeight` blend
+- **Full-featured memory** — Facts, tasks, events, maintenance jobs (librarian, heal, reembed, prune)
 - **Type-safe** — Built with TypeScript, full type exports
 
 ## Installation
@@ -14,6 +19,131 @@ Pure TypeScript business logic for LLM Wiki Memory.
 npm install @equationalapplications/core-llm-wiki
 ```
 
+## Semantic Search with Embeddings
+
+Provide an `embed` function in `llmProvider` to enable vector-based retrieval:
+
+```typescript
+import { WikiMemory } from '@equationalapplications/core-llm-wiki';
+
+const wikiMemory = new WikiMemory(db, {
+  llmProvider: {
+    generateText: async ({ systemPrompt, userPrompt }) => {
+      // Your LLM call for extracting facts, tasks
+      return 'Model output';
+    },
+    embed: async (text: string) => {
+      // Your embedding service (e.g., OpenAI, Cohere, local)
+      const response = await fetch('https://your-app.example.com/api/embed', {
+        method: 'POST',
+        body: JSON.stringify({ text }),
+      });
+      const { embedding } = await response.json();
+      return embedding; // number[]
+    },
+  },
+});
+
+await wikiMemory.setup();
+
+// Query with semantic matching
+const memory = await wikiMemory.read('user-123', 'What should I do this weekend?');
+// Returns facts semantically similar to the query, not lexical matches
+// E.g., fact "Saturday hiking trip" ranks high even though no lexical overlap
+```
+
+When `embed` is unavailable, `read()` silently falls back to MiniSearch keyword search. If an embedding attempt throws, `read()` falls back and calls `onRetrievalFallback` if provided:
+
+```typescript
+const wikiMemory = new WikiMemory(db, {
+  llmProvider: {
+    generateText: async () => { /* ... */ },
+    embed: undefined, // or throws on network error
+  },
+  onRetrievalFallback: (error) => {
+    console.warn('Embedding retrieval unavailable, using keyword search:', error);
+  },
+});
+
+// read() returns MiniSearch results, onRetrievalFallback not called (embed absent is expected)
+// read() returns MiniSearch results, onRetrievalFallback called (embed threw)
+```
+
+## Configuration
+
+All `WikiConfig` fields are optional:
+
+```typescript
+const wikiMemory = new WikiMemory(db, {
+  llmProvider: { /* ... */ },
+  config: {
+    tablePrefix: 'llm_wiki_',          // default: 'llm_wiki_'
+    maxResults: 10,                    // default: 10
+    autoLibrarianThreshold: 20,        // default: 20 — events before librarian auto-runs
+    autoHealThreshold: 100,            // default: 100 — events before heal auto-runs
+    maxChunkLength: 12000,             // default: 12000 (char count per ingestDocument chunk)
+    chunkOverlap: 400,                 // default: 400 (overlap between chunks in characters)
+    chunkConcurrency: 1,               // default: 1 (parallel LLM calls per ingestDocument)
+    pruneRetainSoftDeletedFor: 7,      // default: 7 (days before hard-deleting soft-deleted facts)
+    pruneEventsAfter: 30,              // default: 30 (days before hard-deleting old events)
+    orphanAfterDays: 30,               // default: 30 (days before runHeal flags sourceless facts; null to disable)
+    staleInferredAfterDays: 60,        // default: 60 (days before runHeal downgrades inferred facts; null to disable)
+    preFilterLimit: 50,                // default: undefined — MiniSearch pre-filter before cosine scan; recommended for >500 facts
+    hybridWeight: 0.7,                 // default: undefined — blend semantic (1.0) ↔ keyword (0.0); pure semantic when unset
+  },
+});
+```
+
+## Retrieval Tuning
+
+Optimize `read()` performance and blend retrieval strategies:
+
+```typescript
+const config = {
+  // Limit cosine similarity scoring to top-K MiniSearch keyword candidates
+  preFilterLimit: 50,
+  
+  // Blend semantic and keyword scores (0.0 = pure keyword, 1.0 = pure semantic)
+  hybridWeight: 0.7,
+  
+  // Max results returned per read
+  maxResults: 10,
+};
+
+const wikiMemory = new WikiMemory(db, {
+  config,
+  llmProvider: { /* ... */ },
+});
+
+// Per-call overrides (runtime controls for search dashboards, etc.)
+const memory = await wikiMemory.read('user-123', 'my preferences', {
+  maxResults: 5,
+  preFilterLimit: 20,
+  hybridWeight: 0.5,
+});
+```
+
+**Hybrid scoring blends:**
+- `hybridWeight: 1.0` → pure semantic scoring among the candidates being scored; if `preFilterLimit` is set, semantic scoring is still limited to the top-K MiniSearch matches
+- `hybridWeight: 0.5` → balanced semantic + keyword (50/50 blend)
+- `hybridWeight: 0.0` → pure keyword ranking, skips `embed()` entirely (no LLM API cost)
+
+**Pre-filtering optimization:**
+When `preFilterLimit: 50` is set with 1000 facts, cosine similarity is computed only for the top 50 MiniSearch keyword matches, reducing O(N) scoring to O(50).
+
+## Vector Cache
+
+Parsed embedding vectors from full-scan `read()` calls are cached in memory, keyed by entity ID (max 16 entities, max 500 vectors per entity). This avoids redundant `Float32Array` parsing on repeated queries for the same entity. When the 16-entity limit is reached, the oldest-inserted entity is evicted to make room; if an entity exceeds 500 facts, its vectors are not cached at all for that read.
+
+After heavy read workloads or on memory-constrained runtimes, you can release the entire cache explicitly:
+
+```typescript
+// Release all cached embedding vectors
+wikiMemory.clearVectorCache();
+```
+
+The cache is also automatically invalidated on any mutation (`runLibrarian`, `runHeal`, `runPrune`, `runReembed`, `ingestDocument`, `importDump`, `forget`).
+
 ## Usage
 
 ```typescript
@@ -128,6 +258,46 @@ const adapter: SQLiteAdapter = {
 };
 ```
 
+## How It Works
+
+```mermaid
+flowchart TD
+    A["read(entityId, query)"] --> B{hybridWeight = 0?}
+    B -->|Yes| C["MiniSearch only<br/>(skip embed)"]
+    B -->|No| D{embed available?}
+    D -->|No| C
+    D -->|Yes| F["Embed query"]
+    F -->|throws| E["onRetrievalFallback<br/>callback"]
+    E --> C
+    F -->|succeeds| G{preFilterLimit<br/>active?}
+    G -->|Yes| H["MiniSearch pre-filter<br/>top K candidates"]
+    H --> I["Phase 1: Cosine score<br/>top K candidates"]
+    G -->|No| J["Phase 1: Cosine score<br/>all facts"]
+    J --> K["Cache vectors<br/>in-memory<br/>(full scan only)"]
+    K --> L{hybridWeight = 1?}
+    I --> L
+    L -->|Yes| M["Pure semantic<br/>ranking"]
+    L -->|No| N["Hybrid blend:<br/>semantic + keyword<br/>via MiniSearch"]
+    M --> O["Phase 2: Fetch full rows<br/>top maxResults"]
+    N --> O
+    C --> P["MiniSearch ranking"]
+    P --> O
+    O --> R["Track access"]
+    R --> Q["Return MemoryBundle"]
+```
+
+The flowchart shows:
+1. **Fast-path** when `hybridWeight = 0` (pure keyword, no embed cost)
+2. **Fallback chain** when embed unavailable (MiniSearch silently) or throws (`onRetrievalFallback` callback, then MiniSearch)
+3. **Pre-filtering** to limit cosine scoring to top-K keyword matches (O(N) → O(K))
+4. **Two-phase SELECT**: phase 1 scores all/filtered facts with minimal columns, phase 2 fetches full rows for winners
+5. **Hybrid scoring** to blend semantic and keyword rankings
+6. **Vector caching** on full scans only; reads with `preFilterLimit` active skip cache population
+
 ## License
 
 MIT
+
+---
+
+Made with ❤️ by Equational Applications LLC. [https://equationalapplications.com/](https://equationalapplications.com/)

package/dist/index.d.mts

@@ -28,6 +28,30 @@ interface WikiConfig {
     maxChunkLength?: number;
     chunkOverlap?: number;
     chunkConcurrency?: number;
+    /**
+     * Max MiniSearch candidates passed to cosine scoring.
+     * When set, MiniSearch pre-filters before the cosine scan.
+     * Only applies when embed is provided and succeeds.
+     * Default: undefined (full scan).
+     */
+    preFilterLimit?: number;
+    /**
+     * Hybrid blend weight (0.0–1.0).
+     * 0.0 = pure keyword (skips embed() entirely).
+     * 1.0 = pure semantic.
+     * Values outside [0,1] are clamped. Ignored when embed is absent or throws.
+     * Default: undefined (pure semantic when embed provided).
+     */
+    hybridWeight?: number;
+}
+interface ReadOptions {
+    maxResults?: number;
+    /**
+     * undefined → use WikiConfig.preFilterLimit (or no pre-filter if also unset).
+     * null → explicitly disable a config-level preFilterLimit for this call.
+     */
+    preFilterLimit?: number | null;
+    hybridWeight?: number;
 }
 interface WikiFact {
     id: string;
@@ -41,6 +65,18 @@ interface WikiFact {
     source_ref: string | null;
     created_at: number;
     updated_at: number;
+    /**
+     * Raw Float32Array bytes for the fact's embedding vector.
+     * Set when the fact was fetched via exportDump() with blob preservation.
+     * Accepted in importDump() as a real Uint8Array (in-memory round-trip),
+     * a Node.js Buffer JSON shape `{ type: 'Buffer', data: number[] }`,
+     * or a numeric-keyed plain object `{ 0: byte, 1: byte, ... }` produced
+     * by JSON.stringify(Uint8Array).
+     */
+    embedding_blob?: Uint8Array | {
+        type: 'Buffer';
+        data: number[];
+    } | Record<string, number>;
     last_accessed_at: number | null;
     access_count: number;
     deleted_at: number | null;
@@ -145,10 +181,22 @@ interface EntityStatus {
     librarian: boolean;
     heal: boolean;
 }
+/**
+ * All operations that can appear in a {@link WikiBusyError}.
+ *
+ * @remarks **Breaking change from v2.x** — the union previously only contained
+ * `'ingest' | 'librarian' | 'heal' | 'prune' | 'reembed'`. The values `'import'`
+ * and `'forget'` were added in v3.0. Exhaustive `switch` / narrowing on this type
+ * must be updated (or given a `default` arm) to compile without errors.
+ */
+type WikiBusyOperation = 'ingest' | 'librarian' | 'heal' | 'prune' | 'reembed' | 'import' | 'forget';
+/**
+ * Thrown when a background mutator is already running for the requested entity.
+ */
 declare class WikiBusyError extends Error {
-    readonly operation: 'ingest' | 'librarian' | 'heal' | 'prune' | 'reembed';
+    readonly operation: WikiBusyOperation;
     readonly entityId: string;
-    constructor(operation: 'ingest' | 'librarian' | 'heal' | 'prune' | 'reembed', entityId: string);
+    constructor(operation: WikiBusyOperation, entityId: string);
 }
 
 declare class WikiMemory {
@@ -159,6 +207,19 @@ declare class WikiMemory {
     private activeIngestJobs;
     private miniSearch;
     private miniSearchEntryIdsByEntity;
+    /**
+     * Maximum number of entities whose parsed embedding vectors are held in
+     * memory. This cap is intentionally conservative so the cache remains safe
+     * on memory-constrained runtimes (e.g., mobile/Expo).
+     */
+    private static readonly MAX_VECTOR_CACHE_ENTITIES;
+    /**
+     * Maximum number of fact vectors cached per entity. Keep this high enough to
+     * preserve the parsed-embedding reuse optimization for common mid-sized
+     * entities while still maintaining a bounded memory footprint.
+     */
+    private static readonly MAX_VECTOR_CACHE_FACTS_PER_ENTITY;
+    private vectorCache;
     private normalizeMiniSearchRow;
     private rebuildMiniSearchIndex;
     private storeEmbeddingDimension;
@@ -179,7 +240,12 @@ declare class WikiMemory {
     private _pruneKey;
     private _reembedKey;
     private _globalReembedKey;
+    private _importKey;
+    private _globalImportKey;
+    private _forgetKey;
     private _isReembedActive;
+    private _isImportActiveFor;
+    private _isForgetActiveFor;
     /** Returns true if any maintenance job has the given operation suffix (e.g. ':prune'). */
     private _isAnyMaintenanceActiveWithSuffix;
     /** Returns true if any ingest job is active for the given entity. */
@@ -194,7 +260,7 @@ declare class WikiMemory {
         tasks: number;
         events: number;
     }>;
-    read(entityId: string, query: string): Promise<MemoryBundle>;
+    read(entityId: string, query: string, options?: ReadOptions): Promise<MemoryBundle>;
     getMemoryBundle(entityId: string): Promise<MemoryBundle>;
     write(entityId: string, event: Omit<WikiEvent, 'id' | 'entity_id' | 'created_at'>): Promise<void>;
     private runLibrarianThenMaybeHeal;
@@ -202,16 +268,22 @@ declare class WikiMemory {
     private _doRunHeal;
     runLibrarian(entityId: string): Promise<void>;
     runHeal(entityId: string): Promise<void>;
-    runReembed(entityId?: string): Promise<{
+    runReembed(entityId?: string, opts?: {
+        force?: boolean;
+        skipExisting?: boolean;
+    }): Promise<{
         embedded: number;
         skipped: number;
+        failed: number;
     }>;
     getEntityStatus(entityId: string): EntityStatus;
+    clearVectorCache(): void;
     private _getFullBundle;
     exportDump(entityIds?: string[]): Promise<MemoryDump>;
     importDump(dump: MemoryDump, opts?: {
         merge?: boolean;
     }): Promise<void>;
+    private _doImportEntity;
     forget(entityId: string, params: {
         entryId?: string;
         taskId?: string;
@@ -243,4 +315,4 @@ declare function formatMemoryDump(dump: MemoryDump): FormattedMemoryDump;
 
 declare function createWiki(db: SQLiteAdapter, options: WikiOptions): WikiMemory;
 
-export { type EntityStatus, type ExtractedFact, type ExtractedTask, type FormatContextOptions, type FormattedMemoryDump, type LLMProvider, type MemoryBundle, type MemoryDump, type SQLiteAdapter, WikiBusyError, type WikiCheckpoint, type WikiConfig, type WikiEvent, type WikiFact, WikiMemory, type WikiOptions, type WikiTask, createWiki, formatContext, formatMemoryDump };
+export { type EntityStatus, type ExtractedFact, type ExtractedTask, type FormatContextOptions, type FormattedMemoryDump, type LLMProvider, type MemoryBundle, type MemoryDump, type ReadOptions, type SQLiteAdapter, WikiBusyError, type WikiBusyOperation, type WikiCheckpoint, type WikiConfig, type WikiEvent, type WikiFact, WikiMemory, type WikiOptions, type WikiTask, createWiki, formatContext, formatMemoryDump };

package/dist/index.d.ts

@@ -28,6 +28,30 @@ interface WikiConfig {
     maxChunkLength?: number;
     chunkOverlap?: number;
     chunkConcurrency?: number;
+    /**
+     * Max MiniSearch candidates passed to cosine scoring.
+     * When set, MiniSearch pre-filters before the cosine scan.
+     * Only applies when embed is provided and succeeds.
+     * Default: undefined (full scan).
+     */
+    preFilterLimit?: number;
+    /**
+     * Hybrid blend weight (0.0–1.0).
+     * 0.0 = pure keyword (skips embed() entirely).
+     * 1.0 = pure semantic.
+     * Values outside [0,1] are clamped. Ignored when embed is absent or throws.
+     * Default: undefined (pure semantic when embed provided).
+     */
+    hybridWeight?: number;
+}
+interface ReadOptions {
+    maxResults?: number;
+    /**
+     * undefined → use WikiConfig.preFilterLimit (or no pre-filter if also unset).
+     * null → explicitly disable a config-level preFilterLimit for this call.
+     */
+    preFilterLimit?: number | null;
+    hybridWeight?: number;
 }
 interface WikiFact {
     id: string;
@@ -41,6 +65,18 @@ interface WikiFact {
     source_ref: string | null;
     created_at: number;
     updated_at: number;
+    /**
+     * Raw Float32Array bytes for the fact's embedding vector.
+     * Set when the fact was fetched via exportDump() with blob preservation.
+     * Accepted in importDump() as a real Uint8Array (in-memory round-trip),
+     * a Node.js Buffer JSON shape `{ type: 'Buffer', data: number[] }`,
+     * or a numeric-keyed plain object `{ 0: byte, 1: byte, ... }` produced
+     * by JSON.stringify(Uint8Array).
+     */
+    embedding_blob?: Uint8Array | {
+        type: 'Buffer';
+        data: number[];
+    } | Record<string, number>;
     last_accessed_at: number | null;
     access_count: number;
     deleted_at: number | null;
@@ -145,10 +181,22 @@ interface EntityStatus {
     librarian: boolean;
     heal: boolean;
 }
+/**
+ * All operations that can appear in a {@link WikiBusyError}.
+ *
+ * @remarks **Breaking change from v2.x** — the union previously only contained
+ * `'ingest' | 'librarian' | 'heal' | 'prune' | 'reembed'`. The values `'import'`
+ * and `'forget'` were added in v3.0. Exhaustive `switch` / narrowing on this type
+ * must be updated (or given a `default` arm) to compile without errors.
+ */
+type WikiBusyOperation = 'ingest' | 'librarian' | 'heal' | 'prune' | 'reembed' | 'import' | 'forget';
+/**
+ * Thrown when a background mutator is already running for the requested entity.
+ */
 declare class WikiBusyError extends Error {
-    readonly operation: 'ingest' | 'librarian' | 'heal' | 'prune' | 'reembed';
+    readonly operation: WikiBusyOperation;
     readonly entityId: string;
-    constructor(operation: 'ingest' | 'librarian' | 'heal' | 'prune' | 'reembed', entityId: string);
+    constructor(operation: WikiBusyOperation, entityId: string);
 }
 
 declare class WikiMemory {
@@ -159,6 +207,19 @@ declare class WikiMemory {
     private activeIngestJobs;
     private miniSearch;
     private miniSearchEntryIdsByEntity;
+    /**
+     * Maximum number of entities whose parsed embedding vectors are held in
+     * memory. This cap is intentionally conservative so the cache remains safe
+     * on memory-constrained runtimes (e.g., mobile/Expo).
+     */
+    private static readonly MAX_VECTOR_CACHE_ENTITIES;
+    /**
+     * Maximum number of fact vectors cached per entity. Keep this high enough to
+     * preserve the parsed-embedding reuse optimization for common mid-sized
+     * entities while still maintaining a bounded memory footprint.
+     */
+    private static readonly MAX_VECTOR_CACHE_FACTS_PER_ENTITY;
+    private vectorCache;
     private normalizeMiniSearchRow;
     private rebuildMiniSearchIndex;
     private storeEmbeddingDimension;
@@ -179,7 +240,12 @@ declare class WikiMemory {
     private _pruneKey;
     private _reembedKey;
     private _globalReembedKey;
+    private _importKey;
+    private _globalImportKey;
+    private _forgetKey;
     private _isReembedActive;
+    private _isImportActiveFor;
+    private _isForgetActiveFor;
     /** Returns true if any maintenance job has the given operation suffix (e.g. ':prune'). */
     private _isAnyMaintenanceActiveWithSuffix;
     /** Returns true if any ingest job is active for the given entity. */
@@ -194,7 +260,7 @@ declare class WikiMemory {
         tasks: number;
         events: number;
     }>;
-    read(entityId: string, query: string): Promise<MemoryBundle>;
+    read(entityId: string, query: string, options?: ReadOptions): Promise<MemoryBundle>;
     getMemoryBundle(entityId: string): Promise<MemoryBundle>;
     write(entityId: string, event: Omit<WikiEvent, 'id' | 'entity_id' | 'created_at'>): Promise<void>;
     private runLibrarianThenMaybeHeal;
@@ -202,16 +268,22 @@ declare class WikiMemory {
     private _doRunHeal;
     runLibrarian(entityId: string): Promise<void>;
     runHeal(entityId: string): Promise<void>;
-    runReembed(entityId?: string): Promise<{
+    runReembed(entityId?: string, opts?: {
+        force?: boolean;
+        skipExisting?: boolean;
+    }): Promise<{
         embedded: number;
         skipped: number;
+        failed: number;
     }>;
     getEntityStatus(entityId: string): EntityStatus;
+    clearVectorCache(): void;
     private _getFullBundle;
     exportDump(entityIds?: string[]): Promise<MemoryDump>;
     importDump(dump: MemoryDump, opts?: {
         merge?: boolean;
     }): Promise<void>;
+    private _doImportEntity;
     forget(entityId: string, params: {
         entryId?: string;
         taskId?: string;
@@ -243,4 +315,4 @@ declare function formatMemoryDump(dump: MemoryDump): FormattedMemoryDump;
 
 declare function createWiki(db: SQLiteAdapter, options: WikiOptions): WikiMemory;
 
-export { type EntityStatus, type ExtractedFact, type ExtractedTask, type FormatContextOptions, type FormattedMemoryDump, type LLMProvider, type MemoryBundle, type MemoryDump, type SQLiteAdapter, WikiBusyError, type WikiCheckpoint, type WikiConfig, type WikiEvent, type WikiFact, WikiMemory, type WikiOptions, type WikiTask, createWiki, formatContext, formatMemoryDump };
+export { type EntityStatus, type ExtractedFact, type ExtractedTask, type FormatContextOptions, type FormattedMemoryDump, type LLMProvider, type MemoryBundle, type MemoryDump, type ReadOptions, type SQLiteAdapter, WikiBusyError, type WikiBusyOperation, type WikiCheckpoint, type WikiConfig, type WikiEvent, type WikiFact, WikiMemory, type WikiOptions, type WikiTask, createWiki, formatContext, formatMemoryDump };