@equationalapplications/core-llm-wiki 3.1.0 → 4.0.0

package/README.md

@@ -302,6 +302,39 @@ wikiMemory.clearVectorCache();
 
 The cache is also automatically invalidated on any mutation (`runLibrarian`, `runHeal`, `runPrune`, `runReembed`, `ingestDocument`, `importDump`, `forget`).
 
+## Security
+
+`@equationalapplications/core-llm-wiki` enforces multiple security layers:
+
+### VectorRanker Adapter Security
+
+If implementing a custom `VectorRanker`:
+
+- **SQL Injection**: ALWAYS use parameterized queries for `entityId`, `factId`, `candidateIds`. Never concatenate into SQL strings.
+- **Entity Isolation**: Filter by `entityId` in all queries to prevent cross-tenant data leaks.
+- **Credential Scrubbing**: Strip API keys, tokens, connection strings from thrown errors before surfacing to host.
+- **Resource Limits**: Cap `limit` and `candidateIds.length` to prevent DoS. Do NOT retain `vector` references beyond callback scope — blocks GC.
+
+See [SECURITY.md](../../SECURITY.md) for complete adapter security guidance and code examples.
+
+### Host Application Security
+
+When using `VectorRanker`:
+
+- **Error Sanitization**: `sanitizeRankerErrors: true` (default) scrubs ranker errors before mirroring via `error.cause`.
+- **Fallback Policy**: Choose `vectorRankerFallback` based on availability vs consistency requirements:
+  - `'js-cosine'` (default): Best availability
+  - `'keyword'`: Fast fallback without semantic ranking
+  - `'empty'`: Strict consistency (no facts on failure)
+  - `'throw'`: Fail-fast error propagation
+- **Deletion Hook Contract**: `forget()` / `runPrune()` reject on hook timeout/failure. Prevents GDPR violations (deleted vectors still retrievable). Handle failures with retry or queue for reconciliation.
+- **Timeout Tuning**: Set `deletionHookTimeoutMs` per deployment (default 30s). Interactive UX: 5s. Background jobs: 60s.
+
+Core WikiMemory provides:
+- **Defensive Copies**: Query/embedding vectors copied before ranker/hook calls
+- **Input Validation**: `sourceRef`/`sourceHash` normalized; embedding dimensions validated
+- **Parameterized Queries**: All SQL uses bind parameters
+
 ## Usage
 
 ```typescript

package/dist/index.d.mts

@@ -60,7 +60,15 @@ interface WikiFact {
     body: string;
     tags: string[];
     confidence: 'certain' | 'inferred' | 'tentative';
-    source_type: 'user_stated' | 'agent_inferred' | 'user_confirmed' | 'user_document';
+    /**
+     * Source type of this fact.
+     * - 'immutable_document': From ingestDocument(), cannot be modified by system (librarian/heal).
+     *   Only removable via forget() or replaced via re-ingest.
+     * - 'librarian_inferred': Created by runLibrarian() from events, or by runHeal() when synthesizing new inferred facts.
+     * - 'user_stated': Direct user statement.
+     * - 'user_confirmed': User-confirmed fact.
+     */
+    source_type: 'user_stated' | 'librarian_inferred' | 'user_confirmed' | 'immutable_document';
     source_hash: string | null;
     source_ref: string | null;
     created_at: number;
@@ -145,6 +153,11 @@ interface VectorRankerSemanticResult {
  */
 interface VectorRankerRankArgs {
     entityId: string;
+    /**
+     * Query embedding. Treat as readonly — core provides a defensive copy,
+     * but adapters MUST NOT mutate this array. Mutation can corrupt
+     * WikiMemory's internal vector cache and JS-cosine fallback path.
+     */
     queryVec: Float32Array | number[];
     /**
      * When set (MiniSearch pre-filter path): ranker MUST only produce results for ids in this set.
@@ -172,6 +185,13 @@ interface VectorRanker {
     /**
      * Called after a fact's embedding is successfully persisted to embedding_blob (or cleared).
      * Hosts use this to keep sqlite-vec / external indexes consistent with SQLite as source of truth.
+     *
+     * On deletion paths (forget, prune, hard-delete), core awaits this hook to ensure ANN cleanup
+     * completes before the deletion call resolves (GDPR compliance). Hook failures or timeouts on
+     * those paths reject the deletion call.
+     *
+     * Treat `vector` as readonly — core provides a defensive copy, but adapters MUST NOT mutate.
+     *
      * Optional: if omitted, hosts MUST document "index rebuilt separately" and accept stale ANN until rebuild.
      */
     onEmbeddingPersisted?(event: {
@@ -226,6 +246,29 @@ interface WikiOptions {
      * Ignored when vectorRankerFallback is 'throw'. Default false.
      */
     propagateRankerFailureToRetrievalFallback?: boolean;
+    /**
+     * When true (default), sanitize ranker errors before exposing via error.cause
+     * to prevent credential leakage in host telemetry. Disable only when you
+     * control the ranker implementation.
+     *
+     * Sanitization replaces error message/stack with a generic message preserving
+     * only the error type (constructor name).
+     */
+    sanitizeRankerErrors?: boolean;
+    /**
+     * Timeout (ms) for onEmbeddingPersisted hook on GDPR deletion paths
+     * (forget, _doPrune). Hook must complete within this window or the
+     * deletion operation rejects. Default 30000.
+     * Lower for interactive deletes; raise for slow remote ANN backends.
+     */
+    deletionHookTimeoutMs?: number;
+    /**
+     * Escape hatch: skip onEmbeddingPersisted on deletion paths entirely.
+     * Use ONLY when the ANN backend is permanently decommissioned. Vectors
+     * orphaned in the (unreachable) external index are accepted as a tradeoff.
+     * NOT GDPR-safe for live indexes. Default false.
+     */
+    forceDeleteIgnoreRankerHook?: boolean;
 }
 interface MemoryBundle {
     facts: WikiFact[];
@@ -278,6 +321,15 @@ declare class WikiBusyError extends Error {
     readonly entityId: string;
     constructor(operation: WikiBusyOperation, entityId: string);
 }
+declare class PrunePartialFailureError extends Error {
+    readonly deleted: number;
+    readonly failedAt: string;
+    readonly remaining: number;
+    readonly deletedTasks: number;
+    readonly deletedEvents: number;
+    readonly cause: Error;
+    constructor(deleted: number, failedAt: string, remaining: number, cause: Error, deletedTasks?: number, deletedEvents?: number);
+}
 
 declare class WikiMemory {
     private db;
@@ -314,7 +366,16 @@ declare class WikiMemory {
     private _librarianKey;
     private _healKey;
     private _warnCrossEntityCollision;
+    /** Maps pre-rename enum strings from older dumps to current source_type values. */
+    private _normalizeImportedSourceType;
+    private assertNoLegacySourceTypes;
     private _notifyEmbeddingPersisted;
+    /**
+     * GDPR-critical variant: awaits the hook with a timeout and rethrows failures.
+     * Use ONLY on deletion paths. forget() calls after soft-delete UPDATE; runPrune()
+     * calls before hard DELETE. For best-effort sync, use _notifyEmbeddingPersisted.
+     */
+    private _notifyEmbeddingPersistedOrThrow;
     constructor(db: SQLiteAdapter, options: WikiOptions);
     setup(): Promise<void>;
     hasChanged(entityId: string, sourceRef: string, sourceHash: string): Promise<boolean>;
@@ -351,6 +412,12 @@ declare class WikiMemory {
      * Negative return means "a ranks ahead of b" for descending score order.
      */
     private _compareScoredRows;
+    /**
+     * Strip potentially sensitive data from ranker errors before exposing to host callbacks.
+     * Preserves error type for debugging but removes message/stack that may contain credentials.
+     * Recursively sanitizes one level of .cause; deeper chains collapse to type only.
+     */
+    private _sanitizeRankerError;
     /**
      * Score candidate rows using in-process JS cosine similarity.
      * Applies hybrid blending (if weight set) and tie-break sorting before returning.
@@ -414,6 +481,8 @@ declare function formatContext(bundle: MemoryBundle, options?: FormatContextOpti
 
 declare function formatMemoryDump(dump: MemoryDump): FormattedMemoryDump;
 
+declare function parseEmbedding(blob: Uint8Array | null | undefined, text: string | null | undefined): Float32Array | null;
+
 declare function createWiki(db: SQLiteAdapter, options: WikiOptions): WikiMemory;
 
-export { type EntityStatus, type ExtractedFact, type ExtractedTask, type FormatContextOptions, type FormattedMemoryDump, type LLMProvider, type MemoryBundle, type MemoryDump, type ReadOptions, type SQLiteAdapter, type VectorRanker, type VectorRankerFallback, type VectorRankerRankArgs, type VectorRankerSemanticResult, WikiBusyError, type WikiBusyOperation, type WikiCheckpoint, type WikiConfig, type WikiEvent, type WikiFact, WikiMemory, type WikiOptions, type WikiTask, createWiki, formatContext, formatMemoryDump };
+export { type EntityStatus, type ExtractedFact, type ExtractedTask, type FormatContextOptions, type FormattedMemoryDump, type LLMProvider, type MemoryBundle, type MemoryDump, PrunePartialFailureError, type ReadOptions, type SQLiteAdapter, type VectorRanker, type VectorRankerFallback, type VectorRankerRankArgs, type VectorRankerSemanticResult, WikiBusyError, type WikiBusyOperation, type WikiCheckpoint, type WikiConfig, type WikiEvent, type WikiFact, WikiMemory, type WikiOptions, type WikiTask, createWiki, formatContext, formatMemoryDump, parseEmbedding };

package/dist/index.d.ts

@@ -60,7 +60,15 @@ interface WikiFact {
     body: string;
     tags: string[];
     confidence: 'certain' | 'inferred' | 'tentative';
-    source_type: 'user_stated' | 'agent_inferred' | 'user_confirmed' | 'user_document';
+    /**
+     * Source type of this fact.
+     * - 'immutable_document': From ingestDocument(), cannot be modified by system (librarian/heal).
+     *   Only removable via forget() or replaced via re-ingest.
+     * - 'librarian_inferred': Created by runLibrarian() from events, or by runHeal() when synthesizing new inferred facts.
+     * - 'user_stated': Direct user statement.
+     * - 'user_confirmed': User-confirmed fact.
+     */
+    source_type: 'user_stated' | 'librarian_inferred' | 'user_confirmed' | 'immutable_document';
     source_hash: string | null;
     source_ref: string | null;
     created_at: number;
@@ -145,6 +153,11 @@ interface VectorRankerSemanticResult {
  */
 interface VectorRankerRankArgs {
     entityId: string;
+    /**
+     * Query embedding. Treat as readonly — core provides a defensive copy,
+     * but adapters MUST NOT mutate this array. Mutation can corrupt
+     * WikiMemory's internal vector cache and JS-cosine fallback path.
+     */
     queryVec: Float32Array | number[];
     /**
      * When set (MiniSearch pre-filter path): ranker MUST only produce results for ids in this set.
@@ -172,6 +185,13 @@ interface VectorRanker {
     /**
      * Called after a fact's embedding is successfully persisted to embedding_blob (or cleared).
      * Hosts use this to keep sqlite-vec / external indexes consistent with SQLite as source of truth.
+     *
+     * On deletion paths (forget, prune, hard-delete), core awaits this hook to ensure ANN cleanup
+     * completes before the deletion call resolves (GDPR compliance). Hook failures or timeouts on
+     * those paths reject the deletion call.
+     *
+     * Treat `vector` as readonly — core provides a defensive copy, but adapters MUST NOT mutate.
+     *
      * Optional: if omitted, hosts MUST document "index rebuilt separately" and accept stale ANN until rebuild.
      */
     onEmbeddingPersisted?(event: {
@@ -226,6 +246,29 @@ interface WikiOptions {
      * Ignored when vectorRankerFallback is 'throw'. Default false.
      */
     propagateRankerFailureToRetrievalFallback?: boolean;
+    /**
+     * When true (default), sanitize ranker errors before exposing via error.cause
+     * to prevent credential leakage in host telemetry. Disable only when you
+     * control the ranker implementation.
+     *
+     * Sanitization replaces error message/stack with a generic message preserving
+     * only the error type (constructor name).
+     */
+    sanitizeRankerErrors?: boolean;
+    /**
+     * Timeout (ms) for onEmbeddingPersisted hook on GDPR deletion paths
+     * (forget, _doPrune). Hook must complete within this window or the
+     * deletion operation rejects. Default 30000.
+     * Lower for interactive deletes; raise for slow remote ANN backends.
+     */
+    deletionHookTimeoutMs?: number;
+    /**
+     * Escape hatch: skip onEmbeddingPersisted on deletion paths entirely.
+     * Use ONLY when the ANN backend is permanently decommissioned. Vectors
+     * orphaned in the (unreachable) external index are accepted as a tradeoff.
+     * NOT GDPR-safe for live indexes. Default false.
+     */
+    forceDeleteIgnoreRankerHook?: boolean;
 }
 interface MemoryBundle {
     facts: WikiFact[];
@@ -278,6 +321,15 @@ declare class WikiBusyError extends Error {
     readonly entityId: string;
     constructor(operation: WikiBusyOperation, entityId: string);
 }
+declare class PrunePartialFailureError extends Error {
+    readonly deleted: number;
+    readonly failedAt: string;
+    readonly remaining: number;
+    readonly deletedTasks: number;
+    readonly deletedEvents: number;
+    readonly cause: Error;
+    constructor(deleted: number, failedAt: string, remaining: number, cause: Error, deletedTasks?: number, deletedEvents?: number);
+}
 
 declare class WikiMemory {
     private db;
@@ -314,7 +366,16 @@ declare class WikiMemory {
     private _librarianKey;
     private _healKey;
     private _warnCrossEntityCollision;
+    /** Maps pre-rename enum strings from older dumps to current source_type values. */
+    private _normalizeImportedSourceType;
+    private assertNoLegacySourceTypes;
     private _notifyEmbeddingPersisted;
+    /**
+     * GDPR-critical variant: awaits the hook with a timeout and rethrows failures.
+     * Use ONLY on deletion paths. forget() calls after soft-delete UPDATE; runPrune()
+     * calls before hard DELETE. For best-effort sync, use _notifyEmbeddingPersisted.
+     */
+    private _notifyEmbeddingPersistedOrThrow;
     constructor(db: SQLiteAdapter, options: WikiOptions);
     setup(): Promise<void>;
     hasChanged(entityId: string, sourceRef: string, sourceHash: string): Promise<boolean>;
@@ -351,6 +412,12 @@ declare class WikiMemory {
      * Negative return means "a ranks ahead of b" for descending score order.
      */
     private _compareScoredRows;
+    /**
+     * Strip potentially sensitive data from ranker errors before exposing to host callbacks.
+     * Preserves error type for debugging but removes message/stack that may contain credentials.
+     * Recursively sanitizes one level of .cause; deeper chains collapse to type only.
+     */
+    private _sanitizeRankerError;
     /**
      * Score candidate rows using in-process JS cosine similarity.
      * Applies hybrid blending (if weight set) and tie-break sorting before returning.
@@ -414,6 +481,8 @@ declare function formatContext(bundle: MemoryBundle, options?: FormatContextOpti
 
 declare function formatMemoryDump(dump: MemoryDump): FormattedMemoryDump;
 
+declare function parseEmbedding(blob: Uint8Array | null | undefined, text: string | null | undefined): Float32Array | null;
+
 declare function createWiki(db: SQLiteAdapter, options: WikiOptions): WikiMemory;
 
-export { type EntityStatus, type ExtractedFact, type ExtractedTask, type FormatContextOptions, type FormattedMemoryDump, type LLMProvider, type MemoryBundle, type MemoryDump, type ReadOptions, type SQLiteAdapter, type VectorRanker, type VectorRankerFallback, type VectorRankerRankArgs, type VectorRankerSemanticResult, WikiBusyError, type WikiBusyOperation, type WikiCheckpoint, type WikiConfig, type WikiEvent, type WikiFact, WikiMemory, type WikiOptions, type WikiTask, createWiki, formatContext, formatMemoryDump };
+export { type EntityStatus, type ExtractedFact, type ExtractedTask, type FormatContextOptions, type FormattedMemoryDump, type LLMProvider, type MemoryBundle, type MemoryDump, PrunePartialFailureError, type ReadOptions, type SQLiteAdapter, type VectorRanker, type VectorRankerFallback, type VectorRankerRankArgs, type VectorRankerSemanticResult, WikiBusyError, type WikiBusyOperation, type WikiCheckpoint, type WikiConfig, type WikiEvent, type WikiFact, WikiMemory, type WikiOptions, type WikiTask, createWiki, formatContext, formatMemoryDump, parseEmbedding };