@equationalapplications/core-llm-wiki 4.4.0 → 4.5.1

package/README.md

@@ -9,7 +9,9 @@ Pure TypeScript business logic for LLM Wiki Memory.
 - **Platform-agnostic** — Zero runtime dependencies; works with any SQLite driver via the `SQLiteAdapter` interface
 - **Semantic search** — Vector embeddings via your LLM's `embed` function, ranked by cosine similarity
 - **Keyword fallback** — MiniSearch in-memory index for offline/degraded scenarios when embeddings unavailable
-- **Retrieval tuning** — Per-call overrides for `maxResults`, `preFilterLimit`, and `hybridWeight` blend
+- **Retrieval tuning** — Per-call overrides for `maxResults`, `preFilterLimit`, `hybridWeight`, `tierWeights`, and `includeZeroWeightEntities`
+- **Multi-entity reads** — Search across multiple `entity_id` namespaces in one pass with per-entity score multipliers (`tierWeights`); optional `factScores` and `metadata` for explainability
+- **Immutable vs mutable facts** — Use `WikiFact.source_type` to distinguish document-sourced facts (`immutable_document`) from derived or user-provided facts (`librarian_inferred`, `user_stated`, `user_confirmed`). Immutable document facts are not rewritten by `runLibrarian()` or `runHeal()` and can only be removed by `forget()` or re-ingesting.
 - **Full-featured memory** — Facts, tasks, events, maintenance jobs (librarian, heal, reembed, prune)
 - **Type-safe** — Built with TypeScript, full type exports
 
@@ -121,6 +123,19 @@ const memory = await wikiMemory.read('user-123', 'my preferences', {
   preFilterLimit: 20,
   hybridWeight: 0.5,
 });
+
+// Multi-entity with tier weights
+const multiMemory = await wikiMemory.read(['tier_wisdom', 'tier_fact', 'tier_working'], 'my preferences', {
+  maxResults: 8,
+  tierWeights: {
+    tier_wisdom: 2,      // high-confidence curated notes boosted 2×
+    tier_fact: 1,        // neutral baseline
+    tier_working: 0.25,  // recent but unvetted context downranked
+  },
+  // includeZeroWeightEntities: true — include 0-weight entities as bottom-ranked filler
+});
+// multiMemory.factScores — optional Record<factId, weightedScore> for returned facts; may be absent/undefined
+// multiMemory.metadata  — optional { query, entityIds, tierWeights }; may be absent/undefined
 ```
 
 **Hybrid scoring blends:**
@@ -130,6 +145,15 @@ const memory = await wikiMemory.read('user-123', 'my preferences', {
 
 True cosine-range pure semantic ranking (including negative cosine values) is used when `hybridWeight` is left `undefined`.
 
+**Tier weights:**
+- `tierWeights` applies a per-entity multiplier after semantic/keyword scoring: `finalScore = retrievalScore × weight`
+- Missing weights default to `1.0`. Negative weights clamp to `0`. Non-finite weights default to `1.0`.
+- `tierWeights[entity] = 0` skips that entity's scored retrieval branch (no compute cost).
+- `includeZeroWeightEntities: true` includes zero-weight entities as bottom-ranked filler instead of skipping them.
+- `factScores` is present for array-shaped `entityId` calls only when the query is non-empty and at least one fact is scored; empty-query ("recent facts") reads leave it absent even when `entityId` is an array. Plain string calls never expose it. `metadata` is present for all array-shaped calls regardless of query.
+- `maxResults` applies globally across all requested entities.
+- Tasks are capped at `min(20 × entityCount, 200)`; events at `min(10 × entityCount, 100)` for multi-entity reads.
+
 **Pre-filtering optimization:**
 When `preFilterLimit: 50` is set with 1000 facts, cosine similarity is computed only for the top 50 MiniSearch keyword matches, reducing O(N) scoring to O(50).
 
@@ -398,6 +422,71 @@ await wikiMemory.write('user-123', {
 const memory = await wikiMemory.read('user-123', 'coding style preferences');
 ```
 
+### Multi-entity weighted reads
+
+`read()` accepts either one entity id or an array of entity ids. Facts are always merged globally before `maxResults` is applied. For single-entity reads, tasks are uncapped and events are capped at 10. For multi-entity reads, tasks are capped at `min(20 × entity count, 200)` and events at `min(10 × entity count, 100)` — per-entity representation in the returned bundle is not guaranteed.
+
+```ts
+const memory = await wikiMemory.read(['tier_wisdom', 'tier_fact', 'tier_working'], 'Which source should I trust?', {
+  maxResults: 8,
+  tierWeights: {
+    tier_wisdom: 2,
+    tier_fact: 1,
+    tier_working: 0.25,
+  },
+});
+
+console.log(memory.metadata);
+console.log(memory.factScores);
+```
+
+### Librarian prompt override contract
+
+Core exports prompt utilities for weighted retrieval-based synthesis. Use `mapLibrarianOptionsToReadOptions()` to map `entityWeights` to `tierWeights`, then hydrate a prompt with `query`, `context`, and `tasks`.
+
+```ts
+import {
+  DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT,
+  formatContext,
+  hydrateLibrarianPrompt,
+  mapLibrarianOptionsToReadOptions,
+  validateLibrarianPromptTemplate,
+} from '@equationalapplications/core-llm-wiki';
+
+const options = {
+  entityWeights: { tier_wisdom: 2, tier_fact: 1, tier_working: 0.25 },
+  systemPrompt: `You are a strict fact checker.
+Question:
+{{query}}
+
+Retrieved context:
+{{context}}
+
+{{tasks}}`,
+};
+
+const query = 'Which source should I trust for recent project decisions?';
+
+const memory = await wikiMemory.read(['tier_wisdom', 'tier_fact', 'tier_working'], query, {
+  ...mapLibrarianOptionsToReadOptions(options),
+  maxResults: 8,
+});
+
+const template = options.systemPrompt ?? DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT;
+const warnings = validateLibrarianPromptTemplate(template, {
+  custom: options.systemPrompt != null,
+  taskCount: memory.tasks.length,
+});
+
+for (const warning of warnings) console.warn(warning);
+
+const finalPrompt = hydrateLibrarianPrompt(template, {
+  query,
+  context: formatContext(memory, { includeEntityIds: true, includeFactScores: true }),
+  tasks: formatContext({ facts: [], tasks: memory.tasks, events: [] }, { format: 'plain' }),
+});
+```
+
 ## Adapter Interface
 
 Implement `SQLiteAdapter` to use your platform's SQLite driver:
@@ -488,7 +577,7 @@ const adapter: SQLiteAdapter = {
 
 ```mermaid
 flowchart TD
-    A["read(entityId, query)"] --> B{hybridWeight = 0?}
+    A["read(entityId | entityId[], query, options?)"] --> B{hybridWeight = 0?}
     B -->|Yes| C["MiniSearch only<br/>(skip embed)"]
     B -->|No| D{embed available?}
     D -->|No| C

package/dist/index.d.mts

@@ -313,6 +313,8 @@ interface FormatContextOptions {
     maxEvents?: number;
     includeConfidence?: boolean;
     includeTags?: boolean;
+    includeEntityIds?: boolean;
+    includeFactScores?: boolean;
     factWeights?: {
         confidence?: number;
         accessCount?: number;
@@ -531,6 +533,28 @@ declare function formatMemoryDump(dump: MemoryDump): FormattedMemoryDump;
 
 declare function parseEmbedding(blob: Uint8Array | null | undefined, text: string | null | undefined): Float32Array | null;
 
+interface LibrarianOptions {
+    /** If provided, replaces the default Librarian system instructions. */
+    systemPrompt?: string;
+    /** entity_id -> score multiplier, forwarded to WikiMemory.read() as tierWeights. */
+    entityWeights?: Record<string, number>;
+    /** Forwarded to WikiMemory.read() for zero-weight filler context. */
+    includeZeroWeightEntities?: boolean;
+    temperature?: number;
+}
+interface LibrarianPromptVariables {
+    context: string;
+    tasks: string;
+    query: string;
+}
+declare const DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT = "You are a careful memory synthesis assistant.\nUse only the retrieved context when answering the request.\nPreserve source provenance when facts come from different entity namespaces.\n\nRequest:\n{{query}}\n\nRetrieved context:\n{{context}}\n\nOpen tasks:\n{{tasks}}";
+declare function hydrateLibrarianPrompt(template: string, variables: LibrarianPromptVariables): string;
+declare function validateLibrarianPromptTemplate(template: string, options: {
+    custom: boolean;
+    taskCount: number;
+}): string[];
+declare function mapLibrarianOptionsToReadOptions(options: LibrarianOptions): Pick<ReadOptions, 'tierWeights' | 'includeZeroWeightEntities'>;
+
 declare function createWiki(db: SQLiteAdapter, options: WikiOptions): WikiMemory;
 
-export { type EntityStatus, type ExtractedFact, type ExtractedTask, type FormatContextOptions, type FormattedMemoryDump, type LLMProvider, type MemoryBundle, type MemoryDump, PrunePartialFailureError, type ReadOptions, type SQLiteAdapter, type VectorRanker, type VectorRankerFallback, type VectorRankerRankArgs, type VectorRankerSemanticResult, WikiBusyError, type WikiBusyOperation, type WikiCheckpoint, type WikiConfig, type WikiEvent, type WikiFact, WikiMemory, type WikiOptions, type WikiTask, createWiki, formatContext, formatMemoryDump, parseEmbedding };
+export { DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, type EntityStatus, type ExtractedFact, type ExtractedTask, type FormatContextOptions, type FormattedMemoryDump, type LLMProvider, type LibrarianOptions, type LibrarianPromptVariables, type MemoryBundle, type MemoryDump, PrunePartialFailureError, type ReadOptions, type SQLiteAdapter, type VectorRanker, type VectorRankerFallback, type VectorRankerRankArgs, type VectorRankerSemanticResult, WikiBusyError, type WikiBusyOperation, type WikiCheckpoint, type WikiConfig, type WikiEvent, type WikiFact, WikiMemory, type WikiOptions, type WikiTask, createWiki, formatContext, formatMemoryDump, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, validateLibrarianPromptTemplate };

package/dist/index.d.ts

@@ -313,6 +313,8 @@ interface FormatContextOptions {
     maxEvents?: number;
     includeConfidence?: boolean;
     includeTags?: boolean;
+    includeEntityIds?: boolean;
+    includeFactScores?: boolean;
     factWeights?: {
         confidence?: number;
         accessCount?: number;
@@ -531,6 +533,28 @@ declare function formatMemoryDump(dump: MemoryDump): FormattedMemoryDump;
 
 declare function parseEmbedding(blob: Uint8Array | null | undefined, text: string | null | undefined): Float32Array | null;
 
+interface LibrarianOptions {
+    /** If provided, replaces the default Librarian system instructions. */
+    systemPrompt?: string;
+    /** entity_id -> score multiplier, forwarded to WikiMemory.read() as tierWeights. */
+    entityWeights?: Record<string, number>;
+    /** Forwarded to WikiMemory.read() for zero-weight filler context. */
+    includeZeroWeightEntities?: boolean;
+    temperature?: number;
+}
+interface LibrarianPromptVariables {
+    context: string;
+    tasks: string;
+    query: string;
+}
+declare const DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT = "You are a careful memory synthesis assistant.\nUse only the retrieved context when answering the request.\nPreserve source provenance when facts come from different entity namespaces.\n\nRequest:\n{{query}}\n\nRetrieved context:\n{{context}}\n\nOpen tasks:\n{{tasks}}";
+declare function hydrateLibrarianPrompt(template: string, variables: LibrarianPromptVariables): string;
+declare function validateLibrarianPromptTemplate(template: string, options: {
+    custom: boolean;
+    taskCount: number;
+}): string[];
+declare function mapLibrarianOptionsToReadOptions(options: LibrarianOptions): Pick<ReadOptions, 'tierWeights' | 'includeZeroWeightEntities'>;
+
 declare function createWiki(db: SQLiteAdapter, options: WikiOptions): WikiMemory;
 
-export { type EntityStatus, type ExtractedFact, type ExtractedTask, type FormatContextOptions, type FormattedMemoryDump, type LLMProvider, type MemoryBundle, type MemoryDump, PrunePartialFailureError, type ReadOptions, type SQLiteAdapter, type VectorRanker, type VectorRankerFallback, type VectorRankerRankArgs, type VectorRankerSemanticResult, WikiBusyError, type WikiBusyOperation, type WikiCheckpoint, type WikiConfig, type WikiEvent, type WikiFact, WikiMemory, type WikiOptions, type WikiTask, createWiki, formatContext, formatMemoryDump, parseEmbedding };
+export { DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, type EntityStatus, type ExtractedFact, type ExtractedTask, type FormatContextOptions, type FormattedMemoryDump, type LLMProvider, type LibrarianOptions, type LibrarianPromptVariables, type MemoryBundle, type MemoryDump, PrunePartialFailureError, type ReadOptions, type SQLiteAdapter, type VectorRanker, type VectorRankerFallback, type VectorRankerRankArgs, type VectorRankerSemanticResult, WikiBusyError, type WikiBusyOperation, type WikiCheckpoint, type WikiConfig, type WikiEvent, type WikiFact, WikiMemory, type WikiOptions, type WikiTask, createWiki, formatContext, formatMemoryDump, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, validateLibrarianPromptTemplate };

package/dist/index.js

@@ -2793,16 +2793,20 @@ function scoreFactFor(fact, weights, now) {
   const recencyDecay = Math.exp(-ageDays / 30);
   return confW * weights.confidence + Math.log(1 + fact.access_count) * weights.accessCount + recencyDecay * weights.recency;
 }
-function renderFactMarkdown(fact, includeConfidence, includeTags) {
+function renderFactMarkdown(fact, includeConfidence, includeTags, includeEntityIds, score) {
   const confPart = includeConfidence ? ` (${fact.confidence})` : "";
   const tagPart = includeTags && fact.tags.length > 0 ? ` [${fact.tags.join(", ")}]` : "";
-  return `- **${fact.title}**${confPart}${tagPart}
+  const sourcePart = includeEntityIds ? ` {entity_id=${fact.entity_id}}` : "";
+  const scorePart = score !== void 0 ? ` {score=${score.toFixed(4)}}` : "";
+  return `- **${fact.title}**${confPart}${tagPart}${sourcePart}${scorePart}
   ${fact.body.replace(/\n/g, "\n  ")}`;
 }
-function renderFactPlain(fact, includeConfidence, includeTags) {
+function renderFactPlain(fact, includeConfidence, includeTags, includeEntityIds, score) {
   const confPart = includeConfidence ? ` (${fact.confidence})` : "";
   const tagPart = includeTags && fact.tags.length > 0 ? ` [${fact.tags.join(", ")}]` : "";
-  return `${fact.title}${confPart}${tagPart}: ${fact.body}`;
+  const sourcePart = includeEntityIds ? ` {entity_id=${fact.entity_id}}` : "";
+  const scorePart = score !== void 0 ? ` {score=${score.toFixed(4)}}` : "";
+  return `${fact.title}${confPart}${tagPart}${sourcePart}${scorePart}: ${fact.body}`;
 }
 function renderTaskMarkdown(task) {
   return `- [P${task.priority}] ${task.description.replace(/\n/g, "\n  ")} (${task.status})`;
@@ -2826,6 +2830,8 @@ function formatContext(bundle, options) {
     maxEvents: options?.maxEvents ?? 10,
     includeConfidence: options?.includeConfidence ?? true,
     includeTags: options?.includeTags ?? true,
+    includeEntityIds: options?.includeEntityIds ?? false,
+    includeFactScores: options?.includeFactScores ?? false,
     factWeights: {
       confidence: options?.factWeights?.confidence ?? 1,
       accessCount: options?.factWeights?.accessCount ?? 0.3,
@@ -2837,7 +2843,7 @@ function formatContext(bundle, options) {
   validateMaxOption(opts.maxEvents, "maxEvents");
   const weights = opts.factWeights;
   const now = Date.now();
-  const sortedFacts = [...bundle.facts].sort((a, b) => scoreFactFor(b, weights, now) - scoreFactFor(a, weights, now)).slice(0, opts.maxFacts);
+  const sortedFacts = bundle.factScores ? [...bundle.facts].slice(0, opts.maxFacts) : [...bundle.facts].sort((a, b) => scoreFactFor(b, weights, now) - scoreFactFor(a, weights, now)).slice(0, opts.maxFacts);
   const sortedTasks = [...bundle.tasks].sort((a, b) => b.priority - a.priority || a.created_at - b.created_at).slice(0, opts.maxTasks);
   const sortedEvents = [...bundle.events].sort((a, b) => b.created_at - a.created_at).slice(0, opts.maxEvents);
   if (sortedFacts.length === 0 && sortedTasks.length === 0 && sortedEvents.length === 0) {
@@ -2851,7 +2857,7 @@ function formatContext(bundle, options) {
       lines.push("");
       lines.push("### Known Facts");
       for (const fact of sortedFacts) {
-        lines.push(renderFactMarkdown(fact, opts.includeConfidence, opts.includeTags));
+        lines.push(renderFactMarkdown(fact, opts.includeConfidence, opts.includeTags, opts.includeEntityIds, opts.includeFactScores ? bundle.factScores?.[fact.id] : void 0));
       }
     }
     if (sortedTasks.length > 0) {
@@ -2872,7 +2878,7 @@ function formatContext(bundle, options) {
     if (sortedFacts.length > 0) {
       lines.push("KNOWN FACTS:");
       for (const fact of sortedFacts) {
-        lines.push(renderFactPlain(fact, opts.includeConfidence, opts.includeTags));
+        lines.push(renderFactPlain(fact, opts.includeConfidence, opts.includeTags, opts.includeEntityIds, opts.includeFactScores ? bundle.factScores?.[fact.id] : void 0));
       }
     }
     if (sortedTasks.length > 0) {
@@ -2991,17 +2997,60 @@ function formatMemoryDump(dump) {
   };
 }
 
+// src/librarianPrompt.ts
+var DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT = `You are a careful memory synthesis assistant.
+Use only the retrieved context when answering the request.
+Preserve source provenance when facts come from different entity namespaces.
+
+Request:
+{{query}}
+
+Retrieved context:
+{{context}}
+
+Open tasks:
+{{tasks}}`;
+function hydrateLibrarianPrompt(template, variables) {
+  return template.replace(/\{\{(context|tasks|query)\}\}/g, (_, key) => variables[key]);
+}
+function validateLibrarianPromptTemplate(template, options) {
+  if (!options.custom) return [];
+  const warnings = [];
+  if (!template.includes("{{context}}")) {
+    warnings.push("Custom Librarian systemPrompt omits {{context}}; retrieved memory will not be injected.");
+  }
+  if (!template.includes("{{query}}")) {
+    warnings.push("Custom Librarian systemPrompt omits {{query}}; the original request will not be injected.");
+  }
+  if (options.taskCount > 0 && !template.includes("{{tasks}}")) {
+    warnings.push("Custom Librarian systemPrompt omits {{tasks}} while retrieved tasks are available.");
+  }
+  return warnings;
+}
+function mapLibrarianOptionsToReadOptions(options) {
+  const readOptions = {};
+  if (options.entityWeights !== void 0) readOptions.tierWeights = options.entityWeights;
+  if (options.includeZeroWeightEntities !== void 0) {
+    readOptions.includeZeroWeightEntities = options.includeZeroWeightEntities;
+  }
+  return readOptions;
+}
+
 // src/index.ts
 function createWiki(db, options) {
   return new WikiMemory(db, options);
 }
 
+exports.DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT = DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT;
 exports.PrunePartialFailureError = PrunePartialFailureError;
 exports.WikiBusyError = WikiBusyError;
 exports.WikiMemory = WikiMemory;
 exports.createWiki = createWiki;
 exports.formatContext = formatContext;
 exports.formatMemoryDump = formatMemoryDump;
+exports.hydrateLibrarianPrompt = hydrateLibrarianPrompt;
+exports.mapLibrarianOptionsToReadOptions = mapLibrarianOptionsToReadOptions;
 exports.parseEmbedding = parseEmbedding;
+exports.validateLibrarianPromptTemplate = validateLibrarianPromptTemplate;
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map