@equationalapplications/core-llm-wiki 4.3.0 → 4.5.0

package/README.md

@@ -398,6 +398,71 @@ await wikiMemory.write('user-123', {
 const memory = await wikiMemory.read('user-123', 'coding style preferences');
 ```
 
+### Multi-entity weighted reads
+
+`read()` accepts either one entity id or an array of entity ids. Facts are always merged globally before `maxResults` is applied. For single-entity reads, tasks are uncapped and events are capped at 10. For multi-entity reads, tasks are capped at `min(20 × entity count, 200)` and events at `min(10 × entity count, 100)` — per-entity representation in the returned bundle is not guaranteed.
+
+```ts
+const memory = await wikiMemory.read(['tier_wisdom', 'tier_fact', 'tier_working'], 'Which source should I trust?', {
+  maxResults: 8,
+  tierWeights: {
+    tier_wisdom: 2,
+    tier_fact: 1,
+    tier_working: 0.25,
+  },
+});
+
+console.log(memory.metadata);
+console.log(memory.factScores);
+```
+
+### Librarian prompt override contract
+
+Core does not own a provider-specific synthesis API, but it exports prompt utilities for applications that synthesize answers from weighted retrieval results.
+
+```ts
+import {
+  DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT,
+  formatContext,
+  hydrateLibrarianPrompt,
+  mapLibrarianOptionsToReadOptions,
+  validateLibrarianPromptTemplate,
+} from '@equationalapplications/core-llm-wiki';
+
+const options = {
+  entityWeights: { tier_wisdom: 2, tier_fact: 1, tier_working: 0.25 },
+  systemPrompt: `You are a strict fact checker.
+Question:
+{{query}}
+
+Retrieved context:
+{{context}}
+
+{{tasks}}`,
+};
+
+const query = 'Which source should I trust for recent project decisions?';
+
+const memory = await wikiMemory.read(['tier_wisdom', 'tier_fact', 'tier_working'], query, {
+  ...mapLibrarianOptionsToReadOptions(options),
+  maxResults: 8,
+});
+
+const template = options.systemPrompt ?? DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT;
+const warnings = validateLibrarianPromptTemplate(template, {
+  custom: options.systemPrompt != null,
+  taskCount: memory.tasks.length,
+});
+
+for (const warning of warnings) console.warn(warning);
+
+const finalPrompt = hydrateLibrarianPrompt(template, {
+  query,
+  context: formatContext(memory, { includeEntityIds: true, includeFactScores: true }),
+  tasks: formatContext({ facts: [], tasks: memory.tasks, events: [] }, { format: 'plain' }),
+});
+```
+
 ## Adapter Interface
 
 Implement `SQLiteAdapter` to use your platform's SQLite driver:

package/dist/index.d.mts

@@ -313,6 +313,8 @@ interface FormatContextOptions {
     maxEvents?: number;
     includeConfidence?: boolean;
     includeTags?: boolean;
+    includeEntityIds?: boolean;
+    includeFactScores?: boolean;
     factWeights?: {
         confidence?: number;
         accessCount?: number;
@@ -531,6 +533,28 @@ declare function formatMemoryDump(dump: MemoryDump): FormattedMemoryDump;
 
 declare function parseEmbedding(blob: Uint8Array | null | undefined, text: string | null | undefined): Float32Array | null;
 
+interface LibrarianOptions {
+    /** If provided, replaces the default Librarian system instructions. */
+    systemPrompt?: string;
+    /** entity_id -> score multiplier, forwarded to WikiMemory.read() as tierWeights. */
+    entityWeights?: Record<string, number>;
+    /** Forwarded to WikiMemory.read() for zero-weight filler context. */
+    includeZeroWeightEntities?: boolean;
+    temperature?: number;
+}
+interface LibrarianPromptVariables {
+    context: string;
+    tasks: string;
+    query: string;
+}
+declare const DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT = "You are a careful memory synthesis assistant.\nUse only the retrieved context when answering the request.\nPreserve source provenance when facts come from different entity namespaces.\n\nRequest:\n{{query}}\n\nRetrieved context:\n{{context}}\n\nOpen tasks:\n{{tasks}}";
+declare function hydrateLibrarianPrompt(template: string, variables: LibrarianPromptVariables): string;
+declare function validateLibrarianPromptTemplate(template: string, options: {
+    custom: boolean;
+    taskCount: number;
+}): string[];
+declare function mapLibrarianOptionsToReadOptions(options: LibrarianOptions): Pick<ReadOptions, 'tierWeights' | 'includeZeroWeightEntities'>;
+
 declare function createWiki(db: SQLiteAdapter, options: WikiOptions): WikiMemory;
 
-export { type EntityStatus, type ExtractedFact, type ExtractedTask, type FormatContextOptions, type FormattedMemoryDump, type LLMProvider, type MemoryBundle, type MemoryDump, PrunePartialFailureError, type ReadOptions, type SQLiteAdapter, type VectorRanker, type VectorRankerFallback, type VectorRankerRankArgs, type VectorRankerSemanticResult, WikiBusyError, type WikiBusyOperation, type WikiCheckpoint, type WikiConfig, type WikiEvent, type WikiFact, WikiMemory, type WikiOptions, type WikiTask, createWiki, formatContext, formatMemoryDump, parseEmbedding };
+export { DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, type EntityStatus, type ExtractedFact, type ExtractedTask, type FormatContextOptions, type FormattedMemoryDump, type LLMProvider, type LibrarianOptions, type LibrarianPromptVariables, type MemoryBundle, type MemoryDump, PrunePartialFailureError, type ReadOptions, type SQLiteAdapter, type VectorRanker, type VectorRankerFallback, type VectorRankerRankArgs, type VectorRankerSemanticResult, WikiBusyError, type WikiBusyOperation, type WikiCheckpoint, type WikiConfig, type WikiEvent, type WikiFact, WikiMemory, type WikiOptions, type WikiTask, createWiki, formatContext, formatMemoryDump, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, validateLibrarianPromptTemplate };

package/dist/index.d.ts

@@ -313,6 +313,8 @@ interface FormatContextOptions {
     maxEvents?: number;
     includeConfidence?: boolean;
     includeTags?: boolean;
+    includeEntityIds?: boolean;
+    includeFactScores?: boolean;
     factWeights?: {
         confidence?: number;
         accessCount?: number;
@@ -531,6 +533,28 @@ declare function formatMemoryDump(dump: MemoryDump): FormattedMemoryDump;
 
 declare function parseEmbedding(blob: Uint8Array | null | undefined, text: string | null | undefined): Float32Array | null;
 
+interface LibrarianOptions {
+    /** If provided, replaces the default Librarian system instructions. */
+    systemPrompt?: string;
+    /** entity_id -> score multiplier, forwarded to WikiMemory.read() as tierWeights. */
+    entityWeights?: Record<string, number>;
+    /** Forwarded to WikiMemory.read() for zero-weight filler context. */
+    includeZeroWeightEntities?: boolean;
+    temperature?: number;
+}
+interface LibrarianPromptVariables {
+    context: string;
+    tasks: string;
+    query: string;
+}
+declare const DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT = "You are a careful memory synthesis assistant.\nUse only the retrieved context when answering the request.\nPreserve source provenance when facts come from different entity namespaces.\n\nRequest:\n{{query}}\n\nRetrieved context:\n{{context}}\n\nOpen tasks:\n{{tasks}}";
+declare function hydrateLibrarianPrompt(template: string, variables: LibrarianPromptVariables): string;
+declare function validateLibrarianPromptTemplate(template: string, options: {
+    custom: boolean;
+    taskCount: number;
+}): string[];
+declare function mapLibrarianOptionsToReadOptions(options: LibrarianOptions): Pick<ReadOptions, 'tierWeights' | 'includeZeroWeightEntities'>;
+
 declare function createWiki(db: SQLiteAdapter, options: WikiOptions): WikiMemory;
 
-export { type EntityStatus, type ExtractedFact, type ExtractedTask, type FormatContextOptions, type FormattedMemoryDump, type LLMProvider, type MemoryBundle, type MemoryDump, PrunePartialFailureError, type ReadOptions, type SQLiteAdapter, type VectorRanker, type VectorRankerFallback, type VectorRankerRankArgs, type VectorRankerSemanticResult, WikiBusyError, type WikiBusyOperation, type WikiCheckpoint, type WikiConfig, type WikiEvent, type WikiFact, WikiMemory, type WikiOptions, type WikiTask, createWiki, formatContext, formatMemoryDump, parseEmbedding };
+export { DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, type EntityStatus, type ExtractedFact, type ExtractedTask, type FormatContextOptions, type FormattedMemoryDump, type LLMProvider, type LibrarianOptions, type LibrarianPromptVariables, type MemoryBundle, type MemoryDump, PrunePartialFailureError, type ReadOptions, type SQLiteAdapter, type VectorRanker, type VectorRankerFallback, type VectorRankerRankArgs, type VectorRankerSemanticResult, WikiBusyError, type WikiBusyOperation, type WikiCheckpoint, type WikiConfig, type WikiEvent, type WikiFact, WikiMemory, type WikiOptions, type WikiTask, createWiki, formatContext, formatMemoryDump, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, validateLibrarianPromptTemplate };

package/dist/index.js

@@ -1064,14 +1064,14 @@ After running the migration SQL, restart your application.`
               );
             }
           }
-          const entityScope = this._entityInClause(scoredEntityIds);
+          const mismatchScope = this._entityInClause(entityIds);
           const mismatchedCount = await this.db.getFirstAsync(
             `SELECT COUNT(*) AS cnt FROM ${this.prefix}entries
-             WHERE ${entityScope.clause} AND deleted_at IS NULL
+             WHERE ${mismatchScope.clause} AND deleted_at IS NULL
                AND embedding_blob IS NOT NULL
                AND (CAST(length(embedding_blob) AS INTEGER) % 4 = 0)
                AND (CAST(length(embedding_blob) AS INTEGER) / 4) != ?`,
-            [...entityScope.params, queryVec.length]
+            [...mismatchScope.params, queryVec.length]
           );
           if (mismatchedCount && mismatchedCount.cnt > 0) {
             throw new Error(
@@ -1131,16 +1131,16 @@ After running the migration SQL, restart your application.`
             }
           } else {
             if (useRanker) {
-              const entityScope2 = this._entityInClause(scoredEntityIds);
+              const entityScope = this._entityInClause(scoredEntityIds);
               candidateRows = await this.db.getAllAsync(
-                `SELECT id, entity_id, updated_at, access_count FROM ${this.prefix}entries WHERE ${entityScope2.clause} AND deleted_at IS NULL`,
-                entityScope2.params
+                `SELECT id, entity_id, updated_at, access_count FROM ${this.prefix}entries WHERE ${entityScope.clause} AND deleted_at IS NULL`,
+                entityScope.params
               );
             } else {
-              const entityScope2 = this._entityInClause(scoredEntityIds);
+              const entityScope = this._entityInClause(scoredEntityIds);
               candidateRows = await this.db.getAllAsync(
-                `SELECT id, entity_id, embedding_blob, embedding, updated_at, access_count FROM ${this.prefix}entries WHERE ${entityScope2.clause} AND deleted_at IS NULL`,
-                entityScope2.params
+                `SELECT id, entity_id, embedding_blob, embedding, updated_at, access_count FROM ${this.prefix}entries WHERE ${entityScope.clause} AND deleted_at IS NULL`,
+                entityScope.params
               );
             }
             if (weight !== void 0 && weight < 1) {
@@ -1159,32 +1159,42 @@ After running the migration SQL, restart your application.`
             const entityCacheKey = entityIds.length === 1 ? entityIds[0] : entityIds.join("\0");
             let scored;
             if (useRanker) {
-              const candidateIds = entityIds.length > 1 || effectivePreFilterLimit !== void 0 ? candidateRows.map((r) => r.id) : void 0;
+              const candidateRowsByEntity = /* @__PURE__ */ new Map();
+              for (const row of candidateRows) {
+                const rows = candidateRowsByEntity.get(row.entity_id) ?? [];
+                rows.push(row);
+                candidateRowsByEntity.set(row.entity_id, rows);
+              }
               try {
-                const oversampledLimit = Math.max(maxResults * 2, maxResults + 50);
-                scored = await this._rankWithVectorRanker({
-                  entityId: entityCacheKey,
-                  queryVec,
-                  candidateIds,
-                  candidateRows,
-                  weight,
-                  miniSearchScores,
-                  limit: oversampledLimit
-                });
-                if (scored.length > 0) {
-                  const scoredIds2 = new Set(scored.map((s) => s.id));
-                  const metaMap = /* @__PURE__ */ new Map();
-                  for (const r of candidateRows) {
-                    if (scoredIds2.has(r.id)) {
-                      metaMap.set(r.id, { updated_at: r.updated_at, access_count: r.access_count });
-                    }
-                  }
-                  scored = scored.map((s) => {
-                    const meta = metaMap.get(s.id);
-                    return { ...s, updated_at: meta?.updated_at ?? null, access_count: meta?.access_count ?? null };
-                  });
-                }
+                const rankerResultsByEntity = await Promise.all(
+                  scoredEntityIds.filter((id) => (candidateRowsByEntity.get(id)?.length ?? 0) > 0).map(async (scopedEntityId) => {
+                    const rowsForEntity = candidateRowsByEntity.get(scopedEntityId) ?? [];
+                    const candidateIds = effectivePreFilterLimit !== void 0 ? rowsForEntity.map((row) => row.id) : void 0;
+                    const ranked = await this._rankWithVectorRanker({
+                      entityId: scopedEntityId,
+                      queryVec,
+                      candidateIds,
+                      candidateRows: rowsForEntity,
+                      weight,
+                      miniSearchScores,
+                      limit: Math.max(maxResults * 2, maxResults + 50)
+                    });
+                    return ranked.map((row) => ({ ...row, entity_id: scopedEntityId }));
+                  })
+                );
+                scored = rankerResultsByEntity.flat();
                 const scoredIds = new Set(scored.map((s) => s.id));
+                const metadataById = new Map(
+                  candidateRows.filter((row) => scoredIds.has(row.id)).map((row) => [row.id, row])
+                );
+                scored = scored.map((row) => {
+                  const metadata = metadataById.get(row.id);
+                  return {
+                    ...row,
+                    updated_at: metadata?.updated_at ?? null,
+                    access_count: metadata?.access_count ?? null
+                  };
+                });
                 const isHybrid = weight !== void 0 && weight < 1;
                 const maxBackfill = isHybrid ? maxResults : Math.max(0, maxResults - scored.length);
                 if (maxBackfill > 0) {
@@ -1332,21 +1342,17 @@ After running the migration SQL, restart your application.`
                   });
                   const keywordOversampledLimit = Math.max(maxResults * 2, maxResults + 50);
                   const topResults = msResults.slice(0, keywordOversampledLimit);
-                  const resultIds = new Set(topResults.map((r) => r.id));
-                  const candidateMap = /* @__PURE__ */ new Map();
-                  for (const r of candidateRows) {
-                    if (resultIds.has(r.id)) {
-                      candidateMap.set(r.id, { entity_id: r.entity_id, updated_at: r.updated_at, access_count: r.access_count });
-                    }
-                  }
-                  scored = topResults.map((r) => {
-                    const meta = candidateMap.get(r.id);
+                  const topResultIds = new Set(topResults.map((r) => r.id));
+                  const candidateMap = new Map(candidateRows.filter((r) => topResultIds.has(r.id)).map((row) => [row.id, row]));
+                  scored = topResults.map((result) => {
+                    const metadata = candidateMap.get(result.id);
+                    const entityForScore = metadata?.entity_id ?? result.entity_id ?? "";
                     return {
-                      id: r.id,
-                      entity_id: meta?.entity_id ?? r.entity_id,
-                      score: r.score ?? 0,
-                      access_count: meta?.access_count ?? null,
-                      updated_at: meta?.updated_at ?? null
+                      id: result.id,
+                      entity_id: entityForScore,
+                      score: result.score ?? 0,
+                      access_count: metadata?.access_count ?? null,
+                      updated_at: metadata?.updated_at ?? null
                     };
                   });
                 } else {
@@ -2787,16 +2793,20 @@ function scoreFactFor(fact, weights, now) {
   const recencyDecay = Math.exp(-ageDays / 30);
   return confW * weights.confidence + Math.log(1 + fact.access_count) * weights.accessCount + recencyDecay * weights.recency;
 }
-function renderFactMarkdown(fact, includeConfidence, includeTags) {
+function renderFactMarkdown(fact, includeConfidence, includeTags, includeEntityIds, score) {
   const confPart = includeConfidence ? ` (${fact.confidence})` : "";
   const tagPart = includeTags && fact.tags.length > 0 ? ` [${fact.tags.join(", ")}]` : "";
-  return `- **${fact.title}**${confPart}${tagPart}
+  const sourcePart = includeEntityIds ? ` {entity_id=${fact.entity_id}}` : "";
+  const scorePart = score !== void 0 ? ` {score=${score.toFixed(4)}}` : "";
+  return `- **${fact.title}**${confPart}${tagPart}${sourcePart}${scorePart}
   ${fact.body.replace(/\n/g, "\n  ")}`;
 }
-function renderFactPlain(fact, includeConfidence, includeTags) {
+function renderFactPlain(fact, includeConfidence, includeTags, includeEntityIds, score) {
   const confPart = includeConfidence ? ` (${fact.confidence})` : "";
   const tagPart = includeTags && fact.tags.length > 0 ? ` [${fact.tags.join(", ")}]` : "";
-  return `${fact.title}${confPart}${tagPart}: ${fact.body}`;
+  const sourcePart = includeEntityIds ? ` {entity_id=${fact.entity_id}}` : "";
+  const scorePart = score !== void 0 ? ` {score=${score.toFixed(4)}}` : "";
+  return `${fact.title}${confPart}${tagPart}${sourcePart}${scorePart}: ${fact.body}`;
 }
 function renderTaskMarkdown(task) {
   return `- [P${task.priority}] ${task.description.replace(/\n/g, "\n  ")} (${task.status})`;
@@ -2820,6 +2830,8 @@ function formatContext(bundle, options) {
     maxEvents: options?.maxEvents ?? 10,
     includeConfidence: options?.includeConfidence ?? true,
     includeTags: options?.includeTags ?? true,
+    includeEntityIds: options?.includeEntityIds ?? false,
+    includeFactScores: options?.includeFactScores ?? false,
     factWeights: {
       confidence: options?.factWeights?.confidence ?? 1,
       accessCount: options?.factWeights?.accessCount ?? 0.3,
@@ -2831,7 +2843,7 @@ function formatContext(bundle, options) {
   validateMaxOption(opts.maxEvents, "maxEvents");
   const weights = opts.factWeights;
   const now = Date.now();
-  const sortedFacts = [...bundle.facts].sort((a, b) => scoreFactFor(b, weights, now) - scoreFactFor(a, weights, now)).slice(0, opts.maxFacts);
+  const sortedFacts = bundle.factScores ? [...bundle.facts].slice(0, opts.maxFacts) : [...bundle.facts].sort((a, b) => scoreFactFor(b, weights, now) - scoreFactFor(a, weights, now)).slice(0, opts.maxFacts);
   const sortedTasks = [...bundle.tasks].sort((a, b) => b.priority - a.priority || a.created_at - b.created_at).slice(0, opts.maxTasks);
   const sortedEvents = [...bundle.events].sort((a, b) => b.created_at - a.created_at).slice(0, opts.maxEvents);
   if (sortedFacts.length === 0 && sortedTasks.length === 0 && sortedEvents.length === 0) {
@@ -2845,7 +2857,7 @@ function formatContext(bundle, options) {
       lines.push("");
       lines.push("### Known Facts");
       for (const fact of sortedFacts) {
-        lines.push(renderFactMarkdown(fact, opts.includeConfidence, opts.includeTags));
+        lines.push(renderFactMarkdown(fact, opts.includeConfidence, opts.includeTags, opts.includeEntityIds, opts.includeFactScores ? bundle.factScores?.[fact.id] : void 0));
       }
     }
     if (sortedTasks.length > 0) {
@@ -2866,7 +2878,7 @@ function formatContext(bundle, options) {
     if (sortedFacts.length > 0) {
       lines.push("KNOWN FACTS:");
       for (const fact of sortedFacts) {
-        lines.push(renderFactPlain(fact, opts.includeConfidence, opts.includeTags));
+        lines.push(renderFactPlain(fact, opts.includeConfidence, opts.includeTags, opts.includeEntityIds, opts.includeFactScores ? bundle.factScores?.[fact.id] : void 0));
       }
     }
     if (sortedTasks.length > 0) {
@@ -2985,17 +2997,60 @@ function formatMemoryDump(dump) {
   };
 }
 
+// src/librarianPrompt.ts
+var DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT = `You are a careful memory synthesis assistant.
+Use only the retrieved context when answering the request.
+Preserve source provenance when facts come from different entity namespaces.
+
+Request:
+{{query}}
+
+Retrieved context:
+{{context}}
+
+Open tasks:
+{{tasks}}`;
+function hydrateLibrarianPrompt(template, variables) {
+  return template.replace(/\{\{(context|tasks|query)\}\}/g, (_, key) => variables[key]);
+}
+function validateLibrarianPromptTemplate(template, options) {
+  if (!options.custom) return [];
+  const warnings = [];
+  if (!template.includes("{{context}}")) {
+    warnings.push("Custom Librarian systemPrompt omits {{context}}; retrieved memory will not be injected.");
+  }
+  if (!template.includes("{{query}}")) {
+    warnings.push("Custom Librarian systemPrompt omits {{query}}; the original request will not be injected.");
+  }
+  if (options.taskCount > 0 && !template.includes("{{tasks}}")) {
+    warnings.push("Custom Librarian systemPrompt omits {{tasks}} while retrieved tasks are available.");
+  }
+  return warnings;
+}
+function mapLibrarianOptionsToReadOptions(options) {
+  const readOptions = {};
+  if (options.entityWeights !== void 0) readOptions.tierWeights = options.entityWeights;
+  if (options.includeZeroWeightEntities !== void 0) {
+    readOptions.includeZeroWeightEntities = options.includeZeroWeightEntities;
+  }
+  return readOptions;
+}
+
 // src/index.ts
 function createWiki(db, options) {
   return new WikiMemory(db, options);
 }
 
+exports.DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT = DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT;
 exports.PrunePartialFailureError = PrunePartialFailureError;
 exports.WikiBusyError = WikiBusyError;
 exports.WikiMemory = WikiMemory;
 exports.createWiki = createWiki;
 exports.formatContext = formatContext;
 exports.formatMemoryDump = formatMemoryDump;
+exports.hydrateLibrarianPrompt = hydrateLibrarianPrompt;
+exports.mapLibrarianOptionsToReadOptions = mapLibrarianOptionsToReadOptions;
 exports.parseEmbedding = parseEmbedding;
+exports.validateLibrarianPromptTemplate = validateLibrarianPromptTemplate;
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map