@equationalapplications/core-llm-wiki 6.3.0 → 6.4.0

package/README.md

@@ -16,8 +16,8 @@ Platform-agnostic TypeScript engine for hybrid LLM memory. Features episodic fac
 - **Platform-agnostic** — Zero runtime dependencies; works with any SQLite driver via the `SQLiteAdapter` interface
 - **Semantic search** — Vector embeddings via your LLM's `embed` function, ranked by cosine similarity
 - **Keyword fallback** — [MiniSearch](https://github.com/lucaong/minisearch) in-memory index for offline/degraded scenarios when embeddings unavailable
-- **Retrieval tuning** — Per-call overrides for `maxResults`, `preFilterLimit`, `hybridWeight`, `tierWeights`, and `includeZeroWeightEntities`
-- **Multi-entity reads** — Search across multiple `entity_id` namespaces in one pass with per-entity score multipliers (`tierWeights`); optional `factScores` and `metadata` for explainability
+- **Retrieval tuning** — Per-call overrides for `maxResults`, `preFilterLimit`, `hybridWeight`, `tierWeights`, `tierFloors`, and `includeZeroWeightEntities`
+- **Multi-entity reads** — Search across multiple `entity_id` namespaces in one pass with per-entity score multipliers (`tierWeights`); `tierFloors` reserves each namespace's top-N matching results; optional `factScores` and `metadata` for explainability
 - **Immutable vs mutable facts** — Use `WikiFact.source_type` to distinguish document-sourced facts (`immutable_document`) from derived or user-provided facts (`librarian_inferred`, `user_stated`, `user_confirmed`). Immutable document facts are not rewritten by `runLibrarian()` or `runHeal()` and can only be removed by `forget()` or re-ingesting.
 - **Full-featured memory** — Facts, tasks, events, maintenance jobs (librarian, heal, reembed, prune)
 - **Type-safe** — Built with TypeScript, full type exports
@@ -135,6 +135,7 @@ const wikiMemory = new WikiMemory(db, {
     maxChunkLength: 12000,             // default: 12000 (char count per ingestDocument chunk; exported as DEFAULT_MAX_CHUNK_LENGTH)
     chunkOverlap: 400,                 // default: 400 (overlap between chunks in characters; exported as DEFAULT_CHUNK_OVERLAP)
     chunkConcurrency: 1,               // default: 1 (parallel LLM calls per ingestDocument)
+    maxEmbedChars: 6000,               // default: 6000 (chars of title+body+tags sent to embed(); hard ceiling 16000; exported as DEFAULT_MAX_EMBED_CHARS)
     pruneRetainSoftDeletedFor: 7,      // default: 7 (days before hard-deleting soft-deleted facts)
     pruneEventsAfter: 30,              // default: 30 (days before hard-deleting old events)
     orphanAfterDays: 30,               // default: 30 (days before runHeal flags sourceless facts; null to disable)
@@ -261,7 +262,7 @@ const multiMemory = await wikiMemory.read(['tier_wisdom', 'tier_fact', 'tier_wor
   // includeZeroWeightEntities: true — include 0-weight entities as bottom-ranked filler
 });
 // multiMemory.factScores — optional Record<factId, weightedScore> for returned facts; may be absent/undefined
-// multiMemory.metadata  — optional { query, entityIds, tierWeights }; may be absent/undefined
+// multiMemory.metadata  — optional { query, entityIds, tierWeights, tierFloors }; may be absent/undefined
 ```
 
 **Hybrid scoring blends:**
@@ -276,6 +277,7 @@ True cosine-range pure semantic ranking (including negative cosine values) is us
 - Missing weights default to `1.0`. Negative weights clamp to `0`. Non-finite weights default to `1.0`.
 - `tierWeights[entity] = 0` skips that entity's scored retrieval branch (no compute cost).
 - `includeZeroWeightEntities: true` includes zero-weight entities as bottom-ranked filler instead of skipping them.
+- `tierFloors: Record<string, number>` reserves that entity's top-N scored results before the global `maxResults` cut — a floor on entity `X` means at least `N` of the returned facts come from `X`. Floors apply *after* scoring (including `tierWeights`) and *after* `preFilterLimit`'s candidate selection, so a row excluded by `preFilterLimit` cannot be resurrected by a floor. An entity with fewer matching facts than its floor contributes what it has — this is not an error. Only meaningful when `entityId` is an array; ignored for single-string calls and for empty-query ("recent facts") reads. Throws `WikiInvalidReadOptions` when a floor cannot be satisfied by construction: floors summing above `maxResults`, a floor keyed to an entity not in `entityId`, or a floor on an entity excluded by `tierWeights: 0` when `includeZeroWeightEntities` is not set. With an external `vectorRanker`, ranker-omitted rows (typically un-embedded facts) for a floored entity are pre-reserved before the global backfill budget is spent, so an unembedded floored entity still satisfies its floor.
 - `factScores` is present for array-shaped `entityId` calls only when the query is non-empty and at least one fact is scored; empty-query ("recent facts") reads leave it absent even when `entityId` is an array. Plain string calls never expose it. `metadata` is present for all array-shaped calls regardless of query.
 - `maxResults` applies globally across all requested entities.
 - Tasks are capped at `min(20 × entityCount, 200)`; events at `min(10 × entityCount, 100)` for multi-entity reads.

package/dist/{chunk-YDTJ2OO4.mjs → chunk-DPFM33WV.mjs}

@@ -144,6 +144,15 @@ var WikiIngestEmptyError = class extends Error {
     this.chunks = params.chunks;
   }
 };
+var WikiInvalidReadOptions = class _WikiInvalidReadOptions extends Error {
+  constructor(field, reason) {
+    super(`Invalid ReadOptions.${field}: ${reason}`);
+    this.name = "WikiInvalidReadOptions";
+    this.field = field;
+    this.reason = reason;
+    Object.setPrototypeOf(this, _WikiInvalidReadOptions.prototype);
+  }
+};
 
 // src/utils/ids.ts
 var _injectedGetRandomValues = null;
@@ -3841,6 +3850,10 @@ After running the migration SQL, restart your application.`
   }
 };
 
+// src/utils/embedDefaults.ts
+var DEFAULT_MAX_EMBED_CHARS = 6e3;
+var EMBED_CHARS_CEILING = 16e3;
+
 // src/services/EmbeddingService.ts
 var EmbeddingService = class {
   constructor(db, options, entryRepo, metadataRepo) {
@@ -3888,7 +3901,9 @@ var EmbeddingService = class {
         tagsStr = fact.tags;
       }
     }
-    const text = clip(`${fact.title} ${fact.body} ${tagsStr}`.trim(), 16e3);
+    const configuredMaxEmbedChars = this.options.config?.maxEmbedChars;
+    const maxEmbedChars = Number.isFinite(configuredMaxEmbedChars) ? Math.min(Math.max(0, Math.trunc(configuredMaxEmbedChars)), EMBED_CHARS_CEILING) : DEFAULT_MAX_EMBED_CHARS;
+    const text = clip(`${fact.title} ${fact.body} ${tagsStr}`.trim(), maxEmbedChars);
     try {
       const vector = await embedFn(text);
       if (vector.length === 0 || !vector.every((v) => typeof v === "number" && isFinite(v))) {
@@ -3999,6 +4014,63 @@ function applyTierWeight(score, entityId, sanitizedTierWeights) {
 function shouldExposeReadMetadata(entityId) {
   return Array.isArray(entityId);
 }
+function validateTierFloors(entityIds, tierFloors, sanitizedTierWeights, includeZeroWeightEntities, maxResults) {
+  if (tierFloors === void 0) return void 0;
+  const known = new Set(entityIds);
+  for (const key of Object.keys(tierFloors)) {
+    const raw = tierFloors[key];
+    if (raw === void 0 || !Number.isFinite(raw)) continue;
+    if (!known.has(key)) {
+      throw new WikiInvalidReadOptions(
+        "tierFloors",
+        `"${key}" is not one of the entity IDs passed to read(); a floor on an unrequested entity can never be satisfied`
+      );
+    }
+  }
+  const sanitized = /* @__PURE__ */ Object.create(null);
+  let total = 0;
+  for (const entityId of entityIds) {
+    const raw = tierFloors[entityId];
+    if (raw === void 0 || !Number.isFinite(raw)) continue;
+    const floor = Math.max(0, Math.trunc(raw));
+    if (floor > 0 && includeZeroWeightEntities !== true && sanitizedTierWeights?.[entityId] === 0) {
+      throw new WikiInvalidReadOptions(
+        "tierFloors",
+        `"${entityId}" has tierWeight 0 and includeZeroWeightEntities is not set, so it is excluded from retrieval and its floor of ${floor} can never be met`
+      );
+    }
+    sanitized[entityId] = floor;
+    total += floor;
+  }
+  if (total > maxResults) {
+    throw new WikiInvalidReadOptions(
+      "tierFloors",
+      `floors sum to ${total}, which exceeds maxResults (${maxResults})`
+    );
+  }
+  return sanitized;
+}
+function selectWithFloors(sortedRows, floors, maxResults) {
+  if (maxResults <= 0) return [];
+  if (floors === void 0 || Object.keys(floors).length === 0) {
+    return sortedRows.slice(0, maxResults);
+  }
+  const selected = /* @__PURE__ */ new Set();
+  const taken = /* @__PURE__ */ Object.create(null);
+  for (let i = 0; i < sortedRows.length; i++) {
+    const entityId = sortedRows[i].entity_id;
+    const floor = floors[entityId] ?? 0;
+    if (floor <= 0) continue;
+    const count = taken[entityId] ?? 0;
+    if (count >= floor) continue;
+    taken[entityId] = count + 1;
+    selected.add(i);
+  }
+  for (let i = 0; i < sortedRows.length && selected.size < maxResults; i++) {
+    if (!selected.has(i)) selected.add(i);
+  }
+  return Array.from(selected).sort((a, b) => a - b).map((i) => sortedRows[i]);
+}
 
 // src/services/RetrievalService.ts
 var RetrievalService = class {
@@ -4033,13 +4105,20 @@ var RetrievalService = class {
     }
     const rawMaxResults = options?.maxResults ?? config?.maxResults ?? config?.maxFtsResults ?? 10;
     const maxResults = Number.isFinite(rawMaxResults) ? Math.max(0, Math.trunc(rawMaxResults)) : 10;
+    const trimmedQuery = query.trim();
+    const sanitizedTierFloors = trimmedQuery && exposeMetadata ? validateTierFloors(
+      entityIds,
+      options?.tierFloors,
+      sanitizedTierWeights,
+      options?.includeZeroWeightEntities,
+      maxResults
+    ) : void 0;
     const rawPreFilterLimit = options?.preFilterLimit === null ? void 0 : options?.preFilterLimit ?? config?.preFilterLimit;
     const effectivePreFilterLimit = rawPreFilterLimit === void 0 ? void 0 : Number.isFinite(rawPreFilterLimit) ? Math.max(0, Math.trunc(rawPreFilterLimit)) : void 0;
     const hybridWeight = options?.hybridWeight ?? config?.hybridWeight;
     const weight = hybridWeight !== void 0 && !Number.isNaN(hybridWeight) ? Math.max(0, Math.min(1, hybridWeight)) : void 0;
     const skipEmbed = weight === 0;
     const embedFn = this.options.llmProvider.embed;
-    const trimmedQuery = query.trim();
     let facts = [];
     let scoreByFactId;
     if (maxResults === 0) ; else if (trimmedQuery) {
@@ -4150,13 +4229,74 @@ var RetrievalService = class {
                     access_count: metadata?.access_count ?? null
                   };
                 });
+                const floorShortfallByEntity = /* @__PURE__ */ Object.create(null);
+                let totalFloorShortfall = 0;
+                if (sanitizedTierFloors) {
+                  const scoredCounts = /* @__PURE__ */ new Map();
+                  for (const s of scored) {
+                    scoredCounts.set(s.entity_id, (scoredCounts.get(s.entity_id) ?? 0) + 1);
+                  }
+                  for (const [entityId2, floor] of Object.entries(sanitizedTierFloors)) {
+                    if (floor <= 0) continue;
+                    const currentCount = scoredCounts.get(entityId2) ?? 0;
+                    const shortfall = Math.max(0, floor - currentCount);
+                    if (shortfall > 0) {
+                      floorShortfallByEntity[entityId2] = shortfall;
+                      totalFloorShortfall += shortfall;
+                    }
+                  }
+                }
+                const reservedIds = /* @__PURE__ */ new Set();
+                const reserved = [];
+                if (totalFloorShortfall > 0) {
+                  for (const [entityId2, shortfall] of Object.entries(floorShortfallByEntity)) {
+                    const entityOmitted = [];
+                    for (const row of candidateRows) {
+                      if (scoredIds.has(row.id) || reservedIds.has(row.id)) continue;
+                      if (row.entity_id !== entityId2) continue;
+                      const kwScore = miniSearchScores?.get(row.id) ?? 0;
+                      entityOmitted.push({ row, kwScore });
+                    }
+                    entityOmitted.sort((a, b) => this._compareScoredRows(
+                      { id: a.row.id, score: a.kwScore, updated_at: a.row.updated_at, access_count: a.row.access_count },
+                      { id: b.row.id, score: b.kwScore, updated_at: b.row.updated_at, access_count: b.row.access_count }
+                    ));
+                    for (let i = 0; i < Math.min(shortfall, entityOmitted.length); i++) {
+                      reserved.push(entityOmitted[i]);
+                      reservedIds.add(entityOmitted[i].row.id);
+                    }
+                  }
+                }
                 const isHybrid = weight !== void 0 && weight < 1;
+                if (reserved.length > 0) {
+                  if (isHybrid) {
+                    for (const { row, kwScore } of reserved) {
+                      scored.push({
+                        id: row.id,
+                        entity_id: row.entity_id,
+                        score: (1 - weight) * kwScore,
+                        updated_at: row.updated_at,
+                        access_count: row.access_count
+                      });
+                    }
+                  } else {
+                    for (const { row } of reserved) {
+                      scored.push({
+                        id: row.id,
+                        entity_id: row.entity_id,
+                        score: -2,
+                        updated_at: row.updated_at,
+                        access_count: row.access_count
+                      });
+                    }
+                  }
+                }
                 const maxBackfill = isHybrid ? maxResults : Math.max(0, maxResults - scored.length);
                 if (maxBackfill > 0) {
                   if (isHybrid) {
                     const topK = [];
                     for (const row of candidateRows) {
-                      if (scoredIds.has(row.id)) continue;
+                      if (scoredIds.has(row.id) || reservedIds.has(row.id)) continue;
                       const kwScore = miniSearchScores?.get(row.id) ?? 0;
                       const candidate = { row, kwScore };
                       if (topK.length < maxBackfill) {
@@ -4236,7 +4376,7 @@ var RetrievalService = class {
                   } else {
                     const omitted = [];
                     for (const row of candidateRows) {
-                      if (scoredIds.has(row.id)) continue;
+                      if (scoredIds.has(row.id) || reservedIds.has(row.id)) continue;
                       omitted.push({ id: row.id, entity_id: row.entity_id, score: -2, updated_at: row.updated_at, access_count: row.access_count });
                     }
                     if (omitted.length > 0) {
@@ -4284,8 +4424,11 @@ var RetrievalService = class {
                     // read() re-sorts after applying tier weights
                   });
                 } else if (policy === "keyword") {
-                  const keywordOversampledLimit = Math.max(maxResults * 2, maxResults + 50);
-                  const topResults = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, keywordOversampledLimit);
+                  const hasActiveFloorsKeywordFallback = sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
+                  const keywordOversampledLimit = hasActiveFloorsKeywordFallback ? Number.MAX_SAFE_INTEGER : Math.max(maxResults * 2, maxResults + 50);
+                  const preFilteredIds = effectivePreFilterLimit !== void 0 ? new Set(candidateRows.map((r) => r.id)) : void 0;
+                  const keywordResults = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, keywordOversampledLimit);
+                  const topResults = preFilteredIds === void 0 ? keywordResults : keywordResults.filter((r) => preFilteredIds.has(r.id));
                   const topResultIds = new Set(topResults.map((r) => r.id));
                   const candidateMap = new Map(candidateRows.filter((r) => topResultIds.has(r.id)).map((row) => [row.id, row]));
                   scored = topResults.map((result) => {
@@ -4310,7 +4453,7 @@ var RetrievalService = class {
                 }
               }
             } else {
-              const jsCosineNeedsTierSort = sanitizedTierWeights !== void 0 && Object.values(sanitizedTierWeights).some((w) => w !== 1);
+              const jsCosineNeedsTierSort = sanitizedTierWeights !== void 0 && Object.values(sanitizedTierWeights).some((w) => w !== 1) || sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
               scored = await this.searchService.rankSemantic({
                 entityId: entityCacheKey,
                 queryVec,
@@ -4329,7 +4472,7 @@ var RetrievalService = class {
                 score: applyTierWeight(row.score, row.entity_id, sanitizedTierWeights)
               }));
               this._tieBreakSort(scored);
-              const selectedScored = scored.slice(0, maxResults);
+              const selectedScored = selectWithFloors(scored, sanitizedTierFloors, maxResults);
               const topIds = selectedScored.map((s) => s.id);
               if (exposeMetadata && trimmedQuery) {
                 scoreByFactId = new Map(selectedScored.map((s) => [s.id, Number.isFinite(s.score) ? s.score : 0]));
@@ -4384,7 +4527,8 @@ var RetrievalService = class {
         }
       }
       if (!usedEmbed && scoredEntityIds.length > 0) {
-        const fallbackOversampledLimit = Math.max(maxResults * 2, maxResults + 50);
+        const hasActiveFloors = sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
+        const fallbackOversampledLimit = hasActiveFloors ? Number.MAX_SAFE_INTEGER : Math.max(maxResults * 2, maxResults + 50);
         const results = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, fallbackOversampledLimit);
         const candidates = results.map((r) => ({
           id: r.id,
@@ -4394,7 +4538,7 @@ var RetrievalService = class {
           access_count: null
         }));
         this._tieBreakSort(candidates);
-        const topCandidates = candidates.slice(0, maxResults);
+        const topCandidates = selectWithFloors(candidates, sanitizedTierFloors, maxResults);
         const topIds = topCandidates.map((c) => c.id);
         if (topIds.length > 0) {
           facts = await this._hydrateFactsByIds(topIds, entityIds);
@@ -4424,6 +4568,7 @@ var RetrievalService = class {
     if (exposeMetadata) {
       bundle.metadata = { query, entityIds };
       if (sanitizedTierWeights && Object.keys(sanitizedTierWeights).length > 0) bundle.metadata.tierWeights = sanitizedTierWeights;
+      if (sanitizedTierFloors && Object.keys(sanitizedTierFloors).length > 0) bundle.metadata.tierFloors = sanitizedTierFloors;
       if (factScores && Object.keys(factScores).length > 0) bundle.factScores = factScores;
     }
     return bundle;
@@ -4633,6 +4778,6 @@ var WriteService = class {
   }
 };
 
-export { BaseRepository, DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, EmbeddingService, HEAL_ANCHORS_PER_CANDIDATE, HEAL_BATCH_SIZE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, WikiBusyError, WikiDuplicateHashError, WikiIngestEmptyError, WikiParseError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, chunkText, configureRandomSource, emptyManifest, entitySummaryMetaKey, extractSqliteCode, generateId, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, parseEmbedding, resolveEdgeDefinitions, resolveNodeType, safeSlice, typeSatisfies, validateInlineEdges, validateManifest };
-//# sourceMappingURL=chunk-YDTJ2OO4.mjs.map
-//# sourceMappingURL=chunk-YDTJ2OO4.mjs.map
+export { BaseRepository, DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, DEFAULT_MAX_EMBED_CHARS, EMBED_CHARS_CEILING, EmbeddingService, HEAL_ANCHORS_PER_CANDIDATE, HEAL_BATCH_SIZE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, WikiBusyError, WikiDuplicateHashError, WikiIngestEmptyError, WikiInvalidReadOptions, WikiParseError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, chunkText, configureRandomSource, emptyManifest, entitySummaryMetaKey, extractSqliteCode, generateId, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, parseEmbedding, resolveEdgeDefinitions, resolveNodeType, safeSlice, typeSatisfies, validateInlineEdges, validateManifest };
+//# sourceMappingURL=chunk-DPFM33WV.mjs.map
+//# sourceMappingURL=chunk-DPFM33WV.mjs.map