@equationalapplications/core-llm-wiki 4.21.0 → 4.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.mts CHANGED
@@ -1,5 +1,5 @@
1
- import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-i91HR1TG.mjs';
2
- export { E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HOOK_TIMEOUT_MARKER, L as LLMProvider, O as ONTOLOGY_BACKFILL_BATCH_SIZE, i as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, j as ONTOLOGY_BACKFILL_RECHECK_MS, k as OntologyBackfillResult, l as OntologyConfig, m as OntologyEdgeType, n as OntologyManifest, o as OntologyMode, p as OntologyNodeType, q as OntologyPromptContext, r as OntologyUpdates, P as PromptOverrides, s as PromptService, t as PrunePartialFailureError, V as VectorRanker, u as VectorRankerFallback, v as VectorRankerRankArgs, w as VectorRankerSemanticResult, x as WikiBusyError, y as WikiBusyOperation, z as WikiCheckpoint, A as WikiConfig, B as WikiEdge, C as WikiEvent, D as WikiFact, I as WikiMemoryTestAccess, J as WikiOutboxEvent, K as WikiTask, N as WikiTransactionError } from './testing-i91HR1TG.mjs';
1
+ import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, O as OntologyManifest, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-CBjAuTSl.mjs';
2
+ export { E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HOOK_TIMEOUT_MARKER, L as LLMProvider, i as ONTOLOGY_BACKFILL_BATCH_SIZE, j as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, k as ONTOLOGY_BACKFILL_RECHECK_MS, l as OntologyBackfillResult, m as OntologyConfig, n as OntologyEdgeType, o as OntologyMode, p as OntologyNodeType, q as OntologyPromptContext, r as OntologyUpdates, P as PromptOverrides, s as PromptService, t as PrunePartialFailureError, V as VectorRanker, u as VectorRankerFallback, v as VectorRankerRankArgs, w as VectorRankerSemanticResult, x as WikiBusyError, y as WikiBusyOperation, z as WikiCheckpoint, A as WikiConfig, B as WikiEdge, C as WikiEvent, D as WikiFact, I as WikiMemoryTestAccess, J as WikiOutboxEvent, K as WikiTask, N as WikiTransactionError } from './testing-CBjAuTSl.mjs';
3
3
  import { OkfFile } from '@equationalapplications/core-okf';
4
4
  import 'minisearch';
5
5
 
@@ -26,6 +26,8 @@ declare function parseOkfBundle(entityId: string, files: OkfFile[], options?: Ok
26
26
 
27
27
  declare function parseEmbedding(blob: Uint8Array | null | undefined, text: string | null | undefined): Float32Array | null;
28
28
 
29
+ declare function validateManifest(manifest: OntologyManifest): void;
30
+
29
31
  type GetRandomValues = (bytes: Uint8Array) => void;
30
32
  /**
31
33
  * Inject a platform-specific `getRandomValues` implementation.
@@ -62,4 +64,4 @@ declare const ONTOLOGY_BACKFILL_SYSTEM_PROMPT = "You are a knowledge classificat
62
64
 
63
65
  declare function createWiki(db: SQLiteAdapter, options: WikiOptions): WikiMemory;
64
66
 
65
- export { DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, FormatContextOptions, FormattedMemoryDump, GraphNeighborhood, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, type OkfImportOptions, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, validateLibrarianPromptTemplate };
67
+ export { DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, FormatContextOptions, FormattedMemoryDump, GraphNeighborhood, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, type OkfImportOptions, OntologyManifest, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, validateLibrarianPromptTemplate, validateManifest };
package/dist/index.d.ts CHANGED
@@ -1,5 +1,5 @@
1
- import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-i91HR1TG.js';
2
- export { E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HOOK_TIMEOUT_MARKER, L as LLMProvider, O as ONTOLOGY_BACKFILL_BATCH_SIZE, i as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, j as ONTOLOGY_BACKFILL_RECHECK_MS, k as OntologyBackfillResult, l as OntologyConfig, m as OntologyEdgeType, n as OntologyManifest, o as OntologyMode, p as OntologyNodeType, q as OntologyPromptContext, r as OntologyUpdates, P as PromptOverrides, s as PromptService, t as PrunePartialFailureError, V as VectorRanker, u as VectorRankerFallback, v as VectorRankerRankArgs, w as VectorRankerSemanticResult, x as WikiBusyError, y as WikiBusyOperation, z as WikiCheckpoint, A as WikiConfig, B as WikiEdge, C as WikiEvent, D as WikiFact, I as WikiMemoryTestAccess, J as WikiOutboxEvent, K as WikiTask, N as WikiTransactionError } from './testing-i91HR1TG.js';
1
+ import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, O as OntologyManifest, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-CBjAuTSl.js';
2
+ export { E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HOOK_TIMEOUT_MARKER, L as LLMProvider, i as ONTOLOGY_BACKFILL_BATCH_SIZE, j as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, k as ONTOLOGY_BACKFILL_RECHECK_MS, l as OntologyBackfillResult, m as OntologyConfig, n as OntologyEdgeType, o as OntologyMode, p as OntologyNodeType, q as OntologyPromptContext, r as OntologyUpdates, P as PromptOverrides, s as PromptService, t as PrunePartialFailureError, V as VectorRanker, u as VectorRankerFallback, v as VectorRankerRankArgs, w as VectorRankerSemanticResult, x as WikiBusyError, y as WikiBusyOperation, z as WikiCheckpoint, A as WikiConfig, B as WikiEdge, C as WikiEvent, D as WikiFact, I as WikiMemoryTestAccess, J as WikiOutboxEvent, K as WikiTask, N as WikiTransactionError } from './testing-CBjAuTSl.js';
3
3
  import { OkfFile } from '@equationalapplications/core-okf';
4
4
  import 'minisearch';
5
5
 
@@ -26,6 +26,8 @@ declare function parseOkfBundle(entityId: string, files: OkfFile[], options?: Ok
26
26
 
27
27
  declare function parseEmbedding(blob: Uint8Array | null | undefined, text: string | null | undefined): Float32Array | null;
28
28
 
29
+ declare function validateManifest(manifest: OntologyManifest): void;
30
+
29
31
  type GetRandomValues = (bytes: Uint8Array) => void;
30
32
  /**
31
33
  * Inject a platform-specific `getRandomValues` implementation.
@@ -62,4 +64,4 @@ declare const ONTOLOGY_BACKFILL_SYSTEM_PROMPT = "You are a knowledge classificat
62
64
 
63
65
  declare function createWiki(db: SQLiteAdapter, options: WikiOptions): WikiMemory;
64
66
 
65
- export { DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, FormatContextOptions, FormattedMemoryDump, GraphNeighborhood, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, type OkfImportOptions, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, validateLibrarianPromptTemplate };
67
+ export { DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, FormatContextOptions, FormattedMemoryDump, GraphNeighborhood, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, type OkfImportOptions, OntologyManifest, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, validateLibrarianPromptTemplate, validateManifest };
package/dist/index.js CHANGED
@@ -707,6 +707,45 @@ var EntryRepository = class extends BaseRepository {
707
707
  );
708
708
  return rows.map(mapRowToFact);
709
709
  }
710
+ /**
711
+ * Fetch live, mutable entries for an entity — everything heal is allowed to
712
+ * downgrade or delete. Heal previously loaded every row via
713
+ * findAllByEntityId and filtered in JS, which on a document-heavy corpus
714
+ * meant loading 2560 rows to keep 31.
715
+ */
716
+ async findHealCandidatesByEntityId(entityId, tx) {
717
+ const executor = this.getExecutor(tx);
718
+ const rows = await executor.getAllAsync(
719
+ `SELECT * FROM ${this.prefix}entries
720
+ WHERE entity_id = ? AND deleted_at IS NULL AND source_type != 'immutable_document'
721
+ ORDER BY updated_at DESC`,
722
+ [entityId]
723
+ );
724
+ return rows.map(mapRowToFact);
725
+ }
726
+ /**
727
+ * Resolve search hits to document anchors. The MiniSearch index holds every
728
+ * fact, not only immutable_document rows, so the source-type restriction has
729
+ * to be applied here, after retrieval.
730
+ */
731
+ async findAnchorRowsByIds(entityId, ids, tx) {
732
+ if (ids.length === 0) return [];
733
+ const executor = this.getExecutor(tx);
734
+ const rows = [];
735
+ for (let i = 0; i < ids.length; i += this.chunkSize) {
736
+ const chunk = ids.slice(i, i + this.chunkSize);
737
+ const placeholders = chunk.map(() => "?").join(", ");
738
+ const chunkRows = await executor.getAllAsync(
739
+ `SELECT id, title, source_ref FROM ${this.prefix}entries
740
+ WHERE entity_id = ? AND deleted_at IS NULL
741
+ AND source_type = 'immutable_document'
742
+ AND id IN (${placeholders})`,
743
+ [entityId, ...chunk]
744
+ );
745
+ rows.push(...chunkRows);
746
+ }
747
+ return rows;
748
+ }
710
749
  /**
711
750
  * Fetch recent non-deleted entries for an entity (limited), ordered by updated_at DESC.
712
751
  * Used by MaintenanceService.doRunLibrarian().
@@ -1781,10 +1820,13 @@ function resolveNodeType(raw, manifest) {
1781
1820
  const hit = manifest.node_types.find((n) => n.type.toLowerCase() === slug.toLowerCase());
1782
1821
  return hit?.type ?? null;
1783
1822
  }
1784
- function resolveEdgeDefinition(rawEdgeType, manifest) {
1823
+ function resolveEdgeDefinitions(rawEdgeType, manifest) {
1785
1824
  const slug = rawEdgeType.trim();
1786
- if (!slug) return null;
1787
- return manifest.edge_types.find((e) => e.type.toLowerCase() === slug.toLowerCase()) ?? null;
1825
+ if (!slug) return [];
1826
+ return manifest.edge_types.filter((e) => e.type.toLowerCase() === slug.toLowerCase());
1827
+ }
1828
+ function edgeTripleKey(type, sourceType, targetType) {
1829
+ return `${type.trim().toLowerCase()}|${sourceType.trim().toLowerCase()}|${targetType.trim().toLowerCase()}`;
1788
1830
  }
1789
1831
  function validateManifest(manifest) {
1790
1832
  const nodeSlugs = /* @__PURE__ */ new Set();
@@ -1795,25 +1837,35 @@ function validateManifest(manifest) {
1795
1837
  if (nodeSlugs.has(key)) throw new Error(`Duplicate node type: ${type}`);
1796
1838
  nodeSlugs.add(key);
1797
1839
  }
1798
- const edgeSlugs = /* @__PURE__ */ new Set();
1840
+ const edgeKeys = /* @__PURE__ */ new Set();
1841
+ const edgeNames = /* @__PURE__ */ new Map();
1799
1842
  for (const edge of manifest.edge_types ?? []) {
1800
1843
  const edgeType = edge.type?.trim();
1801
1844
  const sourceType = edge.source_type?.trim();
1802
1845
  const targetType = edge.target_type?.trim();
1803
1846
  if (!edgeType) throw new Error("Ontology edge type slug must be non-empty");
1804
- const edgeKey = edgeType.toLowerCase();
1805
- if (edgeSlugs.has(edgeKey)) throw new Error(`Duplicate edge type: ${edgeType}`);
1806
- edgeSlugs.add(edgeKey);
1807
1847
  if (!sourceType || !targetType || !nodeSlugs.has(sourceType.toLowerCase()) || !nodeSlugs.has(targetType.toLowerCase())) {
1808
1848
  throw new Error(`Edge type ${edgeType} references unknown node type`);
1809
1849
  }
1850
+ const edgeKey = edgeTripleKey(edgeType, sourceType, targetType);
1851
+ if (edgeKeys.has(edgeKey)) {
1852
+ throw new Error(`Duplicate edge definition: ${edgeType} (${sourceType} \u2192 ${targetType})`);
1853
+ }
1854
+ edgeKeys.add(edgeKey);
1855
+ const canonical = edgeNames.get(edgeType.toLowerCase());
1856
+ if (canonical === void 0) {
1857
+ edgeNames.set(edgeType.toLowerCase(), edgeType);
1858
+ } else if (canonical !== edgeType) {
1859
+ throw new Error(`Inconsistent casing for edge type: ${edgeType} conflicts with ${canonical}`);
1860
+ }
1810
1861
  }
1811
1862
  }
1812
1863
  function mergeOntologyUpdates(current, updates) {
1813
1864
  const node_types = [...current.node_types];
1814
1865
  const edge_types = [...current.edge_types];
1815
1866
  const nodeSlugs = new Set(node_types.map((n) => n.type.trim().toLowerCase()));
1816
- const edgeSlugs = new Set(edge_types.map((e) => e.type.trim().toLowerCase()));
1867
+ const edgeKeys = new Set(edge_types.map((e) => edgeTripleKey(e.type, e.source_type, e.target_type)));
1868
+ const edgeNames = new Map(edge_types.map((e) => [e.type.trim().toLowerCase(), e.type.trim()]));
1817
1869
  for (const node of updates.node_types ?? []) {
1818
1870
  const type = node?.type?.trim();
1819
1871
  if (!type) continue;
@@ -1823,20 +1875,22 @@ function mergeOntologyUpdates(current, updates) {
1823
1875
  nodeSlugs.add(key);
1824
1876
  }
1825
1877
  for (const edge of updates.edge_types ?? []) {
1826
- const edgeType = edge?.type?.trim();
1878
+ const rawEdgeType = edge?.type?.trim();
1827
1879
  const sourceType = edge?.source_type?.trim();
1828
1880
  const targetType = edge?.target_type?.trim();
1829
- if (!edgeType || !sourceType || !targetType) continue;
1830
- const edgeKey = edgeType.toLowerCase();
1831
- if (edgeSlugs.has(edgeKey)) continue;
1881
+ if (!rawEdgeType || !sourceType || !targetType) continue;
1882
+ const edgeType = edgeNames.get(rawEdgeType.toLowerCase()) ?? rawEdgeType;
1883
+ const edgeKey = edgeTripleKey(edgeType, sourceType, targetType);
1884
+ if (edgeKeys.has(edgeKey)) continue;
1832
1885
  if (!nodeSlugs.has(sourceType.toLowerCase()) || !nodeSlugs.has(targetType.toLowerCase())) continue;
1886
+ edgeNames.set(edgeType.toLowerCase(), edgeType);
1833
1887
  edge_types.push({
1834
1888
  type: edgeType,
1835
1889
  source_type: sourceType,
1836
1890
  target_type: targetType,
1837
1891
  description: String(edge.description ?? "")
1838
1892
  });
1839
- edgeSlugs.add(edgeKey);
1893
+ edgeKeys.add(edgeKey);
1840
1894
  }
1841
1895
  return { node_types, edge_types };
1842
1896
  }
@@ -1845,10 +1899,10 @@ function validateInlineEdges(sourceType, _targetType, edges, manifest) {
1845
1899
  const valid = [];
1846
1900
  for (const edge of edges) {
1847
1901
  if (typeof edge?.edge_type !== "string" || typeof edge?.target_title !== "string") continue;
1848
- const def = resolveEdgeDefinition(edge.edge_type, manifest);
1849
- if (!def) continue;
1850
- if (def.source_type.toLowerCase() !== sourceType.toLowerCase()) continue;
1851
- valid.push({ edge_type: def.type, target_title: edge.target_title });
1902
+ const defs = resolveEdgeDefinitions(edge.edge_type, manifest);
1903
+ const match = defs.find((d) => d.source_type.toLowerCase() === sourceType.toLowerCase());
1904
+ if (!match) continue;
1905
+ valid.push({ edge_type: match.type, target_title: edge.target_title });
1852
1906
  }
1853
1907
  return valid;
1854
1908
  }
@@ -2048,9 +2102,23 @@ var _SearchService = class _SearchService {
2048
2102
  this.entryRepo = entryRepo;
2049
2103
  this.miniSearchEntryIdsByEntity = /* @__PURE__ */ new Map();
2050
2104
  this.vectorCache = /* @__PURE__ */ new Map();
2105
+ /**
2106
+ * Serializes rebuilds. `rebuildIndex` awaits a repository read between
2107
+ * snapshotting the previous id set and discarding it, so two concurrent
2108
+ * sync() calls for one entity can interleave: a slow, stale read lands last
2109
+ * and discards documents the fresh read just added. Chaining also keeps
2110
+ * discard()/addAll() out of each other's way, which is what accrued the
2111
+ * auto-vacuum debt behind the TypeError in #64.
2112
+ */
2113
+ this.syncChain = Promise.resolve();
2051
2114
  this.miniSearch = new MiniSearch__default.default({
2052
2115
  fields: ["title", "body", "tags"],
2053
2116
  storeFields: ["entity_id"],
2117
+ // Vacuuming is driven explicitly at the end of each serialized rebuild
2118
+ // (see sync). Auto-vacuum fires on its own schedule, asynchronously with
2119
+ // respect to the caller, and traversing the tree mid-rebuild is what
2120
+ // threw the uncaught TypeError in MiniSearch.performVacuuming (#64).
2121
+ autoVacuum: false,
2054
2122
  searchOptions: {
2055
2123
  boost: { title: 2 },
2056
2124
  fuzzy: 0.2,
@@ -2061,10 +2129,26 @@ var _SearchService = class _SearchService {
2061
2129
  /**
2062
2130
  * Rebuilds the search index and clears the vector cache for a given entity.
2063
2131
  * A direct replacement for manually syncing state after a DB transaction.
2132
+ *
2133
+ * Rebuilds are serialized per instance and never reject: the MiniSearch index
2134
+ * is a rebuildable cache over SQLite, so degraded keyword search is the
2135
+ * correct failure mode and killing the host process is not.
2064
2136
  */
2065
2137
  async sync(entityId) {
2066
- await this.rebuildIndex(entityId);
2067
- this.evictCache(entityId);
2138
+ const work = this.syncChain.then(async () => {
2139
+ try {
2140
+ try {
2141
+ await this.rebuildIndex(entityId);
2142
+ await this.miniSearch.vacuum();
2143
+ } finally {
2144
+ this.evictCache(entityId);
2145
+ }
2146
+ } catch (err) {
2147
+ console.warn(`[WikiMemory] search index rebuild failed for ${entityId ?? "*"}:`, err);
2148
+ }
2149
+ });
2150
+ this.syncChain = work;
2151
+ return work;
2068
2152
  }
2069
2153
  /**
2070
2154
  * Clears the parsed vector cache. Useful for mid-loop flush guarantees
@@ -3052,12 +3136,122 @@ var IngestionService = class {
3052
3136
  }
3053
3137
  };
3054
3138
 
3139
+ // src/services/BoundedLlmCall.ts
3140
+ var DEFAULT_BATCH_SIZE = 10;
3141
+ var ESTIMATED_OUTPUT_TOKENS_PER_ITEM = 150;
3142
+ var OUTPUT_BUDGET_FRACTION = 0.8;
3143
+ var TRUNCATION_PATTERNS = [
3144
+ /truncat/i,
3145
+ /token limit/i,
3146
+ /max(imum)?[ _-]?tokens?/i,
3147
+ /output limit/i,
3148
+ /length limit/i,
3149
+ /finish[_ ]?reason/i
3150
+ ];
3151
+ var EXCEEDS_LIMIT_PATTERN = /exceed[a-z]*[^.]{0,40}\b(model|context)?[ _-]?limit/i;
3152
+ function isTruncationError(err) {
3153
+ const message = err instanceof Error ? err.message : String(err ?? "");
3154
+ if (EXCEEDS_LIMIT_PATTERN.test(message)) return false;
3155
+ return TRUNCATION_PATTERNS.some((pattern) => pattern.test(message));
3156
+ }
3157
+ function initialBatchSize(maxOutputTokens) {
3158
+ if (!maxOutputTokens || !Number.isFinite(maxOutputTokens) || maxOutputTokens <= 0) {
3159
+ return DEFAULT_BATCH_SIZE;
3160
+ }
3161
+ const estimate = Math.floor(
3162
+ maxOutputTokens * OUTPUT_BUDGET_FRACTION / ESTIMATED_OUTPUT_TOKENS_PER_ITEM
3163
+ );
3164
+ return Math.max(DEFAULT_BATCH_SIZE, estimate);
3165
+ }
3166
+ var promptLength = (prompts) => prompts.systemPrompt.length + prompts.userPrompt.length;
3167
+ async function runBatched(args) {
3168
+ const { items, buildPrompt, call, parse, maxPromptChars, maxOutputTokens, onSkip } = args;
3169
+ const results = [];
3170
+ const skipped = [];
3171
+ let batches = 0;
3172
+ let batchSize = initialBatchSize(maxOutputTokens);
3173
+ const trim = async (candidate) => {
3174
+ const whole = await buildPrompt(candidate);
3175
+ if (candidate.length <= 1 || promptLength(whole) <= maxPromptChars) {
3176
+ return { batch: candidate, prompts: whole };
3177
+ }
3178
+ let low = 2;
3179
+ let high = candidate.length - 1;
3180
+ let best;
3181
+ let bestPrompts;
3182
+ while (low <= high) {
3183
+ const mid = Math.floor((low + high) / 2);
3184
+ const batch = candidate.slice(0, mid);
3185
+ const prompts = await buildPrompt(batch);
3186
+ if (promptLength(prompts) <= maxPromptChars) {
3187
+ best = batch;
3188
+ bestPrompts = prompts;
3189
+ low = mid + 1;
3190
+ } else {
3191
+ high = mid - 1;
3192
+ }
3193
+ }
3194
+ if (best && bestPrompts) return { batch: best, prompts: bestPrompts };
3195
+ const single = candidate.slice(0, 1);
3196
+ return { batch: single, prompts: await buildPrompt(single) };
3197
+ };
3198
+ const onFailure = async (batch, err) => {
3199
+ if (batch.length <= 1) {
3200
+ if (batch.length === 1) {
3201
+ skipped.push(batch[0]);
3202
+ onSkip?.(batch[0], err);
3203
+ }
3204
+ return;
3205
+ }
3206
+ const mid = Math.ceil(batch.length / 2);
3207
+ if (mid < batchSize) batchSize = mid;
3208
+ let i = 0;
3209
+ while (i < batch.length) {
3210
+ const size = Math.min(batchSize, batch.length - i);
3211
+ const trimmed = await trim(batch.slice(i, i + size));
3212
+ await attempt(trimmed.batch, trimmed.prompts);
3213
+ i += trimmed.batch.length;
3214
+ }
3215
+ };
3216
+ const attempt = async (batch, prebuilt) => {
3217
+ if (batch.length === 0) return;
3218
+ const prompts = prebuilt ?? await buildPrompt(batch);
3219
+ batches++;
3220
+ let responseText;
3221
+ try {
3222
+ responseText = await call(prompts);
3223
+ } catch (err) {
3224
+ if (!isTruncationError(err)) throw err;
3225
+ await onFailure(batch, err);
3226
+ return;
3227
+ }
3228
+ let result;
3229
+ try {
3230
+ result = parse(responseText, batch);
3231
+ } catch (err) {
3232
+ await onFailure(batch, err);
3233
+ return;
3234
+ }
3235
+ results.push(result);
3236
+ };
3237
+ let index = 0;
3238
+ while (index < items.length) {
3239
+ const { batch, prompts } = await trim(items.slice(index, index + batchSize));
3240
+ index += batch.length;
3241
+ await attempt(batch, prompts);
3242
+ }
3243
+ return { results, skipped, batches };
3244
+ }
3245
+
3055
3246
  // src/services/MaintenanceService.ts
3056
3247
  var FUZZY_THRESHOLD = 0.5;
3057
3248
  var MIN_TOKENS_TO_QUALIFY = 3;
3058
3249
  var ONTOLOGY_BACKFILL_BATCH_SIZE = 25;
3059
3250
  var ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS = 4e4;
3060
3251
  var ONTOLOGY_BACKFILL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
3252
+ var HEAL_MAX_ANCHORS = 50;
3253
+ var HEAL_ANCHOR_SEARCH_OVERFETCH = 4;
3254
+ var HEAL_MAX_PROMPT_CHARS = 4e4;
3061
3255
  var MaintenanceService = class {
3062
3256
  constructor(db, prefix, options, entryRepo, taskRepo, eventRepo, metadataRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
3063
3257
  this.db = db;
@@ -3440,30 +3634,56 @@ var MaintenanceService = class {
3440
3634
  console.warn(`[WikiMemory] onEmbeddingPersisted hook failed during heal orphan pass for ${factId}:`, hookErr);
3441
3635
  }
3442
3636
  }
3443
- const allFactsRows = await this.entryRepo.findAllByEntityId(entityId);
3637
+ const healCandidates = await this.entryRepo.findHealCandidatesByEntityId(entityId);
3444
3638
  const allTasks = await this.taskRepo.findAllPending([entityId]);
3445
3639
  const recentEvents = await this.eventRepo.getRecent(entityId, 20);
3446
- const healCandidates = allFactsRows.filter((f) => f.source_type !== "immutable_document");
3447
- const documentAnchors = allFactsRows.filter((f) => f.source_type === "immutable_document").map(({ id, title, source_ref }) => ({ id, title, source_ref }));
3448
- const healCandidatesForPrompt = healCandidates.map((f) => {
3640
+ const toPromptShape = (f) => {
3449
3641
  const { embedding: _embedding, embedding_blob: _blob, ...rest } = f;
3450
3642
  return { ...rest, tags: typeof rest.tags === "string" ? JSON.parse(rest.tags) : rest.tags };
3643
+ };
3644
+ const anchorCache = /* @__PURE__ */ new Map();
3645
+ const outcome = await runBatched({
3646
+ items: healCandidates,
3647
+ buildPrompt: async (batch) => {
3648
+ const documentAnchors = await this._selectHealAnchors(entityId, batch, anchorCache);
3649
+ return this.promptService.buildHealPrompt(
3650
+ batch.map(toPromptShape),
3651
+ documentAnchors,
3652
+ allTasks,
3653
+ recentEvents,
3654
+ promptOverride
3655
+ );
3656
+ },
3657
+ call: (prompts) => this.options.llmProvider.generateText(prompts),
3658
+ parse: (responseText, batch) => {
3659
+ const result = parseJsonResponse(responseText);
3660
+ return {
3661
+ batch,
3662
+ downgraded: Array.isArray(result.downgraded) ? result.downgraded : [],
3663
+ deleted: Array.isArray(result.deleted) ? result.deleted : [],
3664
+ newFacts: Array.isArray(result.newFacts) ? result.newFacts : []
3665
+ };
3666
+ },
3667
+ maxOutputTokens: this.options.llmProvider.maxOutputTokens,
3668
+ maxPromptChars: HEAL_MAX_PROMPT_CHARS,
3669
+ onSkip: (fact, err) => {
3670
+ console.warn(
3671
+ `[WikiMemory] heal skipped ${entityId}/${fact.id}: response could not be bounded`,
3672
+ err
3673
+ );
3674
+ }
3451
3675
  });
3452
- const { systemPrompt, userPrompt } = this.promptService.buildHealPrompt(
3453
- healCandidatesForPrompt,
3454
- documentAnchors,
3455
- allTasks,
3456
- recentEvents,
3457
- promptOverride
3458
- );
3459
- const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
3460
- const result = parseJsonResponse(responseText);
3461
- const mutableIds = new Set(healCandidates.map((f) => f.id));
3462
- const downgraded = Array.isArray(result.downgraded) ? result.downgraded : [];
3463
- const deleted = Array.isArray(result.deleted) ? result.deleted : [];
3464
- const newFacts = Array.isArray(result.newFacts) ? result.newFacts : [];
3465
- const safeDowngraded = Array.from(new Set(downgraded.filter((id) => mutableIds.has(id))));
3466
- const safeDeleted = Array.from(new Set(deleted.filter((id) => mutableIds.has(id))));
3676
+ const safeDowngradedSet = /* @__PURE__ */ new Set();
3677
+ const safeDeletedSet = /* @__PURE__ */ new Set();
3678
+ const newFacts = [];
3679
+ for (const batchResult of outcome.results) {
3680
+ const mutableIds = new Set(batchResult.batch.map((f) => f.id));
3681
+ for (const id of batchResult.downgraded) if (mutableIds.has(id)) safeDowngradedSet.add(id);
3682
+ for (const id of batchResult.deleted) if (mutableIds.has(id)) safeDeletedSet.add(id);
3683
+ newFacts.push(...batchResult.newFacts);
3684
+ }
3685
+ const safeDowngraded = Array.from(safeDowngradedSet);
3686
+ const safeDeleted = Array.from(safeDeletedSet);
3467
3687
  const validNewFacts = newFacts.map(validateFact).filter((f) => f !== null);
3468
3688
  const insertedFacts = [];
3469
3689
  const uniqueDeletedFactIds = Array.from(new Set(safeDeleted));
@@ -3530,7 +3750,7 @@ var MaintenanceService = class {
3530
3750
  }
3531
3751
  const now = Date.now();
3532
3752
  const recheckCutoff = now - ONTOLOGY_BACKFILL_RECHECK_MS;
3533
- const zeroed = { scanned: 0, typed: 0, failedValidation: 0, edgesAdded: 0 };
3753
+ const zeroed = { scanned: 0, typed: 0, failedValidation: 0, edgesAdded: 0, skipped: 0 };
3534
3754
  const ontologyService = this.ontologyService;
3535
3755
  if (!ontologyService) {
3536
3756
  return { ...zeroed, remaining: 0, deferred: 0 };
@@ -3552,18 +3772,79 @@ var MaintenanceService = class {
3552
3772
  options?.promptOverride,
3553
3773
  ontologyContext
3554
3774
  );
3555
- const batch = [candidates[0]];
3556
- let built = buildPrompt(batch);
3557
- for (let i = 1; i < candidates.length; i++) {
3558
- const next = buildPrompt([...batch, candidates[i]]);
3559
- if (next.systemPrompt.length + next.userPrompt.length > ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS) break;
3560
- batch.push(candidates[i]);
3561
- built = next;
3562
- }
3563
- const { systemPrompt, userPrompt } = built;
3564
- const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
3565
- const parsed = parseJsonResponse(responseText);
3566
- const classifications = Array.isArray(parsed.classifications) ? parsed.classifications : [];
3775
+ const outcome = await runBatched({
3776
+ items: candidates,
3777
+ buildPrompt,
3778
+ call: (prompts) => this.options.llmProvider.generateText(prompts),
3779
+ parse: (responseText, batch) => {
3780
+ const parsed = parseJsonResponse(responseText);
3781
+ return {
3782
+ batch,
3783
+ classifications: Array.isArray(parsed.classifications) ? parsed.classifications : [],
3784
+ ontologyUpdates: parsed.ontology_updates
3785
+ };
3786
+ },
3787
+ maxOutputTokens: this.options.llmProvider.maxOutputTokens,
3788
+ maxPromptChars: ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS,
3789
+ onSkip: (fact, err) => {
3790
+ console.warn(
3791
+ `[WikiMemory] ontology backfill skipped ${entityId}/${fact.id}: response could not be bounded`,
3792
+ err
3793
+ );
3794
+ }
3795
+ });
3796
+ let typed = 0;
3797
+ let failedValidation = 0;
3798
+ let edgesAdded = 0;
3799
+ let scanned = 0;
3800
+ let abortedOntologyOff = false;
3801
+ for (const batchResult of outcome.results) {
3802
+ const applied = await this._applyOntologyBackfillBatch(entityId, batchResult, now);
3803
+ if (applied.abortedOntologyOff) {
3804
+ abortedOntologyOff = true;
3805
+ break;
3806
+ }
3807
+ typed += applied.typed;
3808
+ failedValidation += applied.failedValidation;
3809
+ edgesAdded += applied.edgesAdded;
3810
+ scanned += batchResult.batch.length;
3811
+ }
3812
+ if (abortedOntologyOff) {
3813
+ const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
3814
+ return {
3815
+ scanned,
3816
+ typed,
3817
+ failedValidation,
3818
+ edgesAdded,
3819
+ skipped: outcome.skipped.length,
3820
+ remaining: 0,
3821
+ deferred: counts2.deferred
3822
+ };
3823
+ }
3824
+ if (outcome.skipped.length > 0) {
3825
+ await this.entryRepo.markOntologyChecked(outcome.skipped.map((f) => f.id), entityId, now, this.db);
3826
+ }
3827
+ this.searchService.evictCache(entityId);
3828
+ const counts = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
3829
+ return {
3830
+ scanned,
3831
+ typed,
3832
+ failedValidation,
3833
+ edgesAdded,
3834
+ skipped: outcome.skipped.length,
3835
+ remaining: counts.eligible,
3836
+ deferred: counts.deferred
3837
+ };
3838
+ }
3839
+ /**
3840
+ * Applies one parsed backfill batch in its own transaction. Per-batch rather
3841
+ * than one transaction for the pass, so mergeEmergentUpdates semantics and
3842
+ * the mid-flight `mode === 'off'` abort check keep the shape they had when a
3843
+ * pass was a single call.
3844
+ */
3845
+ async _applyOntologyBackfillBatch(entityId, batchResult, now) {
3846
+ const ontologyService = this.ontologyService;
3847
+ const { batch, classifications, ontologyUpdates } = batchResult;
3567
3848
  let typed = 0;
3568
3849
  let failedValidation = 0;
3569
3850
  let edgesAdded = 0;
@@ -3574,8 +3855,8 @@ var MaintenanceService = class {
3574
3855
  abortedOntologyOff = true;
3575
3856
  return;
3576
3857
  }
3577
- if (txMode === "emergent" && parsed.ontology_updates) {
3578
- manifest = await ontologyService.mergeEmergentUpdates(entityId, parsed.ontology_updates, tx);
3858
+ if (txMode === "emergent" && ontologyUpdates) {
3859
+ manifest = await ontologyService.mergeEmergentUpdates(entityId, ontologyUpdates, tx);
3579
3860
  }
3580
3861
  const titleRows = await this.entryRepo.findTitleIndexByEntityId(entityId, tx);
3581
3862
  const titleIndex = /* @__PURE__ */ new Map();
@@ -3629,19 +3910,58 @@ var MaintenanceService = class {
3629
3910
  }
3630
3911
  await this.entryRepo.markOntologyChecked(batch.map((f) => f.id), entityId, now, tx);
3631
3912
  });
3632
- if (abortedOntologyOff) {
3633
- const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
3634
- return { ...zeroed, remaining: 0, deferred: counts2.deferred };
3635
- }
3636
- this.searchService.evictCache(entityId);
3637
- const counts = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
3638
- return { scanned: batch.length, typed, failedValidation, edgesAdded, remaining: counts.eligible, deferred: counts.deferred };
3913
+ return { typed, failedValidation, edgesAdded, abortedOntologyOff };
3639
3914
  }
3640
3915
  _validatePruneDuration(value, name) {
3641
3916
  if (value !== null && value !== void 0 && (typeof value !== "number" || !isFinite(value) || value < 0)) {
3642
3917
  throw new Error(`Invalid ${name}: must be a non-negative finite number or null`);
3643
3918
  }
3644
3919
  }
3920
+ /**
3921
+ * Anchors relevant to one batch of heal candidates.
3922
+ *
3923
+ * Heal used to pass every immutable_document fact for the entity — 2560 rows
3924
+ * against 31 candidates on the corpus behind #63 — which is what blew the
3925
+ * output ceiling. Anchors are now retrieved by keyword relevance to the batch
3926
+ * and capped.
3927
+ *
3928
+ * The MiniSearch index holds all facts, not only anchors, so hits are
3929
+ * overfetched and the source_type restriction is applied after retrieval, in
3930
+ * SQL. Search rank order is preserved through the filter.
3931
+ *
3932
+ * Accepted tradeoff: an anchor that contradicts a candidate while sharing no
3933
+ * vocabulary with it is now missed. Exhaustive-but-broken traded for
3934
+ * relevance-scoped-and-working.
3935
+ *
3936
+ * `cache` is keyed by the derived query rather than by the batch, so two
3937
+ * batches that reduce to the same query share one lookup. Caller-owned and
3938
+ * per-pass — see the call site in doRunHeal.
3939
+ */
3940
+ async _selectHealAnchors(entityId, batch, cache) {
3941
+ const query = batch.map((f) => f.title).join(" ").trim();
3942
+ if (!query) return [];
3943
+ const cached = cache?.get(query);
3944
+ if (cached) return cached;
3945
+ const hits = this.searchService.searchKeyword(
3946
+ query,
3947
+ [entityId],
3948
+ HEAL_MAX_ANCHORS * HEAL_ANCHOR_SEARCH_OVERFETCH
3949
+ );
3950
+ const hitIds = hits.map((h) => h.id);
3951
+ const anchors = [];
3952
+ if (hitIds.length > 0) {
3953
+ const rows = await this.entryRepo.findAnchorRowsByIds(entityId, hitIds);
3954
+ const byId = new Map(rows.map((r) => [r.id, r]));
3955
+ for (const id of hitIds) {
3956
+ const row = byId.get(id);
3957
+ if (!row) continue;
3958
+ anchors.push(row);
3959
+ if (anchors.length >= HEAL_MAX_ANCHORS) break;
3960
+ }
3961
+ }
3962
+ cache?.set(query, anchors);
3963
+ return anchors;
3964
+ }
3645
3965
  _sanitizeRankerError(err) {
3646
3966
  return sanitizeRankerError(err, this.options.sanitizeRankerErrors);
3647
3967
  }
@@ -4854,7 +5174,8 @@ var WriteService = class {
4854
5174
  var FACT_ONTOLOGY_FIELDS = `
4855
5175
  Each fact may optionally include:
4856
5176
  - "okf_type": string \u2014 must be one of the node_types in the manifest
4857
- - "edges": [{ "edge_type": string, "target_title": string }] \u2014 target_title must match another fact's title in this response or existing memory`;
5177
+ - "edges": [{ "edge_type": string, "target_title": string }] \u2014 target_title must match another fact's title in this response or existing memory
5178
+ - An edge_type may appear in the manifest multiple times with different source_type/target_type; use the row whose types match your fact and target.`;
4858
5179
  var EMERGENT_EXTRA = `
4859
5180
  You may also return "ontology_updates" to propose new types:
4860
5181
  "ontology_updates": {
@@ -4939,12 +5260,15 @@ var OntologyService = class {
4939
5260
  if (!sourceType || edges.length === 0) return 0;
4940
5261
  let persisted = 0;
4941
5262
  for (const edge of edges) {
4942
- const def = resolveEdgeDefinition(edge.edge_type, manifest);
4943
- if (!def || def.source_type.toLowerCase() !== sourceType.toLowerCase()) continue;
5263
+ const candidates = resolveEdgeDefinitions(edge.edge_type, manifest).filter((d) => d.source_type.toLowerCase() === sourceType.toLowerCase());
5264
+ if (candidates.length === 0) continue;
4944
5265
  const targetKey = normalizeTitleKey(edge.target_title);
4945
5266
  const target = titleIndex.get(targetKey);
4946
5267
  if (!target) continue;
4947
- if (def.target_type.toLowerCase() !== (target.okf_type ?? "").toLowerCase()) continue;
5268
+ const def = candidates.find(
5269
+ (d) => d.target_type.toLowerCase() === (target.okf_type ?? "").toLowerCase()
5270
+ );
5271
+ if (!def) continue;
4948
5272
  const wikiEdge = {
4949
5273
  id: generateId(),
4950
5274
  entity_id: entityId,
@@ -6073,5 +6397,6 @@ exports.mapLibrarianOptionsToReadOptions = mapLibrarianOptionsToReadOptions;
6073
6397
  exports.parseEmbedding = parseEmbedding;
6074
6398
  exports.parseOkfBundle = parseOkfBundle;
6075
6399
  exports.validateLibrarianPromptTemplate = validateLibrarianPromptTemplate;
6400
+ exports.validateManifest = validateManifest;
6076
6401
  //# sourceMappingURL=index.js.map
6077
6402
  //# sourceMappingURL=index.js.map