@equationalapplications/core-llm-wiki 4.22.0 → 4.23.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -6
- package/dist/{chunk-LCAU6NYC.mjs → chunk-MYZJLVX4.mjs} +311 -45
- package/dist/chunk-MYZJLVX4.mjs.map +1 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +348 -43
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +41 -2
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-Bk8J6QLX.d.mts → testing-CBjAuTSl.d.mts} +72 -0
- package/dist/{testing-Bk8J6QLX.d.ts → testing-CBjAuTSl.d.ts} +72 -0
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +309 -43
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +6 -6
- package/dist/chunk-LCAU6NYC.mjs.map +0 -1
|
@@ -96,6 +96,9 @@ interface OntologyBackfillResult {
|
|
|
96
96
|
remaining: number;
|
|
97
97
|
/** Untyped facts inside the recheck cooldown. */
|
|
98
98
|
deferred: number;
|
|
99
|
+
/** Facts a batch could not process even alone, skipped so the pass could finish.
|
|
100
|
+
* Stamped with the recheck cooldown, so they reappear as `deferred` next pass. */
|
|
101
|
+
skipped: number;
|
|
99
102
|
}
|
|
100
103
|
interface WikiConfig {
|
|
101
104
|
/**
|
|
@@ -303,6 +306,17 @@ interface LLMProvider {
|
|
|
303
306
|
* When absent or throws, `read()` falls back to MiniSearch.
|
|
304
307
|
*/
|
|
305
308
|
embed?: (text: string) => Promise<number[]>;
|
|
309
|
+
/**
|
|
310
|
+
* Optional. The provider's hard output-token ceiling for `generateText`.
|
|
311
|
+
* Core cannot observe it — the token budget lives entirely in the host's
|
|
312
|
+
* adapter — so maintenance passes that generate response-size-proportional
|
|
313
|
+
* output size their batches blind unless this is declared.
|
|
314
|
+
*
|
|
315
|
+
* When absent, batching falls back to a conservative default and adapts
|
|
316
|
+
* downward on failure. When present, it is used as a sizing hint only and is
|
|
317
|
+
* never trusted as a guarantee.
|
|
318
|
+
*/
|
|
319
|
+
maxOutputTokens?: number;
|
|
306
320
|
}
|
|
307
321
|
/**
|
|
308
322
|
* Result of semantic ranking for a single fact.
|
|
@@ -648,6 +662,23 @@ declare class EntryRepository extends BaseRepository {
|
|
|
648
662
|
* Used by _getFullBundle().
|
|
649
663
|
*/
|
|
650
664
|
findAllByEntityId(entityId: string, tx?: SQLiteAdapter): Promise<WikiFact[]>;
|
|
665
|
+
/**
|
|
666
|
+
* Fetch live, mutable entries for an entity — everything heal is allowed to
|
|
667
|
+
* downgrade or delete. Heal previously loaded every row via
|
|
668
|
+
* findAllByEntityId and filtered in JS, which on a document-heavy corpus
|
|
669
|
+
* meant loading 2560 rows to keep 31.
|
|
670
|
+
*/
|
|
671
|
+
findHealCandidatesByEntityId(entityId: string, tx?: SQLiteAdapter): Promise<WikiFact[]>;
|
|
672
|
+
/**
|
|
673
|
+
* Resolve search hits to document anchors. The MiniSearch index holds every
|
|
674
|
+
* fact, not only immutable_document rows, so the source-type restriction has
|
|
675
|
+
* to be applied here, after retrieval.
|
|
676
|
+
*/
|
|
677
|
+
findAnchorRowsByIds(entityId: string, ids: readonly string[], tx?: SQLiteAdapter): Promise<Array<{
|
|
678
|
+
id: string;
|
|
679
|
+
title: string;
|
|
680
|
+
source_ref: string | null;
|
|
681
|
+
}>>;
|
|
651
682
|
/**
|
|
652
683
|
* Fetch recent non-deleted entries for an entity (limited), ordered by updated_at DESC.
|
|
653
684
|
* Used by MaintenanceService.doRunLibrarian().
|
|
@@ -845,10 +876,23 @@ declare class SearchService {
|
|
|
845
876
|
private miniSearch;
|
|
846
877
|
private miniSearchEntryIdsByEntity;
|
|
847
878
|
private vectorCache;
|
|
879
|
+
/**
|
|
880
|
+
* Serializes rebuilds. `rebuildIndex` awaits a repository read between
|
|
881
|
+
* snapshotting the previous id set and discarding it, so two concurrent
|
|
882
|
+
* sync() calls for one entity can interleave: a slow, stale read lands last
|
|
883
|
+
* and discards documents the fresh read just added. Chaining also keeps
|
|
884
|
+
* discard()/addAll() out of each other's way, which is what accrued the
|
|
885
|
+
* auto-vacuum debt behind the TypeError in #64.
|
|
886
|
+
*/
|
|
887
|
+
private syncChain;
|
|
848
888
|
constructor(entryRepo: EntryRepository);
|
|
849
889
|
/**
|
|
850
890
|
* Rebuilds the search index and clears the vector cache for a given entity.
|
|
851
891
|
* A direct replacement for manually syncing state after a DB transaction.
|
|
892
|
+
*
|
|
893
|
+
* Rebuilds are serialized per instance and never reject: the MiniSearch index
|
|
894
|
+
* is a rebuildable cache over SQLite, so degraded keyword search is the
|
|
895
|
+
* correct failure mode and killing the host process is not.
|
|
852
896
|
*/
|
|
853
897
|
sync(entityId?: string): Promise<void>;
|
|
854
898
|
/**
|
|
@@ -1221,7 +1265,35 @@ declare class MaintenanceService {
|
|
|
1221
1265
|
promptOverride?: string;
|
|
1222
1266
|
batchSize?: number;
|
|
1223
1267
|
}): Promise<OntologyBackfillResult>;
|
|
1268
|
+
/**
|
|
1269
|
+
* Applies one parsed backfill batch in its own transaction. Per-batch rather
|
|
1270
|
+
* than one transaction for the pass, so mergeEmergentUpdates semantics and
|
|
1271
|
+
* the mid-flight `mode === 'off'` abort check keep the shape they had when a
|
|
1272
|
+
* pass was a single call.
|
|
1273
|
+
*/
|
|
1274
|
+
private _applyOntologyBackfillBatch;
|
|
1224
1275
|
private _validatePruneDuration;
|
|
1276
|
+
/**
|
|
1277
|
+
* Anchors relevant to one batch of heal candidates.
|
|
1278
|
+
*
|
|
1279
|
+
* Heal used to pass every immutable_document fact for the entity — 2560 rows
|
|
1280
|
+
* against 31 candidates on the corpus behind #63 — which is what blew the
|
|
1281
|
+
* output ceiling. Anchors are now retrieved by keyword relevance to the batch
|
|
1282
|
+
* and capped.
|
|
1283
|
+
*
|
|
1284
|
+
* The MiniSearch index holds all facts, not only anchors, so hits are
|
|
1285
|
+
* overfetched and the source_type restriction is applied after retrieval, in
|
|
1286
|
+
* SQL. Search rank order is preserved through the filter.
|
|
1287
|
+
*
|
|
1288
|
+
* Accepted tradeoff: an anchor that contradicts a candidate while sharing no
|
|
1289
|
+
* vocabulary with it is now missed. Exhaustive-but-broken traded for
|
|
1290
|
+
* relevance-scoped-and-working.
|
|
1291
|
+
*
|
|
1292
|
+
* `cache` is keyed by the derived query rather than by the batch, so two
|
|
1293
|
+
* batches that reduce to the same query share one lookup. Caller-owned and
|
|
1294
|
+
* per-pass — see the call site in doRunHeal.
|
|
1295
|
+
*/
|
|
1296
|
+
private _selectHealAnchors;
|
|
1225
1297
|
private _sanitizeRankerError;
|
|
1226
1298
|
}
|
|
1227
1299
|
|
|
@@ -96,6 +96,9 @@ interface OntologyBackfillResult {
|
|
|
96
96
|
remaining: number;
|
|
97
97
|
/** Untyped facts inside the recheck cooldown. */
|
|
98
98
|
deferred: number;
|
|
99
|
+
/** Facts a batch could not process even alone, skipped so the pass could finish.
|
|
100
|
+
* Stamped with the recheck cooldown, so they reappear as `deferred` next pass. */
|
|
101
|
+
skipped: number;
|
|
99
102
|
}
|
|
100
103
|
interface WikiConfig {
|
|
101
104
|
/**
|
|
@@ -303,6 +306,17 @@ interface LLMProvider {
|
|
|
303
306
|
* When absent or throws, `read()` falls back to MiniSearch.
|
|
304
307
|
*/
|
|
305
308
|
embed?: (text: string) => Promise<number[]>;
|
|
309
|
+
/**
|
|
310
|
+
* Optional. The provider's hard output-token ceiling for `generateText`.
|
|
311
|
+
* Core cannot observe it — the token budget lives entirely in the host's
|
|
312
|
+
* adapter — so maintenance passes that generate response-size-proportional
|
|
313
|
+
* output size their batches blind unless this is declared.
|
|
314
|
+
*
|
|
315
|
+
* When absent, batching falls back to a conservative default and adapts
|
|
316
|
+
* downward on failure. When present, it is used as a sizing hint only and is
|
|
317
|
+
* never trusted as a guarantee.
|
|
318
|
+
*/
|
|
319
|
+
maxOutputTokens?: number;
|
|
306
320
|
}
|
|
307
321
|
/**
|
|
308
322
|
* Result of semantic ranking for a single fact.
|
|
@@ -648,6 +662,23 @@ declare class EntryRepository extends BaseRepository {
|
|
|
648
662
|
* Used by _getFullBundle().
|
|
649
663
|
*/
|
|
650
664
|
findAllByEntityId(entityId: string, tx?: SQLiteAdapter): Promise<WikiFact[]>;
|
|
665
|
+
/**
|
|
666
|
+
* Fetch live, mutable entries for an entity — everything heal is allowed to
|
|
667
|
+
* downgrade or delete. Heal previously loaded every row via
|
|
668
|
+
* findAllByEntityId and filtered in JS, which on a document-heavy corpus
|
|
669
|
+
* meant loading 2560 rows to keep 31.
|
|
670
|
+
*/
|
|
671
|
+
findHealCandidatesByEntityId(entityId: string, tx?: SQLiteAdapter): Promise<WikiFact[]>;
|
|
672
|
+
/**
|
|
673
|
+
* Resolve search hits to document anchors. The MiniSearch index holds every
|
|
674
|
+
* fact, not only immutable_document rows, so the source-type restriction has
|
|
675
|
+
* to be applied here, after retrieval.
|
|
676
|
+
*/
|
|
677
|
+
findAnchorRowsByIds(entityId: string, ids: readonly string[], tx?: SQLiteAdapter): Promise<Array<{
|
|
678
|
+
id: string;
|
|
679
|
+
title: string;
|
|
680
|
+
source_ref: string | null;
|
|
681
|
+
}>>;
|
|
651
682
|
/**
|
|
652
683
|
* Fetch recent non-deleted entries for an entity (limited), ordered by updated_at DESC.
|
|
653
684
|
* Used by MaintenanceService.doRunLibrarian().
|
|
@@ -845,10 +876,23 @@ declare class SearchService {
|
|
|
845
876
|
private miniSearch;
|
|
846
877
|
private miniSearchEntryIdsByEntity;
|
|
847
878
|
private vectorCache;
|
|
879
|
+
/**
|
|
880
|
+
* Serializes rebuilds. `rebuildIndex` awaits a repository read between
|
|
881
|
+
* snapshotting the previous id set and discarding it, so two concurrent
|
|
882
|
+
* sync() calls for one entity can interleave: a slow, stale read lands last
|
|
883
|
+
* and discards documents the fresh read just added. Chaining also keeps
|
|
884
|
+
* discard()/addAll() out of each other's way, which is what accrued the
|
|
885
|
+
* auto-vacuum debt behind the TypeError in #64.
|
|
886
|
+
*/
|
|
887
|
+
private syncChain;
|
|
848
888
|
constructor(entryRepo: EntryRepository);
|
|
849
889
|
/**
|
|
850
890
|
* Rebuilds the search index and clears the vector cache for a given entity.
|
|
851
891
|
* A direct replacement for manually syncing state after a DB transaction.
|
|
892
|
+
*
|
|
893
|
+
* Rebuilds are serialized per instance and never reject: the MiniSearch index
|
|
894
|
+
* is a rebuildable cache over SQLite, so degraded keyword search is the
|
|
895
|
+
* correct failure mode and killing the host process is not.
|
|
852
896
|
*/
|
|
853
897
|
sync(entityId?: string): Promise<void>;
|
|
854
898
|
/**
|
|
@@ -1221,7 +1265,35 @@ declare class MaintenanceService {
|
|
|
1221
1265
|
promptOverride?: string;
|
|
1222
1266
|
batchSize?: number;
|
|
1223
1267
|
}): Promise<OntologyBackfillResult>;
|
|
1268
|
+
/**
|
|
1269
|
+
* Applies one parsed backfill batch in its own transaction. Per-batch rather
|
|
1270
|
+
* than one transaction for the pass, so mergeEmergentUpdates semantics and
|
|
1271
|
+
* the mid-flight `mode === 'off'` abort check keep the shape they had when a
|
|
1272
|
+
* pass was a single call.
|
|
1273
|
+
*/
|
|
1274
|
+
private _applyOntologyBackfillBatch;
|
|
1224
1275
|
private _validatePruneDuration;
|
|
1276
|
+
/**
|
|
1277
|
+
* Anchors relevant to one batch of heal candidates.
|
|
1278
|
+
*
|
|
1279
|
+
* Heal used to pass every immutable_document fact for the entity — 2560 rows
|
|
1280
|
+
* against 31 candidates on the corpus behind #63 — which is what blew the
|
|
1281
|
+
* output ceiling. Anchors are now retrieved by keyword relevance to the batch
|
|
1282
|
+
* and capped.
|
|
1283
|
+
*
|
|
1284
|
+
* The MiniSearch index holds all facts, not only anchors, so hits are
|
|
1285
|
+
* overfetched and the source_type restriction is applied after retrieval, in
|
|
1286
|
+
* SQL. Search rank order is preserved through the filter.
|
|
1287
|
+
*
|
|
1288
|
+
* Accepted tradeoff: an anchor that contradicts a candidate while sharing no
|
|
1289
|
+
* vocabulary with it is now missed. Exhaustive-but-broken traded for
|
|
1290
|
+
* relevance-scoped-and-working.
|
|
1291
|
+
*
|
|
1292
|
+
* `cache` is keyed by the derived query rather than by the batch, so two
|
|
1293
|
+
* batches that reduce to the same query share one lookup. Caller-owned and
|
|
1294
|
+
* per-pass — see the call site in doRunHeal.
|
|
1295
|
+
*/
|
|
1296
|
+
private _selectHealAnchors;
|
|
1225
1297
|
private _sanitizeRankerError;
|
|
1226
1298
|
}
|
|
1227
1299
|
|
package/dist/testing.d.mts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export { Q as EmbeddingService, T as ImportExportService, U as IngestionService, X as JobManager, X as JobManagerType, Y as MaintenanceService, Z as RetrievalService, _ as SearchService, _ as SearchServiceType, I as WikiMemoryTestAccess, $ as WriteService } from './testing-
|
|
1
|
+
export { Q as EmbeddingService, T as ImportExportService, U as IngestionService, X as JobManager, X as JobManagerType, Y as MaintenanceService, Z as RetrievalService, _ as SearchService, _ as SearchServiceType, I as WikiMemoryTestAccess, $ as WriteService } from './testing-CBjAuTSl.mjs';
|
|
2
2
|
import 'minisearch';
|
package/dist/testing.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export { Q as EmbeddingService, T as ImportExportService, U as IngestionService, X as JobManager, X as JobManagerType, Y as MaintenanceService, Z as RetrievalService, _ as SearchService, _ as SearchServiceType, I as WikiMemoryTestAccess, $ as WriteService } from './testing-
|
|
1
|
+
export { Q as EmbeddingService, T as ImportExportService, U as IngestionService, X as JobManager, X as JobManagerType, Y as MaintenanceService, Z as RetrievalService, _ as SearchService, _ as SearchServiceType, I as WikiMemoryTestAccess, $ as WriteService } from './testing-CBjAuTSl.js';
|
|
2
2
|
import 'minisearch';
|
package/dist/testing.js
CHANGED
|
@@ -1145,12 +1145,122 @@ function parseEmbedding(blob, text) {
|
|
|
1145
1145
|
return null;
|
|
1146
1146
|
}
|
|
1147
1147
|
|
|
1148
|
+
// src/services/BoundedLlmCall.ts
|
|
1149
|
+
var DEFAULT_BATCH_SIZE = 10;
|
|
1150
|
+
var ESTIMATED_OUTPUT_TOKENS_PER_ITEM = 150;
|
|
1151
|
+
var OUTPUT_BUDGET_FRACTION = 0.8;
|
|
1152
|
+
var TRUNCATION_PATTERNS = [
|
|
1153
|
+
/truncat/i,
|
|
1154
|
+
/token limit/i,
|
|
1155
|
+
/max(imum)?[ _-]?tokens?/i,
|
|
1156
|
+
/output limit/i,
|
|
1157
|
+
/length limit/i,
|
|
1158
|
+
/finish[_ ]?reason/i
|
|
1159
|
+
];
|
|
1160
|
+
var EXCEEDS_LIMIT_PATTERN = /exceed[a-z]*[^.]{0,40}\b(model|context)?[ _-]?limit/i;
|
|
1161
|
+
function isTruncationError(err) {
|
|
1162
|
+
const message = err instanceof Error ? err.message : String(err ?? "");
|
|
1163
|
+
if (EXCEEDS_LIMIT_PATTERN.test(message)) return false;
|
|
1164
|
+
return TRUNCATION_PATTERNS.some((pattern) => pattern.test(message));
|
|
1165
|
+
}
|
|
1166
|
+
function initialBatchSize(maxOutputTokens) {
|
|
1167
|
+
if (!maxOutputTokens || !Number.isFinite(maxOutputTokens) || maxOutputTokens <= 0) {
|
|
1168
|
+
return DEFAULT_BATCH_SIZE;
|
|
1169
|
+
}
|
|
1170
|
+
const estimate = Math.floor(
|
|
1171
|
+
maxOutputTokens * OUTPUT_BUDGET_FRACTION / ESTIMATED_OUTPUT_TOKENS_PER_ITEM
|
|
1172
|
+
);
|
|
1173
|
+
return Math.max(DEFAULT_BATCH_SIZE, estimate);
|
|
1174
|
+
}
|
|
1175
|
+
var promptLength = (prompts) => prompts.systemPrompt.length + prompts.userPrompt.length;
|
|
1176
|
+
async function runBatched(args) {
|
|
1177
|
+
const { items, buildPrompt, call, parse, maxPromptChars, maxOutputTokens, onSkip } = args;
|
|
1178
|
+
const results = [];
|
|
1179
|
+
const skipped = [];
|
|
1180
|
+
let batches = 0;
|
|
1181
|
+
let batchSize = initialBatchSize(maxOutputTokens);
|
|
1182
|
+
const trim = async (candidate) => {
|
|
1183
|
+
const whole = await buildPrompt(candidate);
|
|
1184
|
+
if (candidate.length <= 1 || promptLength(whole) <= maxPromptChars) {
|
|
1185
|
+
return { batch: candidate, prompts: whole };
|
|
1186
|
+
}
|
|
1187
|
+
let low = 2;
|
|
1188
|
+
let high = candidate.length - 1;
|
|
1189
|
+
let best;
|
|
1190
|
+
let bestPrompts;
|
|
1191
|
+
while (low <= high) {
|
|
1192
|
+
const mid = Math.floor((low + high) / 2);
|
|
1193
|
+
const batch = candidate.slice(0, mid);
|
|
1194
|
+
const prompts = await buildPrompt(batch);
|
|
1195
|
+
if (promptLength(prompts) <= maxPromptChars) {
|
|
1196
|
+
best = batch;
|
|
1197
|
+
bestPrompts = prompts;
|
|
1198
|
+
low = mid + 1;
|
|
1199
|
+
} else {
|
|
1200
|
+
high = mid - 1;
|
|
1201
|
+
}
|
|
1202
|
+
}
|
|
1203
|
+
if (best && bestPrompts) return { batch: best, prompts: bestPrompts };
|
|
1204
|
+
const single = candidate.slice(0, 1);
|
|
1205
|
+
return { batch: single, prompts: await buildPrompt(single) };
|
|
1206
|
+
};
|
|
1207
|
+
const onFailure = async (batch, err) => {
|
|
1208
|
+
if (batch.length <= 1) {
|
|
1209
|
+
if (batch.length === 1) {
|
|
1210
|
+
skipped.push(batch[0]);
|
|
1211
|
+
onSkip?.(batch[0], err);
|
|
1212
|
+
}
|
|
1213
|
+
return;
|
|
1214
|
+
}
|
|
1215
|
+
const mid = Math.ceil(batch.length / 2);
|
|
1216
|
+
if (mid < batchSize) batchSize = mid;
|
|
1217
|
+
let i = 0;
|
|
1218
|
+
while (i < batch.length) {
|
|
1219
|
+
const size = Math.min(batchSize, batch.length - i);
|
|
1220
|
+
const trimmed = await trim(batch.slice(i, i + size));
|
|
1221
|
+
await attempt(trimmed.batch, trimmed.prompts);
|
|
1222
|
+
i += trimmed.batch.length;
|
|
1223
|
+
}
|
|
1224
|
+
};
|
|
1225
|
+
const attempt = async (batch, prebuilt) => {
|
|
1226
|
+
if (batch.length === 0) return;
|
|
1227
|
+
const prompts = prebuilt ?? await buildPrompt(batch);
|
|
1228
|
+
batches++;
|
|
1229
|
+
let responseText;
|
|
1230
|
+
try {
|
|
1231
|
+
responseText = await call(prompts);
|
|
1232
|
+
} catch (err) {
|
|
1233
|
+
if (!isTruncationError(err)) throw err;
|
|
1234
|
+
await onFailure(batch, err);
|
|
1235
|
+
return;
|
|
1236
|
+
}
|
|
1237
|
+
let result;
|
|
1238
|
+
try {
|
|
1239
|
+
result = parse(responseText, batch);
|
|
1240
|
+
} catch (err) {
|
|
1241
|
+
await onFailure(batch, err);
|
|
1242
|
+
return;
|
|
1243
|
+
}
|
|
1244
|
+
results.push(result);
|
|
1245
|
+
};
|
|
1246
|
+
let index = 0;
|
|
1247
|
+
while (index < items.length) {
|
|
1248
|
+
const { batch, prompts } = await trim(items.slice(index, index + batchSize));
|
|
1249
|
+
index += batch.length;
|
|
1250
|
+
await attempt(batch, prompts);
|
|
1251
|
+
}
|
|
1252
|
+
return { results, skipped, batches };
|
|
1253
|
+
}
|
|
1254
|
+
|
|
1148
1255
|
// src/services/MaintenanceService.ts
|
|
1149
1256
|
var FUZZY_THRESHOLD = 0.5;
|
|
1150
1257
|
var MIN_TOKENS_TO_QUALIFY = 3;
|
|
1151
1258
|
var ONTOLOGY_BACKFILL_BATCH_SIZE = 25;
|
|
1152
1259
|
var ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS = 4e4;
|
|
1153
1260
|
var ONTOLOGY_BACKFILL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
1261
|
+
var HEAL_MAX_ANCHORS = 50;
|
|
1262
|
+
var HEAL_ANCHOR_SEARCH_OVERFETCH = 4;
|
|
1263
|
+
var HEAL_MAX_PROMPT_CHARS = 4e4;
|
|
1154
1264
|
var MaintenanceService = class {
|
|
1155
1265
|
constructor(db, prefix, options, entryRepo, taskRepo, eventRepo, metadataRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
1156
1266
|
this.db = db;
|
|
@@ -1533,30 +1643,56 @@ var MaintenanceService = class {
|
|
|
1533
1643
|
console.warn(`[WikiMemory] onEmbeddingPersisted hook failed during heal orphan pass for ${factId}:`, hookErr);
|
|
1534
1644
|
}
|
|
1535
1645
|
}
|
|
1536
|
-
const
|
|
1646
|
+
const healCandidates = await this.entryRepo.findHealCandidatesByEntityId(entityId);
|
|
1537
1647
|
const allTasks = await this.taskRepo.findAllPending([entityId]);
|
|
1538
1648
|
const recentEvents = await this.eventRepo.getRecent(entityId, 20);
|
|
1539
|
-
const
|
|
1540
|
-
const documentAnchors = allFactsRows.filter((f) => f.source_type === "immutable_document").map(({ id, title, source_ref }) => ({ id, title, source_ref }));
|
|
1541
|
-
const healCandidatesForPrompt = healCandidates.map((f) => {
|
|
1649
|
+
const toPromptShape = (f) => {
|
|
1542
1650
|
const { embedding: _embedding, embedding_blob: _blob, ...rest } = f;
|
|
1543
1651
|
return { ...rest, tags: typeof rest.tags === "string" ? JSON.parse(rest.tags) : rest.tags };
|
|
1652
|
+
};
|
|
1653
|
+
const anchorCache = /* @__PURE__ */ new Map();
|
|
1654
|
+
const outcome = await runBatched({
|
|
1655
|
+
items: healCandidates,
|
|
1656
|
+
buildPrompt: async (batch) => {
|
|
1657
|
+
const documentAnchors = await this._selectHealAnchors(entityId, batch, anchorCache);
|
|
1658
|
+
return this.promptService.buildHealPrompt(
|
|
1659
|
+
batch.map(toPromptShape),
|
|
1660
|
+
documentAnchors,
|
|
1661
|
+
allTasks,
|
|
1662
|
+
recentEvents,
|
|
1663
|
+
promptOverride
|
|
1664
|
+
);
|
|
1665
|
+
},
|
|
1666
|
+
call: (prompts) => this.options.llmProvider.generateText(prompts),
|
|
1667
|
+
parse: (responseText, batch) => {
|
|
1668
|
+
const result = parseJsonResponse(responseText);
|
|
1669
|
+
return {
|
|
1670
|
+
batch,
|
|
1671
|
+
downgraded: Array.isArray(result.downgraded) ? result.downgraded : [],
|
|
1672
|
+
deleted: Array.isArray(result.deleted) ? result.deleted : [],
|
|
1673
|
+
newFacts: Array.isArray(result.newFacts) ? result.newFacts : []
|
|
1674
|
+
};
|
|
1675
|
+
},
|
|
1676
|
+
maxOutputTokens: this.options.llmProvider.maxOutputTokens,
|
|
1677
|
+
maxPromptChars: HEAL_MAX_PROMPT_CHARS,
|
|
1678
|
+
onSkip: (fact, err) => {
|
|
1679
|
+
console.warn(
|
|
1680
|
+
`[WikiMemory] heal skipped ${entityId}/${fact.id}: response could not be bounded`,
|
|
1681
|
+
err
|
|
1682
|
+
);
|
|
1683
|
+
}
|
|
1544
1684
|
});
|
|
1545
|
-
const
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
const
|
|
1555
|
-
const
|
|
1556
|
-
const deleted = Array.isArray(result.deleted) ? result.deleted : [];
|
|
1557
|
-
const newFacts = Array.isArray(result.newFacts) ? result.newFacts : [];
|
|
1558
|
-
const safeDowngraded = Array.from(new Set(downgraded.filter((id) => mutableIds.has(id))));
|
|
1559
|
-
const safeDeleted = Array.from(new Set(deleted.filter((id) => mutableIds.has(id))));
|
|
1685
|
+
const safeDowngradedSet = /* @__PURE__ */ new Set();
|
|
1686
|
+
const safeDeletedSet = /* @__PURE__ */ new Set();
|
|
1687
|
+
const newFacts = [];
|
|
1688
|
+
for (const batchResult of outcome.results) {
|
|
1689
|
+
const mutableIds = new Set(batchResult.batch.map((f) => f.id));
|
|
1690
|
+
for (const id of batchResult.downgraded) if (mutableIds.has(id)) safeDowngradedSet.add(id);
|
|
1691
|
+
for (const id of batchResult.deleted) if (mutableIds.has(id)) safeDeletedSet.add(id);
|
|
1692
|
+
newFacts.push(...batchResult.newFacts);
|
|
1693
|
+
}
|
|
1694
|
+
const safeDowngraded = Array.from(safeDowngradedSet);
|
|
1695
|
+
const safeDeleted = Array.from(safeDeletedSet);
|
|
1560
1696
|
const validNewFacts = newFacts.map(validateFact).filter((f) => f !== null);
|
|
1561
1697
|
const insertedFacts = [];
|
|
1562
1698
|
const uniqueDeletedFactIds = Array.from(new Set(safeDeleted));
|
|
@@ -1623,7 +1759,7 @@ var MaintenanceService = class {
|
|
|
1623
1759
|
}
|
|
1624
1760
|
const now = Date.now();
|
|
1625
1761
|
const recheckCutoff = now - ONTOLOGY_BACKFILL_RECHECK_MS;
|
|
1626
|
-
const zeroed = { scanned: 0, typed: 0, failedValidation: 0, edgesAdded: 0 };
|
|
1762
|
+
const zeroed = { scanned: 0, typed: 0, failedValidation: 0, edgesAdded: 0, skipped: 0 };
|
|
1627
1763
|
const ontologyService = this.ontologyService;
|
|
1628
1764
|
if (!ontologyService) {
|
|
1629
1765
|
return { ...zeroed, remaining: 0, deferred: 0 };
|
|
@@ -1645,18 +1781,79 @@ var MaintenanceService = class {
|
|
|
1645
1781
|
options?.promptOverride,
|
|
1646
1782
|
ontologyContext
|
|
1647
1783
|
);
|
|
1648
|
-
const
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1784
|
+
const outcome = await runBatched({
|
|
1785
|
+
items: candidates,
|
|
1786
|
+
buildPrompt,
|
|
1787
|
+
call: (prompts) => this.options.llmProvider.generateText(prompts),
|
|
1788
|
+
parse: (responseText, batch) => {
|
|
1789
|
+
const parsed = parseJsonResponse(responseText);
|
|
1790
|
+
return {
|
|
1791
|
+
batch,
|
|
1792
|
+
classifications: Array.isArray(parsed.classifications) ? parsed.classifications : [],
|
|
1793
|
+
ontologyUpdates: parsed.ontology_updates
|
|
1794
|
+
};
|
|
1795
|
+
},
|
|
1796
|
+
maxOutputTokens: this.options.llmProvider.maxOutputTokens,
|
|
1797
|
+
maxPromptChars: ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS,
|
|
1798
|
+
onSkip: (fact, err) => {
|
|
1799
|
+
console.warn(
|
|
1800
|
+
`[WikiMemory] ontology backfill skipped ${entityId}/${fact.id}: response could not be bounded`,
|
|
1801
|
+
err
|
|
1802
|
+
);
|
|
1803
|
+
}
|
|
1804
|
+
});
|
|
1805
|
+
let typed = 0;
|
|
1806
|
+
let failedValidation = 0;
|
|
1807
|
+
let edgesAdded = 0;
|
|
1808
|
+
let scanned = 0;
|
|
1809
|
+
let abortedOntologyOff = false;
|
|
1810
|
+
for (const batchResult of outcome.results) {
|
|
1811
|
+
const applied = await this._applyOntologyBackfillBatch(entityId, batchResult, now);
|
|
1812
|
+
if (applied.abortedOntologyOff) {
|
|
1813
|
+
abortedOntologyOff = true;
|
|
1814
|
+
break;
|
|
1815
|
+
}
|
|
1816
|
+
typed += applied.typed;
|
|
1817
|
+
failedValidation += applied.failedValidation;
|
|
1818
|
+
edgesAdded += applied.edgesAdded;
|
|
1819
|
+
scanned += batchResult.batch.length;
|
|
1820
|
+
}
|
|
1821
|
+
if (abortedOntologyOff) {
|
|
1822
|
+
const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
|
|
1823
|
+
return {
|
|
1824
|
+
scanned,
|
|
1825
|
+
typed,
|
|
1826
|
+
failedValidation,
|
|
1827
|
+
edgesAdded,
|
|
1828
|
+
skipped: outcome.skipped.length,
|
|
1829
|
+
remaining: 0,
|
|
1830
|
+
deferred: counts2.deferred
|
|
1831
|
+
};
|
|
1832
|
+
}
|
|
1833
|
+
if (outcome.skipped.length > 0) {
|
|
1834
|
+
await this.entryRepo.markOntologyChecked(outcome.skipped.map((f) => f.id), entityId, now, this.db);
|
|
1835
|
+
}
|
|
1836
|
+
this.searchService.evictCache(entityId);
|
|
1837
|
+
const counts = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
|
|
1838
|
+
return {
|
|
1839
|
+
scanned,
|
|
1840
|
+
typed,
|
|
1841
|
+
failedValidation,
|
|
1842
|
+
edgesAdded,
|
|
1843
|
+
skipped: outcome.skipped.length,
|
|
1844
|
+
remaining: counts.eligible,
|
|
1845
|
+
deferred: counts.deferred
|
|
1846
|
+
};
|
|
1847
|
+
}
|
|
1848
|
+
/**
|
|
1849
|
+
* Applies one parsed backfill batch in its own transaction. Per-batch rather
|
|
1850
|
+
* than one transaction for the pass, so mergeEmergentUpdates semantics and
|
|
1851
|
+
* the mid-flight `mode === 'off'` abort check keep the shape they had when a
|
|
1852
|
+
* pass was a single call.
|
|
1853
|
+
*/
|
|
1854
|
+
async _applyOntologyBackfillBatch(entityId, batchResult, now) {
|
|
1855
|
+
const ontologyService = this.ontologyService;
|
|
1856
|
+
const { batch, classifications, ontologyUpdates } = batchResult;
|
|
1660
1857
|
let typed = 0;
|
|
1661
1858
|
let failedValidation = 0;
|
|
1662
1859
|
let edgesAdded = 0;
|
|
@@ -1667,8 +1864,8 @@ var MaintenanceService = class {
|
|
|
1667
1864
|
abortedOntologyOff = true;
|
|
1668
1865
|
return;
|
|
1669
1866
|
}
|
|
1670
|
-
if (txMode === "emergent" &&
|
|
1671
|
-
manifest = await ontologyService.mergeEmergentUpdates(entityId,
|
|
1867
|
+
if (txMode === "emergent" && ontologyUpdates) {
|
|
1868
|
+
manifest = await ontologyService.mergeEmergentUpdates(entityId, ontologyUpdates, tx);
|
|
1672
1869
|
}
|
|
1673
1870
|
const titleRows = await this.entryRepo.findTitleIndexByEntityId(entityId, tx);
|
|
1674
1871
|
const titleIndex = /* @__PURE__ */ new Map();
|
|
@@ -1722,19 +1919,58 @@ var MaintenanceService = class {
|
|
|
1722
1919
|
}
|
|
1723
1920
|
await this.entryRepo.markOntologyChecked(batch.map((f) => f.id), entityId, now, tx);
|
|
1724
1921
|
});
|
|
1725
|
-
|
|
1726
|
-
const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
|
|
1727
|
-
return { ...zeroed, remaining: 0, deferred: counts2.deferred };
|
|
1728
|
-
}
|
|
1729
|
-
this.searchService.evictCache(entityId);
|
|
1730
|
-
const counts = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
|
|
1731
|
-
return { scanned: batch.length, typed, failedValidation, edgesAdded, remaining: counts.eligible, deferred: counts.deferred };
|
|
1922
|
+
return { typed, failedValidation, edgesAdded, abortedOntologyOff };
|
|
1732
1923
|
}
|
|
1733
1924
|
_validatePruneDuration(value, name) {
|
|
1734
1925
|
if (value !== null && value !== void 0 && (typeof value !== "number" || !isFinite(value) || value < 0)) {
|
|
1735
1926
|
throw new Error(`Invalid ${name}: must be a non-negative finite number or null`);
|
|
1736
1927
|
}
|
|
1737
1928
|
}
|
|
1929
|
+
/**
|
|
1930
|
+
* Anchors relevant to one batch of heal candidates.
|
|
1931
|
+
*
|
|
1932
|
+
* Heal used to pass every immutable_document fact for the entity — 2560 rows
|
|
1933
|
+
* against 31 candidates on the corpus behind #63 — which is what blew the
|
|
1934
|
+
* output ceiling. Anchors are now retrieved by keyword relevance to the batch
|
|
1935
|
+
* and capped.
|
|
1936
|
+
*
|
|
1937
|
+
* The MiniSearch index holds all facts, not only anchors, so hits are
|
|
1938
|
+
* overfetched and the source_type restriction is applied after retrieval, in
|
|
1939
|
+
* SQL. Search rank order is preserved through the filter.
|
|
1940
|
+
*
|
|
1941
|
+
* Accepted tradeoff: an anchor that contradicts a candidate while sharing no
|
|
1942
|
+
* vocabulary with it is now missed. Exhaustive-but-broken traded for
|
|
1943
|
+
* relevance-scoped-and-working.
|
|
1944
|
+
*
|
|
1945
|
+
* `cache` is keyed by the derived query rather than by the batch, so two
|
|
1946
|
+
* batches that reduce to the same query share one lookup. Caller-owned and
|
|
1947
|
+
* per-pass — see the call site in doRunHeal.
|
|
1948
|
+
*/
|
|
1949
|
+
async _selectHealAnchors(entityId, batch, cache) {
|
|
1950
|
+
const query = batch.map((f) => f.title).join(" ").trim();
|
|
1951
|
+
if (!query) return [];
|
|
1952
|
+
const cached = cache?.get(query);
|
|
1953
|
+
if (cached) return cached;
|
|
1954
|
+
const hits = this.searchService.searchKeyword(
|
|
1955
|
+
query,
|
|
1956
|
+
[entityId],
|
|
1957
|
+
HEAL_MAX_ANCHORS * HEAL_ANCHOR_SEARCH_OVERFETCH
|
|
1958
|
+
);
|
|
1959
|
+
const hitIds = hits.map((h) => h.id);
|
|
1960
|
+
const anchors = [];
|
|
1961
|
+
if (hitIds.length > 0) {
|
|
1962
|
+
const rows = await this.entryRepo.findAnchorRowsByIds(entityId, hitIds);
|
|
1963
|
+
const byId = new Map(rows.map((r) => [r.id, r]));
|
|
1964
|
+
for (const id of hitIds) {
|
|
1965
|
+
const row = byId.get(id);
|
|
1966
|
+
if (!row) continue;
|
|
1967
|
+
anchors.push(row);
|
|
1968
|
+
if (anchors.length >= HEAL_MAX_ANCHORS) break;
|
|
1969
|
+
}
|
|
1970
|
+
}
|
|
1971
|
+
cache?.set(query, anchors);
|
|
1972
|
+
return anchors;
|
|
1973
|
+
}
|
|
1738
1974
|
_sanitizeRankerError(err) {
|
|
1739
1975
|
return sanitizeRankerError(err, this.options.sanitizeRankerErrors);
|
|
1740
1976
|
}
|
|
@@ -2293,9 +2529,23 @@ var _SearchService = class _SearchService {
|
|
|
2293
2529
|
this.entryRepo = entryRepo;
|
|
2294
2530
|
this.miniSearchEntryIdsByEntity = /* @__PURE__ */ new Map();
|
|
2295
2531
|
this.vectorCache = /* @__PURE__ */ new Map();
|
|
2532
|
+
/**
|
|
2533
|
+
* Serializes rebuilds. `rebuildIndex` awaits a repository read between
|
|
2534
|
+
* snapshotting the previous id set and discarding it, so two concurrent
|
|
2535
|
+
* sync() calls for one entity can interleave: a slow, stale read lands last
|
|
2536
|
+
* and discards documents the fresh read just added. Chaining also keeps
|
|
2537
|
+
* discard()/addAll() out of each other's way, which is what accrued the
|
|
2538
|
+
* auto-vacuum debt behind the TypeError in #64.
|
|
2539
|
+
*/
|
|
2540
|
+
this.syncChain = Promise.resolve();
|
|
2296
2541
|
this.miniSearch = new MiniSearch__default.default({
|
|
2297
2542
|
fields: ["title", "body", "tags"],
|
|
2298
2543
|
storeFields: ["entity_id"],
|
|
2544
|
+
// Vacuuming is driven explicitly at the end of each serialized rebuild
|
|
2545
|
+
// (see sync). Auto-vacuum fires on its own schedule, asynchronously with
|
|
2546
|
+
// respect to the caller, and traversing the tree mid-rebuild is what
|
|
2547
|
+
// threw the uncaught TypeError in MiniSearch.performVacuuming (#64).
|
|
2548
|
+
autoVacuum: false,
|
|
2299
2549
|
searchOptions: {
|
|
2300
2550
|
boost: { title: 2 },
|
|
2301
2551
|
fuzzy: 0.2,
|
|
@@ -2306,10 +2556,26 @@ var _SearchService = class _SearchService {
|
|
|
2306
2556
|
/**
|
|
2307
2557
|
* Rebuilds the search index and clears the vector cache for a given entity.
|
|
2308
2558
|
* A direct replacement for manually syncing state after a DB transaction.
|
|
2559
|
+
*
|
|
2560
|
+
* Rebuilds are serialized per instance and never reject: the MiniSearch index
|
|
2561
|
+
* is a rebuildable cache over SQLite, so degraded keyword search is the
|
|
2562
|
+
* correct failure mode and killing the host process is not.
|
|
2309
2563
|
*/
|
|
2310
2564
|
async sync(entityId) {
|
|
2311
|
-
|
|
2312
|
-
|
|
2565
|
+
const work = this.syncChain.then(async () => {
|
|
2566
|
+
try {
|
|
2567
|
+
try {
|
|
2568
|
+
await this.rebuildIndex(entityId);
|
|
2569
|
+
await this.miniSearch.vacuum();
|
|
2570
|
+
} finally {
|
|
2571
|
+
this.evictCache(entityId);
|
|
2572
|
+
}
|
|
2573
|
+
} catch (err) {
|
|
2574
|
+
console.warn(`[WikiMemory] search index rebuild failed for ${entityId ?? "*"}:`, err);
|
|
2575
|
+
}
|
|
2576
|
+
});
|
|
2577
|
+
this.syncChain = work;
|
|
2578
|
+
return work;
|
|
2313
2579
|
}
|
|
2314
2580
|
/**
|
|
2315
2581
|
* Clears the parsed vector cache. Useful for mid-loop flush guarantees
|