@equationalapplications/core-llm-wiki 4.20.0 → 4.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -0
- package/dist/{chunk-HKB5FEJG.mjs → chunk-UYLYN5N4.mjs} +170 -9
- package/dist/chunk-UYLYN5N4.mjs.map +1 -0
- package/dist/index.d.mts +5 -3
- package/dist/index.d.ts +5 -3
- package/dist/index.js +287 -11
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +118 -7
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-BVN_g414.d.mts → testing-i91HR1TG.d.mts} +100 -7
- package/dist/{testing-BVN_g414.d.ts → testing-i91HR1TG.d.ts} +100 -7
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +167 -6
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +2 -2
- package/dist/chunk-HKB5FEJG.mjs.map +0 -1
package/dist/index.d.mts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-
|
|
2
|
-
export { E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HOOK_TIMEOUT_MARKER, L as LLMProvider, O as
|
|
1
|
+
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-i91HR1TG.mjs';
|
|
2
|
+
export { E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HOOK_TIMEOUT_MARKER, L as LLMProvider, O as ONTOLOGY_BACKFILL_BATCH_SIZE, i as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, j as ONTOLOGY_BACKFILL_RECHECK_MS, k as OntologyBackfillResult, l as OntologyConfig, m as OntologyEdgeType, n as OntologyManifest, o as OntologyMode, p as OntologyNodeType, q as OntologyPromptContext, r as OntologyUpdates, P as PromptOverrides, s as PromptService, t as PrunePartialFailureError, V as VectorRanker, u as VectorRankerFallback, v as VectorRankerRankArgs, w as VectorRankerSemanticResult, x as WikiBusyError, y as WikiBusyOperation, z as WikiCheckpoint, A as WikiConfig, B as WikiEdge, C as WikiEvent, D as WikiFact, I as WikiMemoryTestAccess, J as WikiOutboxEvent, K as WikiTask, N as WikiTransactionError } from './testing-i91HR1TG.mjs';
|
|
3
3
|
import { OkfFile } from '@equationalapplications/core-okf';
|
|
4
4
|
import 'minisearch';
|
|
5
5
|
|
|
@@ -58,6 +58,8 @@ declare function validateLibrarianPromptTemplate(template: string, options: {
|
|
|
58
58
|
}): string[];
|
|
59
59
|
declare function mapLibrarianOptionsToReadOptions(options: LibrarianOptions): Pick<ReadOptions, 'tierWeights' | 'includeZeroWeightEntities'>;
|
|
60
60
|
|
|
61
|
+
declare const ONTOLOGY_BACKFILL_SYSTEM_PROMPT = "You are a knowledge classification agent. You will receive existing memory facts that currently have no ontology type. For each input fact { \"id\", \"title\", \"body\", \"tags\" }, assign the best matching okf_type from the ontology manifest and optionally propose edges to related facts by title.\nReturn ONLY a valid JSON object matching this schema:\n{\n \"classifications\": [\n { \"id\": \"string (input fact id, copied verbatim)\", \"okf_type\": \"string (manifest node type slug)\", \"edges\": [{ \"edge_type\": \"string\", \"target_title\": \"string\" }] }\n ]\n}\nIf no manifest type fits a fact, omit that fact from \"classifications\" entirely \u2014 do not guess.\nDo not return markdown, just raw JSON.";
|
|
62
|
+
|
|
61
63
|
declare function createWiki(db: SQLiteAdapter, options: WikiOptions): WikiMemory;
|
|
62
64
|
|
|
63
|
-
export { DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, FormatContextOptions, FormattedMemoryDump, GraphNeighborhood, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, type OkfImportOptions, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, validateLibrarianPromptTemplate };
|
|
65
|
+
export { DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, FormatContextOptions, FormattedMemoryDump, GraphNeighborhood, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, type OkfImportOptions, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, validateLibrarianPromptTemplate };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-
|
|
2
|
-
export { E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HOOK_TIMEOUT_MARKER, L as LLMProvider, O as
|
|
1
|
+
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-i91HR1TG.js';
|
|
2
|
+
export { E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HOOK_TIMEOUT_MARKER, L as LLMProvider, O as ONTOLOGY_BACKFILL_BATCH_SIZE, i as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, j as ONTOLOGY_BACKFILL_RECHECK_MS, k as OntologyBackfillResult, l as OntologyConfig, m as OntologyEdgeType, n as OntologyManifest, o as OntologyMode, p as OntologyNodeType, q as OntologyPromptContext, r as OntologyUpdates, P as PromptOverrides, s as PromptService, t as PrunePartialFailureError, V as VectorRanker, u as VectorRankerFallback, v as VectorRankerRankArgs, w as VectorRankerSemanticResult, x as WikiBusyError, y as WikiBusyOperation, z as WikiCheckpoint, A as WikiConfig, B as WikiEdge, C as WikiEvent, D as WikiFact, I as WikiMemoryTestAccess, J as WikiOutboxEvent, K as WikiTask, N as WikiTransactionError } from './testing-i91HR1TG.js';
|
|
3
3
|
import { OkfFile } from '@equationalapplications/core-okf';
|
|
4
4
|
import 'minisearch';
|
|
5
5
|
|
|
@@ -58,6 +58,8 @@ declare function validateLibrarianPromptTemplate(template: string, options: {
|
|
|
58
58
|
}): string[];
|
|
59
59
|
declare function mapLibrarianOptionsToReadOptions(options: LibrarianOptions): Pick<ReadOptions, 'tierWeights' | 'includeZeroWeightEntities'>;
|
|
60
60
|
|
|
61
|
+
declare const ONTOLOGY_BACKFILL_SYSTEM_PROMPT = "You are a knowledge classification agent. You will receive existing memory facts that currently have no ontology type. For each input fact { \"id\", \"title\", \"body\", \"tags\" }, assign the best matching okf_type from the ontology manifest and optionally propose edges to related facts by title.\nReturn ONLY a valid JSON object matching this schema:\n{\n \"classifications\": [\n { \"id\": \"string (input fact id, copied verbatim)\", \"okf_type\": \"string (manifest node type slug)\", \"edges\": [{ \"edge_type\": \"string\", \"target_title\": \"string\" }] }\n ]\n}\nIf no manifest type fits a fact, omit that fact from \"classifications\" entirely \u2014 do not guess.\nDo not return markdown, just raw JSON.";
|
|
62
|
+
|
|
61
63
|
declare function createWiki(db: SQLiteAdapter, options: WikiOptions): WikiMemory;
|
|
62
64
|
|
|
63
|
-
export { DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, FormatContextOptions, FormattedMemoryDump, GraphNeighborhood, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, type OkfImportOptions, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, validateLibrarianPromptTemplate };
|
|
65
|
+
export { DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, FormatContextOptions, FormattedMemoryDump, GraphNeighborhood, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, type OkfImportOptions, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, validateLibrarianPromptTemplate };
|
package/dist/index.js
CHANGED
|
@@ -35,7 +35,8 @@ async function setupDatabase(db, prefix) {
|
|
|
35
35
|
deleted_at INTEGER,
|
|
36
36
|
embedding TEXT,
|
|
37
37
|
embedding_blob BLOB,
|
|
38
|
-
okf_type TEXT
|
|
38
|
+
okf_type TEXT,
|
|
39
|
+
ontology_checked_at INTEGER
|
|
39
40
|
);
|
|
40
41
|
|
|
41
42
|
CREATE INDEX IF NOT EXISTS ${prefix}entries_entity_idx ON ${prefix}entries(entity_id);
|
|
@@ -338,6 +339,20 @@ var MIGRATIONS = [
|
|
|
338
339
|
);
|
|
339
340
|
`);
|
|
340
341
|
}
|
|
342
|
+
},
|
|
343
|
+
{
|
|
344
|
+
version: 7,
|
|
345
|
+
description: "Add ontology_checked_at to entries for ontology backfill recheck cooldown",
|
|
346
|
+
run: async (db, prefix) => {
|
|
347
|
+
const cols = await db.getAllAsync(
|
|
348
|
+
`PRAGMA table_info(${prefix}entries)`
|
|
349
|
+
);
|
|
350
|
+
if (!cols.some((c) => c.name === "ontology_checked_at")) {
|
|
351
|
+
await db.execAsync(
|
|
352
|
+
`ALTER TABLE ${prefix}entries ADD COLUMN ontology_checked_at INTEGER`
|
|
353
|
+
);
|
|
354
|
+
}
|
|
355
|
+
}
|
|
341
356
|
}
|
|
342
357
|
];
|
|
343
358
|
for (let i = 1; i < MIGRATIONS.length; i++) {
|
|
@@ -1101,6 +1116,79 @@ var EntryRepository = class extends BaseRepository {
|
|
|
1101
1116
|
);
|
|
1102
1117
|
return rows.map(mapRowToFact);
|
|
1103
1118
|
}
|
|
1119
|
+
/**
|
|
1120
|
+
* Live untyped facts eligible for ontology backfill, oldest first.
|
|
1121
|
+
* Skips facts checked within the recheck cooldown (ontology_checked_at > recheckCutoff).
|
|
1122
|
+
*/
|
|
1123
|
+
async findUntypedByEntityId(entityId, limit, recheckCutoff, tx) {
|
|
1124
|
+
const executor = this.getExecutor(tx);
|
|
1125
|
+
const rows = await executor.getAllAsync(
|
|
1126
|
+
`SELECT * FROM ${this.prefix}entries
|
|
1127
|
+
WHERE entity_id = ? AND okf_type IS NULL AND deleted_at IS NULL
|
|
1128
|
+
AND (ontology_checked_at IS NULL OR ontology_checked_at <= ?)
|
|
1129
|
+
ORDER BY updated_at ASC LIMIT ?`,
|
|
1130
|
+
[entityId, recheckCutoff, limit]
|
|
1131
|
+
);
|
|
1132
|
+
return rows.map(mapRowToFact);
|
|
1133
|
+
}
|
|
1134
|
+
/** Counts live untyped facts: eligible (past cooldown) vs deferred (in cooldown). */
|
|
1135
|
+
async countUntypedByEntityId(entityId, recheckCutoff, tx) {
|
|
1136
|
+
const executor = this.getExecutor(tx);
|
|
1137
|
+
const row = await executor.getFirstAsync(
|
|
1138
|
+
`SELECT
|
|
1139
|
+
SUM(CASE WHEN ontology_checked_at IS NULL OR ontology_checked_at <= ? THEN 1 ELSE 0 END) AS eligible,
|
|
1140
|
+
SUM(CASE WHEN ontology_checked_at IS NOT NULL AND ontology_checked_at > ? THEN 1 ELSE 0 END) AS deferred
|
|
1141
|
+
FROM ${this.prefix}entries
|
|
1142
|
+
WHERE entity_id = ? AND okf_type IS NULL AND deleted_at IS NULL`,
|
|
1143
|
+
[recheckCutoff, recheckCutoff, entityId]
|
|
1144
|
+
);
|
|
1145
|
+
return { eligible: Number(row?.eligible ?? 0), deferred: Number(row?.deferred ?? 0) };
|
|
1146
|
+
}
|
|
1147
|
+
/**
|
|
1148
|
+
* Sets okf_type only when currently NULL — additive by construction: a
|
|
1149
|
+
* concurrently-typed fact is never overwritten. Returns changes for the caller
|
|
1150
|
+
* to distinguish applied vs skipped.
|
|
1151
|
+
*/
|
|
1152
|
+
async updateOkfType(id, entityId, okfType, tx) {
|
|
1153
|
+
const result = await tx.runAsync(
|
|
1154
|
+
`UPDATE ${this.prefix}entries SET okf_type = ?
|
|
1155
|
+
WHERE id = ? AND entity_id = ? AND okf_type IS NULL AND deleted_at IS NULL`,
|
|
1156
|
+
[okfType, id, entityId]
|
|
1157
|
+
);
|
|
1158
|
+
return { changes: result.changes };
|
|
1159
|
+
}
|
|
1160
|
+
/**
|
|
1161
|
+
* Stamps the backfill recheck cooldown. NEVER touches updated_at — import
|
|
1162
|
+
* merge resolution is last-write-wins on updated_at and a bump here would
|
|
1163
|
+
* make an unchanged local fact beat a genuinely newer remote edit.
|
|
1164
|
+
*/
|
|
1165
|
+
async markOntologyChecked(ids, entityId, now, tx) {
|
|
1166
|
+
if (ids.length === 0) return;
|
|
1167
|
+
for (let i = 0; i < ids.length; i += this.chunkSize) {
|
|
1168
|
+
const chunk = ids.slice(i, i + this.chunkSize);
|
|
1169
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
1170
|
+
await tx.runAsync(
|
|
1171
|
+
`UPDATE ${this.prefix}entries SET ontology_checked_at = ?
|
|
1172
|
+
WHERE id IN (${placeholders}) AND entity_id = ? AND deleted_at IS NULL`,
|
|
1173
|
+
[now, ...chunk, entityId]
|
|
1174
|
+
);
|
|
1175
|
+
}
|
|
1176
|
+
}
|
|
1177
|
+
/**
|
|
1178
|
+
* Lightweight full-breadth title index (id, title, okf_type) over all live
|
|
1179
|
+
* typed facts. Used by ontology backfill edge resolution — a recent-N window
|
|
1180
|
+
* would miss an old fact's contemporaries. Untyped rows are excluded: they
|
|
1181
|
+
* can never pass the edge target-type check, and batch facts typed during
|
|
1182
|
+
* the run are re-added to the in-memory index by the caller.
|
|
1183
|
+
*/
|
|
1184
|
+
async findTitleIndexByEntityId(entityId, tx) {
|
|
1185
|
+
const executor = this.getExecutor(tx);
|
|
1186
|
+
const rows = await executor.getAllAsync(
|
|
1187
|
+
`SELECT id, title, okf_type FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL AND okf_type IS NOT NULL`,
|
|
1188
|
+
[entityId]
|
|
1189
|
+
);
|
|
1190
|
+
return rows.map((r) => ({ id: r.id, title: r.title, okf_type: r.okf_type ?? null }));
|
|
1191
|
+
}
|
|
1104
1192
|
};
|
|
1105
1193
|
|
|
1106
1194
|
// src/utils/ids.ts
|
|
@@ -1545,6 +1633,7 @@ var CONFIDENCE_RANK = {
|
|
|
1545
1633
|
var EdgeRepository = class extends BaseRepository {
|
|
1546
1634
|
/**
|
|
1547
1635
|
* Insert an edge, silently skipping on primary-key or uniqueness conflicts.
|
|
1636
|
+
* Returns true when a row was inserted, false when skipped as a duplicate.
|
|
1548
1637
|
* Throws when the insert was skipped due to an id collision with a different edge tuple.
|
|
1549
1638
|
*/
|
|
1550
1639
|
async addIgnoreDuplicate(edge, tx) {
|
|
@@ -1554,17 +1643,18 @@ var EdgeRepository = class extends BaseRepository {
|
|
|
1554
1643
|
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
1555
1644
|
[edge.id, edge.entity_id, edge.source_id, edge.target_id, edge.edge_type, edge.created_at]
|
|
1556
1645
|
);
|
|
1557
|
-
if (result.changes > 0) return;
|
|
1646
|
+
if (result.changes > 0) return true;
|
|
1558
1647
|
const existing = await executor.getFirstAsync(
|
|
1559
1648
|
`SELECT entity_id, source_id, target_id, edge_type FROM ${this.prefix}edges WHERE id = ?`,
|
|
1560
1649
|
[edge.id]
|
|
1561
1650
|
);
|
|
1562
|
-
if (!existing) return;
|
|
1651
|
+
if (!existing) return false;
|
|
1563
1652
|
if (String(existing.entity_id) !== edge.entity_id || String(existing.source_id) !== edge.source_id || String(existing.target_id) !== edge.target_id || String(existing.edge_type) !== edge.edge_type) {
|
|
1564
1653
|
throw new Error(
|
|
1565
1654
|
`Edge id collision: ${JSON.stringify(edge.id)} already exists with a different (entity_id, source_id, target_id, edge_type) tuple`
|
|
1566
1655
|
);
|
|
1567
1656
|
}
|
|
1657
|
+
return false;
|
|
1568
1658
|
}
|
|
1569
1659
|
async getByEntityId(entityId, tx) {
|
|
1570
1660
|
const executor = this.getExecutor(tx);
|
|
@@ -2176,7 +2266,8 @@ var JobManager = class {
|
|
|
2176
2266
|
heal: (id) => this._healKey(id),
|
|
2177
2267
|
reembed: (id) => this._reembedKey(id),
|
|
2178
2268
|
import: (id) => this._importKey(id),
|
|
2179
|
-
forget: (id) => this._forgetKey(id)
|
|
2269
|
+
forget: (id) => this._forgetKey(id),
|
|
2270
|
+
ontologyBackfill: (id) => this._ontologyBackfillKey(id)
|
|
2180
2271
|
};
|
|
2181
2272
|
}
|
|
2182
2273
|
_pruneKey(entityId) {
|
|
@@ -2203,6 +2294,9 @@ var JobManager = class {
|
|
|
2203
2294
|
_healKey(entityId) {
|
|
2204
2295
|
return `${this.prefix}:${entityId}:heal`;
|
|
2205
2296
|
}
|
|
2297
|
+
_ontologyBackfillKey(entityId) {
|
|
2298
|
+
return `${this.prefix}:${entityId}:ontologyBackfill`;
|
|
2299
|
+
}
|
|
2206
2300
|
_isReembedActive(entityId) {
|
|
2207
2301
|
return this.activeMaintenanceJobs.has(this._reembedKey(entityId)) || this.activeMaintenanceJobs.has(this._globalReembedKey());
|
|
2208
2302
|
}
|
|
@@ -2253,14 +2347,16 @@ var JobManager = class {
|
|
|
2253
2347
|
if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
|
|
2254
2348
|
else if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) blockingOperation = "librarian";
|
|
2255
2349
|
else if (this.activeMaintenanceJobs.has(this._healKey(entityId))) blockingOperation = "heal";
|
|
2350
|
+
else if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) blockingOperation = "ontologyBackfill";
|
|
2256
2351
|
else if (this._isReembedActive(entityId)) blockingOperation = "reembed";
|
|
2257
2352
|
else if (this._isIngestActiveFor(entityId)) blockingOperation = "ingest";
|
|
2258
2353
|
else if (this._isImportActiveFor(entityId)) blockingOperation = "import";
|
|
2259
2354
|
else if (this._isForgetActiveFor(entityId)) blockingOperation = "forget";
|
|
2260
2355
|
break;
|
|
2261
2356
|
case "librarian":
|
|
2262
|
-
case "heal":
|
|
2263
|
-
|
|
2357
|
+
case "heal":
|
|
2358
|
+
case "ontologyBackfill": {
|
|
2359
|
+
const opKey = this.lockKeyFns[operation](entityId);
|
|
2264
2360
|
if (this.activeMaintenanceJobs.has(opKey)) blockingOperation = operation;
|
|
2265
2361
|
else if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
|
|
2266
2362
|
else if (this._isReembedActive(entityId)) blockingOperation = "reembed";
|
|
@@ -2274,6 +2370,7 @@ var JobManager = class {
|
|
|
2274
2370
|
else if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
|
|
2275
2371
|
else if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) blockingOperation = "librarian";
|
|
2276
2372
|
else if (this.activeMaintenanceJobs.has(this._healKey(entityId))) blockingOperation = "heal";
|
|
2373
|
+
else if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) blockingOperation = "ontologyBackfill";
|
|
2277
2374
|
else if (this._isIngestActiveFor(entityId)) blockingOperation = "ingest";
|
|
2278
2375
|
else if (this._isImportActiveFor(entityId)) blockingOperation = "import";
|
|
2279
2376
|
else if (this._isForgetActiveFor(entityId)) blockingOperation = "forget";
|
|
@@ -2284,6 +2381,7 @@ var JobManager = class {
|
|
|
2284
2381
|
else if (this._isAnyMaintenanceActiveWithSuffix(":prune")) blockingOperation = "prune";
|
|
2285
2382
|
else if (this._isAnyMaintenanceActiveWithSuffix(":librarian")) blockingOperation = "librarian";
|
|
2286
2383
|
else if (this._isAnyMaintenanceActiveWithSuffix(":heal")) blockingOperation = "heal";
|
|
2384
|
+
else if (this._isAnyMaintenanceActiveWithSuffix(":ontologyBackfill")) blockingOperation = "ontologyBackfill";
|
|
2287
2385
|
else if (this.activeIngestJobs.size > 0) blockingOperation = "ingest";
|
|
2288
2386
|
else if (this._isAnyMaintenanceActiveWithSuffix(":import")) blockingOperation = "import";
|
|
2289
2387
|
else if (this._isAnyMaintenanceActiveWithSuffix(":forget")) blockingOperation = "forget";
|
|
@@ -2294,6 +2392,7 @@ var JobManager = class {
|
|
|
2294
2392
|
if (this.activeMaintenanceJobs.has(selfKey)) blockingOperation = operation;
|
|
2295
2393
|
else if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) blockingOperation = "librarian";
|
|
2296
2394
|
else if (this.activeMaintenanceJobs.has(this._healKey(entityId))) blockingOperation = "heal";
|
|
2395
|
+
else if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) blockingOperation = "ontologyBackfill";
|
|
2297
2396
|
else if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
|
|
2298
2397
|
else if (this._isReembedActive(entityId)) blockingOperation = "reembed";
|
|
2299
2398
|
else if (this._isIngestActiveFor(entityId)) blockingOperation = "ingest";
|
|
@@ -2351,11 +2450,11 @@ var JobManager = class {
|
|
|
2351
2450
|
if (operation !== "global_import" && this.activeMaintenanceJobs.has(this._globalImportKey())) return true;
|
|
2352
2451
|
switch (operation) {
|
|
2353
2452
|
case "librarian":
|
|
2354
|
-
return this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
2355
2453
|
case "heal":
|
|
2356
|
-
|
|
2454
|
+
case "ontologyBackfill":
|
|
2455
|
+
return this.activeMaintenanceJobs.has(this.lockKeyFns[operation](entityId)) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
2357
2456
|
case "prune":
|
|
2358
|
-
return this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._healKey(entityId)) || this._isReembedActive(entityId) || this._isIngestActiveFor(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
2457
|
+
return this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._healKey(entityId)) || this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId)) || this._isReembedActive(entityId) || this._isIngestActiveFor(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
2359
2458
|
default:
|
|
2360
2459
|
return false;
|
|
2361
2460
|
}
|
|
@@ -2380,6 +2479,7 @@ var JobManager = class {
|
|
|
2380
2479
|
if (this.activeMaintenanceJobs.has(this._importKey(entityId))) throw new WikiBusyError("import", entityId);
|
|
2381
2480
|
if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) throw new WikiBusyError("librarian", entityId);
|
|
2382
2481
|
if (this.activeMaintenanceJobs.has(this._healKey(entityId))) throw new WikiBusyError("heal", entityId);
|
|
2482
|
+
if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) throw new WikiBusyError("ontologyBackfill", entityId);
|
|
2383
2483
|
if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) throw new WikiBusyError("prune", entityId);
|
|
2384
2484
|
if (this._isReembedActive(entityId)) throw new WikiBusyError("reembed", entityId);
|
|
2385
2485
|
if (this._isIngestActiveFor(entityId)) throw new WikiBusyError("ingest", entityId);
|
|
@@ -2688,6 +2788,15 @@ Return ONLY a valid JSON object matching this schema:
|
|
|
2688
2788
|
"facts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }]
|
|
2689
2789
|
}
|
|
2690
2790
|
Extract verbatim factual content. Do not return markdown, just raw JSON.`;
|
|
2791
|
+
var ONTOLOGY_BACKFILL_SYSTEM_PROMPT = `You are a knowledge classification agent. You will receive existing memory facts that currently have no ontology type. For each input fact { "id", "title", "body", "tags" }, assign the best matching okf_type from the ontology manifest and optionally propose edges to related facts by title.
|
|
2792
|
+
Return ONLY a valid JSON object matching this schema:
|
|
2793
|
+
{
|
|
2794
|
+
"classifications": [
|
|
2795
|
+
{ "id": "string (input fact id, copied verbatim)", "okf_type": "string (manifest node type slug)", "edges": [{ "edge_type": "string", "target_title": "string" }] }
|
|
2796
|
+
]
|
|
2797
|
+
}
|
|
2798
|
+
If no manifest type fits a fact, omit that fact from "classifications" entirely \u2014 do not guess.
|
|
2799
|
+
Do not return markdown, just raw JSON.`;
|
|
2691
2800
|
|
|
2692
2801
|
// src/services/PromptService.ts
|
|
2693
2802
|
var PromptService = class {
|
|
@@ -2777,6 +2886,22 @@ ${JSON.stringify(recentEvents, null, 2)}
|
|
|
2777
2886
|
The following document anchors are provided for contradiction detection only. Do not include them in \`downgraded\`, \`deleted\`, or \`newFacts\`.`
|
|
2778
2887
|
};
|
|
2779
2888
|
}
|
|
2889
|
+
buildOntologyBackfillPrompt(facts, runtimeOverride, ontologyContext) {
|
|
2890
|
+
const template = runtimeOverride ?? this.globalOverrides?.ontologyBackfillSystemPrompt ?? ONTOLOGY_BACKFILL_SYSTEM_PROMPT;
|
|
2891
|
+
const hasFacts = /\{\{\s*facts\s*\}\}/.test(template);
|
|
2892
|
+
if (hasFacts || this.hasOntologyPlaceholders(template)) {
|
|
2893
|
+
return {
|
|
2894
|
+
systemPrompt: this.buildSystemPrompt(template, { facts }, ontologyContext),
|
|
2895
|
+
userPrompt: hasFacts ? "Please classify the facts." : `Facts:
|
|
2896
|
+
${JSON.stringify(facts, null, 2)}`
|
|
2897
|
+
};
|
|
2898
|
+
}
|
|
2899
|
+
return {
|
|
2900
|
+
systemPrompt: this.appendOntology(template, ontologyContext),
|
|
2901
|
+
userPrompt: `Facts:
|
|
2902
|
+
${JSON.stringify(facts, null, 2)}`
|
|
2903
|
+
};
|
|
2904
|
+
}
|
|
2780
2905
|
};
|
|
2781
2906
|
|
|
2782
2907
|
// src/services/IngestionService.ts
|
|
@@ -2930,6 +3055,9 @@ var IngestionService = class {
|
|
|
2930
3055
|
// src/services/MaintenanceService.ts
|
|
2931
3056
|
var FUZZY_THRESHOLD = 0.5;
|
|
2932
3057
|
var MIN_TOKENS_TO_QUALIFY = 3;
|
|
3058
|
+
var ONTOLOGY_BACKFILL_BATCH_SIZE = 25;
|
|
3059
|
+
var ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS = 4e4;
|
|
3060
|
+
var ONTOLOGY_BACKFILL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
2933
3061
|
var MaintenanceService = class {
|
|
2934
3062
|
constructor(db, prefix, options, entryRepo, taskRepo, eventRepo, metadataRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
2935
3063
|
this.db = db;
|
|
@@ -3035,6 +3163,14 @@ var MaintenanceService = class {
|
|
|
3035
3163
|
this.jobManager.releaseLock("heal", entityId);
|
|
3036
3164
|
}
|
|
3037
3165
|
}
|
|
3166
|
+
async runOntologyBackfill(entityId, options) {
|
|
3167
|
+
this.jobManager.acquireLock("ontologyBackfill", entityId);
|
|
3168
|
+
try {
|
|
3169
|
+
return await this.doRunOntologyBackfill(entityId, options);
|
|
3170
|
+
} finally {
|
|
3171
|
+
this.jobManager.releaseLock("ontologyBackfill", entityId);
|
|
3172
|
+
}
|
|
3173
|
+
}
|
|
3038
3174
|
async runReembed(entityId, opts) {
|
|
3039
3175
|
const embedFn = this.options.llmProvider.embed;
|
|
3040
3176
|
if (!embedFn) return { embedded: 0, skipped: 0, failed: 0 };
|
|
@@ -3386,6 +3522,121 @@ var MaintenanceService = class {
|
|
|
3386
3522
|
}
|
|
3387
3523
|
this.searchService.evictCache(entityId);
|
|
3388
3524
|
}
|
|
3525
|
+
/** Core ontology backfill pass (locks handled by {@link runOntologyBackfill}). Package-internal orchestration hook. */
|
|
3526
|
+
async doRunOntologyBackfill(entityId, options) {
|
|
3527
|
+
const batchSize = options?.batchSize ?? ONTOLOGY_BACKFILL_BATCH_SIZE;
|
|
3528
|
+
if (!Number.isInteger(batchSize) || batchSize < 1) {
|
|
3529
|
+
throw new Error("Invalid batchSize: must be an integer >= 1");
|
|
3530
|
+
}
|
|
3531
|
+
const now = Date.now();
|
|
3532
|
+
const recheckCutoff = now - ONTOLOGY_BACKFILL_RECHECK_MS;
|
|
3533
|
+
const zeroed = { scanned: 0, typed: 0, failedValidation: 0, edgesAdded: 0 };
|
|
3534
|
+
const ontologyService = this.ontologyService;
|
|
3535
|
+
if (!ontologyService) {
|
|
3536
|
+
return { ...zeroed, remaining: 0, deferred: 0 };
|
|
3537
|
+
}
|
|
3538
|
+
const { mode } = await ontologyService.getEffectiveState(entityId);
|
|
3539
|
+
if (mode === "off") {
|
|
3540
|
+
const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
|
|
3541
|
+
return { ...zeroed, remaining: 0, deferred: counts2.deferred };
|
|
3542
|
+
}
|
|
3543
|
+
const candidates = await this.entryRepo.findUntypedByEntityId(entityId, batchSize, recheckCutoff);
|
|
3544
|
+
if (candidates.length === 0) {
|
|
3545
|
+
const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
|
|
3546
|
+
return { ...zeroed, remaining: counts2.eligible, deferred: counts2.deferred };
|
|
3547
|
+
}
|
|
3548
|
+
const ontologyContext = await ontologyService.buildPromptContext(entityId);
|
|
3549
|
+
const toPromptShape = (f) => ({ id: f.id, title: f.title, body: f.body, tags: f.tags });
|
|
3550
|
+
const buildPrompt = (facts) => this.promptService.buildOntologyBackfillPrompt(
|
|
3551
|
+
facts.map(toPromptShape),
|
|
3552
|
+
options?.promptOverride,
|
|
3553
|
+
ontologyContext
|
|
3554
|
+
);
|
|
3555
|
+
const batch = [candidates[0]];
|
|
3556
|
+
let built = buildPrompt(batch);
|
|
3557
|
+
for (let i = 1; i < candidates.length; i++) {
|
|
3558
|
+
const next = buildPrompt([...batch, candidates[i]]);
|
|
3559
|
+
if (next.systemPrompt.length + next.userPrompt.length > ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS) break;
|
|
3560
|
+
batch.push(candidates[i]);
|
|
3561
|
+
built = next;
|
|
3562
|
+
}
|
|
3563
|
+
const { systemPrompt, userPrompt } = built;
|
|
3564
|
+
const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
|
|
3565
|
+
const parsed = parseJsonResponse(responseText);
|
|
3566
|
+
const classifications = Array.isArray(parsed.classifications) ? parsed.classifications : [];
|
|
3567
|
+
let typed = 0;
|
|
3568
|
+
let failedValidation = 0;
|
|
3569
|
+
let edgesAdded = 0;
|
|
3570
|
+
let abortedOntologyOff = false;
|
|
3571
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
3572
|
+
let { mode: txMode, manifest } = await ontologyService.getEffectiveState(entityId, tx);
|
|
3573
|
+
if (txMode === "off") {
|
|
3574
|
+
abortedOntologyOff = true;
|
|
3575
|
+
return;
|
|
3576
|
+
}
|
|
3577
|
+
if (txMode === "emergent" && parsed.ontology_updates) {
|
|
3578
|
+
manifest = await ontologyService.mergeEmergentUpdates(entityId, parsed.ontology_updates, tx);
|
|
3579
|
+
}
|
|
3580
|
+
const titleRows = await this.entryRepo.findTitleIndexByEntityId(entityId, tx);
|
|
3581
|
+
const titleIndex = /* @__PURE__ */ new Map();
|
|
3582
|
+
for (const row of titleRows) {
|
|
3583
|
+
titleIndex.set(normalizeTitleKey(row.title), { id: row.id, okf_type: row.okf_type });
|
|
3584
|
+
}
|
|
3585
|
+
const batchById = new Map(batch.map((f) => [f.id, f]));
|
|
3586
|
+
const applied = /* @__PURE__ */ new Set();
|
|
3587
|
+
const pendingEdges = [];
|
|
3588
|
+
for (const classification of classifications) {
|
|
3589
|
+
const fact = typeof classification.id === "string" ? batchById.get(classification.id) : void 0;
|
|
3590
|
+
if (!fact || applied.has(fact.id)) {
|
|
3591
|
+
failedValidation++;
|
|
3592
|
+
continue;
|
|
3593
|
+
}
|
|
3594
|
+
applied.add(fact.id);
|
|
3595
|
+
const normalized = ontologyService.validateAndNormalizeFact(
|
|
3596
|
+
{
|
|
3597
|
+
title: fact.title,
|
|
3598
|
+
body: fact.body,
|
|
3599
|
+
tags: fact.tags,
|
|
3600
|
+
confidence: fact.confidence,
|
|
3601
|
+
okf_type: classification.okf_type,
|
|
3602
|
+
edges: classification.edges
|
|
3603
|
+
},
|
|
3604
|
+
manifest
|
|
3605
|
+
);
|
|
3606
|
+
if (!normalized.okf_type) {
|
|
3607
|
+
failedValidation++;
|
|
3608
|
+
continue;
|
|
3609
|
+
}
|
|
3610
|
+
const result = await this.entryRepo.updateOkfType(fact.id, entityId, normalized.okf_type, tx);
|
|
3611
|
+
if (result.changes === 0) continue;
|
|
3612
|
+
typed++;
|
|
3613
|
+
titleIndex.set(normalizeTitleKey(fact.title), { id: fact.id, okf_type: normalized.okf_type });
|
|
3614
|
+
if (normalized.edges.length > 0) {
|
|
3615
|
+
pendingEdges.push({ sourceId: fact.id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
3616
|
+
}
|
|
3617
|
+
}
|
|
3618
|
+
for (const item of pendingEdges) {
|
|
3619
|
+
edgesAdded += await ontologyService.resolveAndPersistEdges(
|
|
3620
|
+
entityId,
|
|
3621
|
+
item.sourceId,
|
|
3622
|
+
item.sourceType,
|
|
3623
|
+
item.edges,
|
|
3624
|
+
manifest,
|
|
3625
|
+
titleIndex,
|
|
3626
|
+
tx,
|
|
3627
|
+
now
|
|
3628
|
+
);
|
|
3629
|
+
}
|
|
3630
|
+
await this.entryRepo.markOntologyChecked(batch.map((f) => f.id), entityId, now, tx);
|
|
3631
|
+
});
|
|
3632
|
+
if (abortedOntologyOff) {
|
|
3633
|
+
const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
|
|
3634
|
+
return { ...zeroed, remaining: 0, deferred: counts2.deferred };
|
|
3635
|
+
}
|
|
3636
|
+
this.searchService.evictCache(entityId);
|
|
3637
|
+
const counts = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
|
|
3638
|
+
return { scanned: batch.length, typed, failedValidation, edgesAdded, remaining: counts.eligible, deferred: counts.deferred };
|
|
3639
|
+
}
|
|
3389
3640
|
_validatePruneDuration(value, name) {
|
|
3390
3641
|
if (value !== null && value !== void 0 && (typeof value !== "number" || !isFinite(value) || value < 0)) {
|
|
3391
3642
|
throw new Error(`Invalid ${name}: must be a non-negative finite number or null`);
|
|
@@ -4685,7 +4936,8 @@ var OntologyService = class {
|
|
|
4685
4936
|
return { okf_type: canonical, edges };
|
|
4686
4937
|
}
|
|
4687
4938
|
async resolveAndPersistEdges(entityId, sourceId, sourceType, edges, manifest, titleIndex, tx, now) {
|
|
4688
|
-
if (!sourceType || edges.length === 0) return;
|
|
4939
|
+
if (!sourceType || edges.length === 0) return 0;
|
|
4940
|
+
let persisted = 0;
|
|
4689
4941
|
for (const edge of edges) {
|
|
4690
4942
|
const def = resolveEdgeDefinition(edge.edge_type, manifest);
|
|
4691
4943
|
if (!def || def.source_type.toLowerCase() !== sourceType.toLowerCase()) continue;
|
|
@@ -4701,8 +4953,10 @@ var OntologyService = class {
|
|
|
4701
4953
|
edge_type: def.type,
|
|
4702
4954
|
created_at: now
|
|
4703
4955
|
};
|
|
4704
|
-
await this.edgeRepo.addIgnoreDuplicate(wikiEdge, tx);
|
|
4956
|
+
const inserted = await this.edgeRepo.addIgnoreDuplicate(wikiEdge, tx);
|
|
4957
|
+
if (inserted) persisted++;
|
|
4705
4958
|
}
|
|
4959
|
+
return persisted;
|
|
4706
4960
|
}
|
|
4707
4961
|
};
|
|
4708
4962
|
|
|
@@ -4942,6 +5196,24 @@ var WikiMemory = class {
|
|
|
4942
5196
|
async runHeal(entityId, options) {
|
|
4943
5197
|
return this.maintenanceService.runHeal(entityId, options);
|
|
4944
5198
|
}
|
|
5199
|
+
/**
|
|
5200
|
+
* Types already-persisted untyped facts (okf_type IS NULL) in place via one
|
|
5201
|
+
* librarian-style LLM call. Strictly additive: never creates, deletes, or
|
|
5202
|
+
* rewrites facts; never overwrites an existing okf_type. Free when there is
|
|
5203
|
+
* nothing to do (ontology off, or zero eligible untyped facts).
|
|
5204
|
+
*
|
|
5205
|
+
* Hosts own the trigger cadence (e.g. after each sync); loop `while
|
|
5206
|
+
* result.remaining > 0` for convergence. Throws WikiBusyError when another
|
|
5207
|
+
* backfill (or conflicting maintenance op) is running for the entity.
|
|
5208
|
+
*
|
|
5209
|
+
* @param options.promptOverride - Applies only to this call. For persistent
|
|
5210
|
+
* customization set `options.config.prompts.ontologyBackfillSystemPrompt`.
|
|
5211
|
+
* @param options.batchSize - Facts per run (default 25) for providers with
|
|
5212
|
+
* tighter context limits.
|
|
5213
|
+
*/
|
|
5214
|
+
async runOntologyBackfill(entityId, options) {
|
|
5215
|
+
return this.maintenanceService.runOntologyBackfill(entityId, options);
|
|
5216
|
+
}
|
|
4945
5217
|
async runReembed(entityId, opts) {
|
|
4946
5218
|
return this.maintenanceService.runReembed(entityId, opts);
|
|
4947
5219
|
}
|
|
@@ -5781,6 +6053,10 @@ function createWiki(db, options) {
|
|
|
5781
6053
|
|
|
5782
6054
|
exports.DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT = DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT;
|
|
5783
6055
|
exports.HOOK_TIMEOUT_MARKER = HOOK_TIMEOUT_MARKER;
|
|
6056
|
+
exports.ONTOLOGY_BACKFILL_BATCH_SIZE = ONTOLOGY_BACKFILL_BATCH_SIZE;
|
|
6057
|
+
exports.ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS = ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS;
|
|
6058
|
+
exports.ONTOLOGY_BACKFILL_RECHECK_MS = ONTOLOGY_BACKFILL_RECHECK_MS;
|
|
6059
|
+
exports.ONTOLOGY_BACKFILL_SYSTEM_PROMPT = ONTOLOGY_BACKFILL_SYSTEM_PROMPT;
|
|
5784
6060
|
exports.PromptService = PromptService;
|
|
5785
6061
|
exports.PrunePartialFailureError = PrunePartialFailureError;
|
|
5786
6062
|
exports.WikiBusyError = WikiBusyError;
|