@equationalapplications/core-llm-wiki 4.20.0 → 4.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -0
- package/dist/{chunk-HKB5FEJG.mjs → chunk-LCAU6NYC.mjs} +270 -94
- package/dist/chunk-LCAU6NYC.mjs.map +1 -0
- package/dist/index.d.mts +7 -3
- package/dist/index.d.ts +7 -3
- package/dist/index.js +328 -32
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +126 -11
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-BVN_g414.d.mts → testing-Bk8J6QLX.d.mts} +100 -7
- package/dist/{testing-BVN_g414.d.ts → testing-Bk8J6QLX.d.ts} +100 -7
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +167 -6
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +2 -2
- package/dist/chunk-HKB5FEJG.mjs.map +0 -1
package/dist/index.d.mts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-
|
|
2
|
-
export { E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HOOK_TIMEOUT_MARKER, L as LLMProvider,
|
|
1
|
+
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, O as OntologyManifest, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-Bk8J6QLX.mjs';
|
|
2
|
+
export { E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HOOK_TIMEOUT_MARKER, L as LLMProvider, i as ONTOLOGY_BACKFILL_BATCH_SIZE, j as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, k as ONTOLOGY_BACKFILL_RECHECK_MS, l as OntologyBackfillResult, m as OntologyConfig, n as OntologyEdgeType, o as OntologyMode, p as OntologyNodeType, q as OntologyPromptContext, r as OntologyUpdates, P as PromptOverrides, s as PromptService, t as PrunePartialFailureError, V as VectorRanker, u as VectorRankerFallback, v as VectorRankerRankArgs, w as VectorRankerSemanticResult, x as WikiBusyError, y as WikiBusyOperation, z as WikiCheckpoint, A as WikiConfig, B as WikiEdge, C as WikiEvent, D as WikiFact, I as WikiMemoryTestAccess, J as WikiOutboxEvent, K as WikiTask, N as WikiTransactionError } from './testing-Bk8J6QLX.mjs';
|
|
3
3
|
import { OkfFile } from '@equationalapplications/core-okf';
|
|
4
4
|
import 'minisearch';
|
|
5
5
|
|
|
@@ -26,6 +26,8 @@ declare function parseOkfBundle(entityId: string, files: OkfFile[], options?: Ok
|
|
|
26
26
|
|
|
27
27
|
declare function parseEmbedding(blob: Uint8Array | null | undefined, text: string | null | undefined): Float32Array | null;
|
|
28
28
|
|
|
29
|
+
declare function validateManifest(manifest: OntologyManifest): void;
|
|
30
|
+
|
|
29
31
|
type GetRandomValues = (bytes: Uint8Array) => void;
|
|
30
32
|
/**
|
|
31
33
|
* Inject a platform-specific `getRandomValues` implementation.
|
|
@@ -58,6 +60,8 @@ declare function validateLibrarianPromptTemplate(template: string, options: {
|
|
|
58
60
|
}): string[];
|
|
59
61
|
declare function mapLibrarianOptionsToReadOptions(options: LibrarianOptions): Pick<ReadOptions, 'tierWeights' | 'includeZeroWeightEntities'>;
|
|
60
62
|
|
|
63
|
+
declare const ONTOLOGY_BACKFILL_SYSTEM_PROMPT = "You are a knowledge classification agent. You will receive existing memory facts that currently have no ontology type. For each input fact { \"id\", \"title\", \"body\", \"tags\" }, assign the best matching okf_type from the ontology manifest and optionally propose edges to related facts by title.\nReturn ONLY a valid JSON object matching this schema:\n{\n \"classifications\": [\n { \"id\": \"string (input fact id, copied verbatim)\", \"okf_type\": \"string (manifest node type slug)\", \"edges\": [{ \"edge_type\": \"string\", \"target_title\": \"string\" }] }\n ]\n}\nIf no manifest type fits a fact, omit that fact from \"classifications\" entirely \u2014 do not guess.\nDo not return markdown, just raw JSON.";
|
|
64
|
+
|
|
61
65
|
declare function createWiki(db: SQLiteAdapter, options: WikiOptions): WikiMemory;
|
|
62
66
|
|
|
63
|
-
export { DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, FormatContextOptions, FormattedMemoryDump, GraphNeighborhood, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, type OkfImportOptions, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, validateLibrarianPromptTemplate };
|
|
67
|
+
export { DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, FormatContextOptions, FormattedMemoryDump, GraphNeighborhood, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, type OkfImportOptions, OntologyManifest, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, validateLibrarianPromptTemplate, validateManifest };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-
|
|
2
|
-
export { E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HOOK_TIMEOUT_MARKER, L as LLMProvider,
|
|
1
|
+
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, O as OntologyManifest, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-Bk8J6QLX.js';
|
|
2
|
+
export { E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HOOK_TIMEOUT_MARKER, L as LLMProvider, i as ONTOLOGY_BACKFILL_BATCH_SIZE, j as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, k as ONTOLOGY_BACKFILL_RECHECK_MS, l as OntologyBackfillResult, m as OntologyConfig, n as OntologyEdgeType, o as OntologyMode, p as OntologyNodeType, q as OntologyPromptContext, r as OntologyUpdates, P as PromptOverrides, s as PromptService, t as PrunePartialFailureError, V as VectorRanker, u as VectorRankerFallback, v as VectorRankerRankArgs, w as VectorRankerSemanticResult, x as WikiBusyError, y as WikiBusyOperation, z as WikiCheckpoint, A as WikiConfig, B as WikiEdge, C as WikiEvent, D as WikiFact, I as WikiMemoryTestAccess, J as WikiOutboxEvent, K as WikiTask, N as WikiTransactionError } from './testing-Bk8J6QLX.js';
|
|
3
3
|
import { OkfFile } from '@equationalapplications/core-okf';
|
|
4
4
|
import 'minisearch';
|
|
5
5
|
|
|
@@ -26,6 +26,8 @@ declare function parseOkfBundle(entityId: string, files: OkfFile[], options?: Ok
|
|
|
26
26
|
|
|
27
27
|
declare function parseEmbedding(blob: Uint8Array | null | undefined, text: string | null | undefined): Float32Array | null;
|
|
28
28
|
|
|
29
|
+
declare function validateManifest(manifest: OntologyManifest): void;
|
|
30
|
+
|
|
29
31
|
type GetRandomValues = (bytes: Uint8Array) => void;
|
|
30
32
|
/**
|
|
31
33
|
* Inject a platform-specific `getRandomValues` implementation.
|
|
@@ -58,6 +60,8 @@ declare function validateLibrarianPromptTemplate(template: string, options: {
|
|
|
58
60
|
}): string[];
|
|
59
61
|
declare function mapLibrarianOptionsToReadOptions(options: LibrarianOptions): Pick<ReadOptions, 'tierWeights' | 'includeZeroWeightEntities'>;
|
|
60
62
|
|
|
63
|
+
declare const ONTOLOGY_BACKFILL_SYSTEM_PROMPT = "You are a knowledge classification agent. You will receive existing memory facts that currently have no ontology type. For each input fact { \"id\", \"title\", \"body\", \"tags\" }, assign the best matching okf_type from the ontology manifest and optionally propose edges to related facts by title.\nReturn ONLY a valid JSON object matching this schema:\n{\n \"classifications\": [\n { \"id\": \"string (input fact id, copied verbatim)\", \"okf_type\": \"string (manifest node type slug)\", \"edges\": [{ \"edge_type\": \"string\", \"target_title\": \"string\" }] }\n ]\n}\nIf no manifest type fits a fact, omit that fact from \"classifications\" entirely \u2014 do not guess.\nDo not return markdown, just raw JSON.";
|
|
64
|
+
|
|
61
65
|
declare function createWiki(db: SQLiteAdapter, options: WikiOptions): WikiMemory;
|
|
62
66
|
|
|
63
|
-
export { DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, FormatContextOptions, FormattedMemoryDump, GraphNeighborhood, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, type OkfImportOptions, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, validateLibrarianPromptTemplate };
|
|
67
|
+
export { DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, FormatContextOptions, FormattedMemoryDump, GraphNeighborhood, type LibrarianOptions, type LibrarianPromptVariables, MemoryBundle, MemoryDump, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, type OkfImportOptions, OntologyManifest, ReadOptions, SQLiteAdapter, WikiMemory, WikiOptions, configureRandomSource, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseEmbedding, parseOkfBundle, validateLibrarianPromptTemplate, validateManifest };
|
package/dist/index.js
CHANGED
|
@@ -35,7 +35,8 @@ async function setupDatabase(db, prefix) {
|
|
|
35
35
|
deleted_at INTEGER,
|
|
36
36
|
embedding TEXT,
|
|
37
37
|
embedding_blob BLOB,
|
|
38
|
-
okf_type TEXT
|
|
38
|
+
okf_type TEXT,
|
|
39
|
+
ontology_checked_at INTEGER
|
|
39
40
|
);
|
|
40
41
|
|
|
41
42
|
CREATE INDEX IF NOT EXISTS ${prefix}entries_entity_idx ON ${prefix}entries(entity_id);
|
|
@@ -338,6 +339,20 @@ var MIGRATIONS = [
|
|
|
338
339
|
);
|
|
339
340
|
`);
|
|
340
341
|
}
|
|
342
|
+
},
|
|
343
|
+
{
|
|
344
|
+
version: 7,
|
|
345
|
+
description: "Add ontology_checked_at to entries for ontology backfill recheck cooldown",
|
|
346
|
+
run: async (db, prefix) => {
|
|
347
|
+
const cols = await db.getAllAsync(
|
|
348
|
+
`PRAGMA table_info(${prefix}entries)`
|
|
349
|
+
);
|
|
350
|
+
if (!cols.some((c) => c.name === "ontology_checked_at")) {
|
|
351
|
+
await db.execAsync(
|
|
352
|
+
`ALTER TABLE ${prefix}entries ADD COLUMN ontology_checked_at INTEGER`
|
|
353
|
+
);
|
|
354
|
+
}
|
|
355
|
+
}
|
|
341
356
|
}
|
|
342
357
|
];
|
|
343
358
|
for (let i = 1; i < MIGRATIONS.length; i++) {
|
|
@@ -1101,6 +1116,79 @@ var EntryRepository = class extends BaseRepository {
|
|
|
1101
1116
|
);
|
|
1102
1117
|
return rows.map(mapRowToFact);
|
|
1103
1118
|
}
|
|
1119
|
+
/**
|
|
1120
|
+
* Live untyped facts eligible for ontology backfill, oldest first.
|
|
1121
|
+
* Skips facts checked within the recheck cooldown (ontology_checked_at > recheckCutoff).
|
|
1122
|
+
*/
|
|
1123
|
+
async findUntypedByEntityId(entityId, limit, recheckCutoff, tx) {
|
|
1124
|
+
const executor = this.getExecutor(tx);
|
|
1125
|
+
const rows = await executor.getAllAsync(
|
|
1126
|
+
`SELECT * FROM ${this.prefix}entries
|
|
1127
|
+
WHERE entity_id = ? AND okf_type IS NULL AND deleted_at IS NULL
|
|
1128
|
+
AND (ontology_checked_at IS NULL OR ontology_checked_at <= ?)
|
|
1129
|
+
ORDER BY updated_at ASC LIMIT ?`,
|
|
1130
|
+
[entityId, recheckCutoff, limit]
|
|
1131
|
+
);
|
|
1132
|
+
return rows.map(mapRowToFact);
|
|
1133
|
+
}
|
|
1134
|
+
/** Counts live untyped facts: eligible (past cooldown) vs deferred (in cooldown). */
|
|
1135
|
+
async countUntypedByEntityId(entityId, recheckCutoff, tx) {
|
|
1136
|
+
const executor = this.getExecutor(tx);
|
|
1137
|
+
const row = await executor.getFirstAsync(
|
|
1138
|
+
`SELECT
|
|
1139
|
+
SUM(CASE WHEN ontology_checked_at IS NULL OR ontology_checked_at <= ? THEN 1 ELSE 0 END) AS eligible,
|
|
1140
|
+
SUM(CASE WHEN ontology_checked_at IS NOT NULL AND ontology_checked_at > ? THEN 1 ELSE 0 END) AS deferred
|
|
1141
|
+
FROM ${this.prefix}entries
|
|
1142
|
+
WHERE entity_id = ? AND okf_type IS NULL AND deleted_at IS NULL`,
|
|
1143
|
+
[recheckCutoff, recheckCutoff, entityId]
|
|
1144
|
+
);
|
|
1145
|
+
return { eligible: Number(row?.eligible ?? 0), deferred: Number(row?.deferred ?? 0) };
|
|
1146
|
+
}
|
|
1147
|
+
/**
|
|
1148
|
+
* Sets okf_type only when currently NULL — additive by construction: a
|
|
1149
|
+
* concurrently-typed fact is never overwritten. Returns changes for the caller
|
|
1150
|
+
* to distinguish applied vs skipped.
|
|
1151
|
+
*/
|
|
1152
|
+
async updateOkfType(id, entityId, okfType, tx) {
|
|
1153
|
+
const result = await tx.runAsync(
|
|
1154
|
+
`UPDATE ${this.prefix}entries SET okf_type = ?
|
|
1155
|
+
WHERE id = ? AND entity_id = ? AND okf_type IS NULL AND deleted_at IS NULL`,
|
|
1156
|
+
[okfType, id, entityId]
|
|
1157
|
+
);
|
|
1158
|
+
return { changes: result.changes };
|
|
1159
|
+
}
|
|
1160
|
+
/**
|
|
1161
|
+
* Stamps the backfill recheck cooldown. NEVER touches updated_at — import
|
|
1162
|
+
* merge resolution is last-write-wins on updated_at and a bump here would
|
|
1163
|
+
* make an unchanged local fact beat a genuinely newer remote edit.
|
|
1164
|
+
*/
|
|
1165
|
+
async markOntologyChecked(ids, entityId, now, tx) {
|
|
1166
|
+
if (ids.length === 0) return;
|
|
1167
|
+
for (let i = 0; i < ids.length; i += this.chunkSize) {
|
|
1168
|
+
const chunk = ids.slice(i, i + this.chunkSize);
|
|
1169
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
1170
|
+
await tx.runAsync(
|
|
1171
|
+
`UPDATE ${this.prefix}entries SET ontology_checked_at = ?
|
|
1172
|
+
WHERE id IN (${placeholders}) AND entity_id = ? AND deleted_at IS NULL`,
|
|
1173
|
+
[now, ...chunk, entityId]
|
|
1174
|
+
);
|
|
1175
|
+
}
|
|
1176
|
+
}
|
|
1177
|
+
/**
|
|
1178
|
+
* Lightweight full-breadth title index (id, title, okf_type) over all live
|
|
1179
|
+
* typed facts. Used by ontology backfill edge resolution — a recent-N window
|
|
1180
|
+
* would miss an old fact's contemporaries. Untyped rows are excluded: they
|
|
1181
|
+
* can never pass the edge target-type check, and batch facts typed during
|
|
1182
|
+
* the run are re-added to the in-memory index by the caller.
|
|
1183
|
+
*/
|
|
1184
|
+
async findTitleIndexByEntityId(entityId, tx) {
|
|
1185
|
+
const executor = this.getExecutor(tx);
|
|
1186
|
+
const rows = await executor.getAllAsync(
|
|
1187
|
+
`SELECT id, title, okf_type FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL AND okf_type IS NOT NULL`,
|
|
1188
|
+
[entityId]
|
|
1189
|
+
);
|
|
1190
|
+
return rows.map((r) => ({ id: r.id, title: r.title, okf_type: r.okf_type ?? null }));
|
|
1191
|
+
}
|
|
1104
1192
|
};
|
|
1105
1193
|
|
|
1106
1194
|
// src/utils/ids.ts
|
|
@@ -1545,6 +1633,7 @@ var CONFIDENCE_RANK = {
|
|
|
1545
1633
|
var EdgeRepository = class extends BaseRepository {
|
|
1546
1634
|
/**
|
|
1547
1635
|
* Insert an edge, silently skipping on primary-key or uniqueness conflicts.
|
|
1636
|
+
* Returns true when a row was inserted, false when skipped as a duplicate.
|
|
1548
1637
|
* Throws when the insert was skipped due to an id collision with a different edge tuple.
|
|
1549
1638
|
*/
|
|
1550
1639
|
async addIgnoreDuplicate(edge, tx) {
|
|
@@ -1554,17 +1643,18 @@ var EdgeRepository = class extends BaseRepository {
|
|
|
1554
1643
|
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
1555
1644
|
[edge.id, edge.entity_id, edge.source_id, edge.target_id, edge.edge_type, edge.created_at]
|
|
1556
1645
|
);
|
|
1557
|
-
if (result.changes > 0) return;
|
|
1646
|
+
if (result.changes > 0) return true;
|
|
1558
1647
|
const existing = await executor.getFirstAsync(
|
|
1559
1648
|
`SELECT entity_id, source_id, target_id, edge_type FROM ${this.prefix}edges WHERE id = ?`,
|
|
1560
1649
|
[edge.id]
|
|
1561
1650
|
);
|
|
1562
|
-
if (!existing) return;
|
|
1651
|
+
if (!existing) return false;
|
|
1563
1652
|
if (String(existing.entity_id) !== edge.entity_id || String(existing.source_id) !== edge.source_id || String(existing.target_id) !== edge.target_id || String(existing.edge_type) !== edge.edge_type) {
|
|
1564
1653
|
throw new Error(
|
|
1565
1654
|
`Edge id collision: ${JSON.stringify(edge.id)} already exists with a different (entity_id, source_id, target_id, edge_type) tuple`
|
|
1566
1655
|
);
|
|
1567
1656
|
}
|
|
1657
|
+
return false;
|
|
1568
1658
|
}
|
|
1569
1659
|
async getByEntityId(entityId, tx) {
|
|
1570
1660
|
const executor = this.getExecutor(tx);
|
|
@@ -1691,10 +1781,13 @@ function resolveNodeType(raw, manifest) {
|
|
|
1691
1781
|
const hit = manifest.node_types.find((n) => n.type.toLowerCase() === slug.toLowerCase());
|
|
1692
1782
|
return hit?.type ?? null;
|
|
1693
1783
|
}
|
|
1694
|
-
function
|
|
1784
|
+
function resolveEdgeDefinitions(rawEdgeType, manifest) {
|
|
1695
1785
|
const slug = rawEdgeType.trim();
|
|
1696
|
-
if (!slug) return
|
|
1697
|
-
return manifest.edge_types.
|
|
1786
|
+
if (!slug) return [];
|
|
1787
|
+
return manifest.edge_types.filter((e) => e.type.toLowerCase() === slug.toLowerCase());
|
|
1788
|
+
}
|
|
1789
|
+
function edgeTripleKey(type, sourceType, targetType) {
|
|
1790
|
+
return `${type.trim().toLowerCase()}|${sourceType.trim().toLowerCase()}|${targetType.trim().toLowerCase()}`;
|
|
1698
1791
|
}
|
|
1699
1792
|
function validateManifest(manifest) {
|
|
1700
1793
|
const nodeSlugs = /* @__PURE__ */ new Set();
|
|
@@ -1705,25 +1798,35 @@ function validateManifest(manifest) {
|
|
|
1705
1798
|
if (nodeSlugs.has(key)) throw new Error(`Duplicate node type: ${type}`);
|
|
1706
1799
|
nodeSlugs.add(key);
|
|
1707
1800
|
}
|
|
1708
|
-
const
|
|
1801
|
+
const edgeKeys = /* @__PURE__ */ new Set();
|
|
1802
|
+
const edgeNames = /* @__PURE__ */ new Map();
|
|
1709
1803
|
for (const edge of manifest.edge_types ?? []) {
|
|
1710
1804
|
const edgeType = edge.type?.trim();
|
|
1711
1805
|
const sourceType = edge.source_type?.trim();
|
|
1712
1806
|
const targetType = edge.target_type?.trim();
|
|
1713
1807
|
if (!edgeType) throw new Error("Ontology edge type slug must be non-empty");
|
|
1714
|
-
const edgeKey = edgeType.toLowerCase();
|
|
1715
|
-
if (edgeSlugs.has(edgeKey)) throw new Error(`Duplicate edge type: ${edgeType}`);
|
|
1716
|
-
edgeSlugs.add(edgeKey);
|
|
1717
1808
|
if (!sourceType || !targetType || !nodeSlugs.has(sourceType.toLowerCase()) || !nodeSlugs.has(targetType.toLowerCase())) {
|
|
1718
1809
|
throw new Error(`Edge type ${edgeType} references unknown node type`);
|
|
1719
1810
|
}
|
|
1811
|
+
const edgeKey = edgeTripleKey(edgeType, sourceType, targetType);
|
|
1812
|
+
if (edgeKeys.has(edgeKey)) {
|
|
1813
|
+
throw new Error(`Duplicate edge definition: ${edgeType} (${sourceType} \u2192 ${targetType})`);
|
|
1814
|
+
}
|
|
1815
|
+
edgeKeys.add(edgeKey);
|
|
1816
|
+
const canonical = edgeNames.get(edgeType.toLowerCase());
|
|
1817
|
+
if (canonical === void 0) {
|
|
1818
|
+
edgeNames.set(edgeType.toLowerCase(), edgeType);
|
|
1819
|
+
} else if (canonical !== edgeType) {
|
|
1820
|
+
throw new Error(`Inconsistent casing for edge type: ${edgeType} conflicts with ${canonical}`);
|
|
1821
|
+
}
|
|
1720
1822
|
}
|
|
1721
1823
|
}
|
|
1722
1824
|
function mergeOntologyUpdates(current, updates) {
|
|
1723
1825
|
const node_types = [...current.node_types];
|
|
1724
1826
|
const edge_types = [...current.edge_types];
|
|
1725
1827
|
const nodeSlugs = new Set(node_types.map((n) => n.type.trim().toLowerCase()));
|
|
1726
|
-
const
|
|
1828
|
+
const edgeKeys = new Set(edge_types.map((e) => edgeTripleKey(e.type, e.source_type, e.target_type)));
|
|
1829
|
+
const edgeNames = new Map(edge_types.map((e) => [e.type.trim().toLowerCase(), e.type.trim()]));
|
|
1727
1830
|
for (const node of updates.node_types ?? []) {
|
|
1728
1831
|
const type = node?.type?.trim();
|
|
1729
1832
|
if (!type) continue;
|
|
@@ -1733,20 +1836,22 @@ function mergeOntologyUpdates(current, updates) {
|
|
|
1733
1836
|
nodeSlugs.add(key);
|
|
1734
1837
|
}
|
|
1735
1838
|
for (const edge of updates.edge_types ?? []) {
|
|
1736
|
-
const
|
|
1839
|
+
const rawEdgeType = edge?.type?.trim();
|
|
1737
1840
|
const sourceType = edge?.source_type?.trim();
|
|
1738
1841
|
const targetType = edge?.target_type?.trim();
|
|
1739
|
-
if (!
|
|
1740
|
-
const
|
|
1741
|
-
|
|
1842
|
+
if (!rawEdgeType || !sourceType || !targetType) continue;
|
|
1843
|
+
const edgeType = edgeNames.get(rawEdgeType.toLowerCase()) ?? rawEdgeType;
|
|
1844
|
+
const edgeKey = edgeTripleKey(edgeType, sourceType, targetType);
|
|
1845
|
+
if (edgeKeys.has(edgeKey)) continue;
|
|
1742
1846
|
if (!nodeSlugs.has(sourceType.toLowerCase()) || !nodeSlugs.has(targetType.toLowerCase())) continue;
|
|
1847
|
+
edgeNames.set(edgeType.toLowerCase(), edgeType);
|
|
1743
1848
|
edge_types.push({
|
|
1744
1849
|
type: edgeType,
|
|
1745
1850
|
source_type: sourceType,
|
|
1746
1851
|
target_type: targetType,
|
|
1747
1852
|
description: String(edge.description ?? "")
|
|
1748
1853
|
});
|
|
1749
|
-
|
|
1854
|
+
edgeKeys.add(edgeKey);
|
|
1750
1855
|
}
|
|
1751
1856
|
return { node_types, edge_types };
|
|
1752
1857
|
}
|
|
@@ -1755,10 +1860,10 @@ function validateInlineEdges(sourceType, _targetType, edges, manifest) {
|
|
|
1755
1860
|
const valid = [];
|
|
1756
1861
|
for (const edge of edges) {
|
|
1757
1862
|
if (typeof edge?.edge_type !== "string" || typeof edge?.target_title !== "string") continue;
|
|
1758
|
-
const
|
|
1759
|
-
|
|
1760
|
-
if (
|
|
1761
|
-
valid.push({ edge_type:
|
|
1863
|
+
const defs = resolveEdgeDefinitions(edge.edge_type, manifest);
|
|
1864
|
+
const match = defs.find((d) => d.source_type.toLowerCase() === sourceType.toLowerCase());
|
|
1865
|
+
if (!match) continue;
|
|
1866
|
+
valid.push({ edge_type: match.type, target_title: edge.target_title });
|
|
1762
1867
|
}
|
|
1763
1868
|
return valid;
|
|
1764
1869
|
}
|
|
@@ -2176,7 +2281,8 @@ var JobManager = class {
|
|
|
2176
2281
|
heal: (id) => this._healKey(id),
|
|
2177
2282
|
reembed: (id) => this._reembedKey(id),
|
|
2178
2283
|
import: (id) => this._importKey(id),
|
|
2179
|
-
forget: (id) => this._forgetKey(id)
|
|
2284
|
+
forget: (id) => this._forgetKey(id),
|
|
2285
|
+
ontologyBackfill: (id) => this._ontologyBackfillKey(id)
|
|
2180
2286
|
};
|
|
2181
2287
|
}
|
|
2182
2288
|
_pruneKey(entityId) {
|
|
@@ -2203,6 +2309,9 @@ var JobManager = class {
|
|
|
2203
2309
|
_healKey(entityId) {
|
|
2204
2310
|
return `${this.prefix}:${entityId}:heal`;
|
|
2205
2311
|
}
|
|
2312
|
+
_ontologyBackfillKey(entityId) {
|
|
2313
|
+
return `${this.prefix}:${entityId}:ontologyBackfill`;
|
|
2314
|
+
}
|
|
2206
2315
|
_isReembedActive(entityId) {
|
|
2207
2316
|
return this.activeMaintenanceJobs.has(this._reembedKey(entityId)) || this.activeMaintenanceJobs.has(this._globalReembedKey());
|
|
2208
2317
|
}
|
|
@@ -2253,14 +2362,16 @@ var JobManager = class {
|
|
|
2253
2362
|
if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
|
|
2254
2363
|
else if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) blockingOperation = "librarian";
|
|
2255
2364
|
else if (this.activeMaintenanceJobs.has(this._healKey(entityId))) blockingOperation = "heal";
|
|
2365
|
+
else if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) blockingOperation = "ontologyBackfill";
|
|
2256
2366
|
else if (this._isReembedActive(entityId)) blockingOperation = "reembed";
|
|
2257
2367
|
else if (this._isIngestActiveFor(entityId)) blockingOperation = "ingest";
|
|
2258
2368
|
else if (this._isImportActiveFor(entityId)) blockingOperation = "import";
|
|
2259
2369
|
else if (this._isForgetActiveFor(entityId)) blockingOperation = "forget";
|
|
2260
2370
|
break;
|
|
2261
2371
|
case "librarian":
|
|
2262
|
-
case "heal":
|
|
2263
|
-
|
|
2372
|
+
case "heal":
|
|
2373
|
+
case "ontologyBackfill": {
|
|
2374
|
+
const opKey = this.lockKeyFns[operation](entityId);
|
|
2264
2375
|
if (this.activeMaintenanceJobs.has(opKey)) blockingOperation = operation;
|
|
2265
2376
|
else if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
|
|
2266
2377
|
else if (this._isReembedActive(entityId)) blockingOperation = "reembed";
|
|
@@ -2274,6 +2385,7 @@ var JobManager = class {
|
|
|
2274
2385
|
else if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
|
|
2275
2386
|
else if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) blockingOperation = "librarian";
|
|
2276
2387
|
else if (this.activeMaintenanceJobs.has(this._healKey(entityId))) blockingOperation = "heal";
|
|
2388
|
+
else if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) blockingOperation = "ontologyBackfill";
|
|
2277
2389
|
else if (this._isIngestActiveFor(entityId)) blockingOperation = "ingest";
|
|
2278
2390
|
else if (this._isImportActiveFor(entityId)) blockingOperation = "import";
|
|
2279
2391
|
else if (this._isForgetActiveFor(entityId)) blockingOperation = "forget";
|
|
@@ -2284,6 +2396,7 @@ var JobManager = class {
|
|
|
2284
2396
|
else if (this._isAnyMaintenanceActiveWithSuffix(":prune")) blockingOperation = "prune";
|
|
2285
2397
|
else if (this._isAnyMaintenanceActiveWithSuffix(":librarian")) blockingOperation = "librarian";
|
|
2286
2398
|
else if (this._isAnyMaintenanceActiveWithSuffix(":heal")) blockingOperation = "heal";
|
|
2399
|
+
else if (this._isAnyMaintenanceActiveWithSuffix(":ontologyBackfill")) blockingOperation = "ontologyBackfill";
|
|
2287
2400
|
else if (this.activeIngestJobs.size > 0) blockingOperation = "ingest";
|
|
2288
2401
|
else if (this._isAnyMaintenanceActiveWithSuffix(":import")) blockingOperation = "import";
|
|
2289
2402
|
else if (this._isAnyMaintenanceActiveWithSuffix(":forget")) blockingOperation = "forget";
|
|
@@ -2294,6 +2407,7 @@ var JobManager = class {
|
|
|
2294
2407
|
if (this.activeMaintenanceJobs.has(selfKey)) blockingOperation = operation;
|
|
2295
2408
|
else if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) blockingOperation = "librarian";
|
|
2296
2409
|
else if (this.activeMaintenanceJobs.has(this._healKey(entityId))) blockingOperation = "heal";
|
|
2410
|
+
else if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) blockingOperation = "ontologyBackfill";
|
|
2297
2411
|
else if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
|
|
2298
2412
|
else if (this._isReembedActive(entityId)) blockingOperation = "reembed";
|
|
2299
2413
|
else if (this._isIngestActiveFor(entityId)) blockingOperation = "ingest";
|
|
@@ -2351,11 +2465,11 @@ var JobManager = class {
|
|
|
2351
2465
|
if (operation !== "global_import" && this.activeMaintenanceJobs.has(this._globalImportKey())) return true;
|
|
2352
2466
|
switch (operation) {
|
|
2353
2467
|
case "librarian":
|
|
2354
|
-
return this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
2355
2468
|
case "heal":
|
|
2356
|
-
|
|
2469
|
+
case "ontologyBackfill":
|
|
2470
|
+
return this.activeMaintenanceJobs.has(this.lockKeyFns[operation](entityId)) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
2357
2471
|
case "prune":
|
|
2358
|
-
return this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._healKey(entityId)) || this._isReembedActive(entityId) || this._isIngestActiveFor(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
2472
|
+
return this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._healKey(entityId)) || this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId)) || this._isReembedActive(entityId) || this._isIngestActiveFor(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
2359
2473
|
default:
|
|
2360
2474
|
return false;
|
|
2361
2475
|
}
|
|
@@ -2380,6 +2494,7 @@ var JobManager = class {
|
|
|
2380
2494
|
if (this.activeMaintenanceJobs.has(this._importKey(entityId))) throw new WikiBusyError("import", entityId);
|
|
2381
2495
|
if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) throw new WikiBusyError("librarian", entityId);
|
|
2382
2496
|
if (this.activeMaintenanceJobs.has(this._healKey(entityId))) throw new WikiBusyError("heal", entityId);
|
|
2497
|
+
if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) throw new WikiBusyError("ontologyBackfill", entityId);
|
|
2383
2498
|
if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) throw new WikiBusyError("prune", entityId);
|
|
2384
2499
|
if (this._isReembedActive(entityId)) throw new WikiBusyError("reembed", entityId);
|
|
2385
2500
|
if (this._isIngestActiveFor(entityId)) throw new WikiBusyError("ingest", entityId);
|
|
@@ -2688,6 +2803,15 @@ Return ONLY a valid JSON object matching this schema:
|
|
|
2688
2803
|
"facts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }]
|
|
2689
2804
|
}
|
|
2690
2805
|
Extract verbatim factual content. Do not return markdown, just raw JSON.`;
|
|
2806
|
+
var ONTOLOGY_BACKFILL_SYSTEM_PROMPT = `You are a knowledge classification agent. You will receive existing memory facts that currently have no ontology type. For each input fact { "id", "title", "body", "tags" }, assign the best matching okf_type from the ontology manifest and optionally propose edges to related facts by title.
|
|
2807
|
+
Return ONLY a valid JSON object matching this schema:
|
|
2808
|
+
{
|
|
2809
|
+
"classifications": [
|
|
2810
|
+
{ "id": "string (input fact id, copied verbatim)", "okf_type": "string (manifest node type slug)", "edges": [{ "edge_type": "string", "target_title": "string" }] }
|
|
2811
|
+
]
|
|
2812
|
+
}
|
|
2813
|
+
If no manifest type fits a fact, omit that fact from "classifications" entirely \u2014 do not guess.
|
|
2814
|
+
Do not return markdown, just raw JSON.`;
|
|
2691
2815
|
|
|
2692
2816
|
// src/services/PromptService.ts
|
|
2693
2817
|
var PromptService = class {
|
|
@@ -2777,6 +2901,22 @@ ${JSON.stringify(recentEvents, null, 2)}
|
|
|
2777
2901
|
The following document anchors are provided for contradiction detection only. Do not include them in \`downgraded\`, \`deleted\`, or \`newFacts\`.`
|
|
2778
2902
|
};
|
|
2779
2903
|
}
|
|
2904
|
+
buildOntologyBackfillPrompt(facts, runtimeOverride, ontologyContext) {
|
|
2905
|
+
const template = runtimeOverride ?? this.globalOverrides?.ontologyBackfillSystemPrompt ?? ONTOLOGY_BACKFILL_SYSTEM_PROMPT;
|
|
2906
|
+
const hasFacts = /\{\{\s*facts\s*\}\}/.test(template);
|
|
2907
|
+
if (hasFacts || this.hasOntologyPlaceholders(template)) {
|
|
2908
|
+
return {
|
|
2909
|
+
systemPrompt: this.buildSystemPrompt(template, { facts }, ontologyContext),
|
|
2910
|
+
userPrompt: hasFacts ? "Please classify the facts." : `Facts:
|
|
2911
|
+
${JSON.stringify(facts, null, 2)}`
|
|
2912
|
+
};
|
|
2913
|
+
}
|
|
2914
|
+
return {
|
|
2915
|
+
systemPrompt: this.appendOntology(template, ontologyContext),
|
|
2916
|
+
userPrompt: `Facts:
|
|
2917
|
+
${JSON.stringify(facts, null, 2)}`
|
|
2918
|
+
};
|
|
2919
|
+
}
|
|
2780
2920
|
};
|
|
2781
2921
|
|
|
2782
2922
|
// src/services/IngestionService.ts
|
|
@@ -2930,6 +3070,9 @@ var IngestionService = class {
|
|
|
2930
3070
|
// src/services/MaintenanceService.ts
|
|
2931
3071
|
var FUZZY_THRESHOLD = 0.5;
|
|
2932
3072
|
var MIN_TOKENS_TO_QUALIFY = 3;
|
|
3073
|
+
var ONTOLOGY_BACKFILL_BATCH_SIZE = 25;
|
|
3074
|
+
var ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS = 4e4;
|
|
3075
|
+
var ONTOLOGY_BACKFILL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
2933
3076
|
var MaintenanceService = class {
|
|
2934
3077
|
constructor(db, prefix, options, entryRepo, taskRepo, eventRepo, metadataRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
2935
3078
|
this.db = db;
|
|
@@ -3035,6 +3178,14 @@ var MaintenanceService = class {
|
|
|
3035
3178
|
this.jobManager.releaseLock("heal", entityId);
|
|
3036
3179
|
}
|
|
3037
3180
|
}
|
|
3181
|
+
async runOntologyBackfill(entityId, options) {
|
|
3182
|
+
this.jobManager.acquireLock("ontologyBackfill", entityId);
|
|
3183
|
+
try {
|
|
3184
|
+
return await this.doRunOntologyBackfill(entityId, options);
|
|
3185
|
+
} finally {
|
|
3186
|
+
this.jobManager.releaseLock("ontologyBackfill", entityId);
|
|
3187
|
+
}
|
|
3188
|
+
}
|
|
3038
3189
|
async runReembed(entityId, opts) {
|
|
3039
3190
|
const embedFn = this.options.llmProvider.embed;
|
|
3040
3191
|
if (!embedFn) return { embedded: 0, skipped: 0, failed: 0 };
|
|
@@ -3386,6 +3537,121 @@ var MaintenanceService = class {
|
|
|
3386
3537
|
}
|
|
3387
3538
|
this.searchService.evictCache(entityId);
|
|
3388
3539
|
}
|
|
3540
|
+
/** Core ontology backfill pass (locks handled by {@link runOntologyBackfill}). Package-internal orchestration hook. */
|
|
3541
|
+
async doRunOntologyBackfill(entityId, options) {
|
|
3542
|
+
const batchSize = options?.batchSize ?? ONTOLOGY_BACKFILL_BATCH_SIZE;
|
|
3543
|
+
if (!Number.isInteger(batchSize) || batchSize < 1) {
|
|
3544
|
+
throw new Error("Invalid batchSize: must be an integer >= 1");
|
|
3545
|
+
}
|
|
3546
|
+
const now = Date.now();
|
|
3547
|
+
const recheckCutoff = now - ONTOLOGY_BACKFILL_RECHECK_MS;
|
|
3548
|
+
const zeroed = { scanned: 0, typed: 0, failedValidation: 0, edgesAdded: 0 };
|
|
3549
|
+
const ontologyService = this.ontologyService;
|
|
3550
|
+
if (!ontologyService) {
|
|
3551
|
+
return { ...zeroed, remaining: 0, deferred: 0 };
|
|
3552
|
+
}
|
|
3553
|
+
const { mode } = await ontologyService.getEffectiveState(entityId);
|
|
3554
|
+
if (mode === "off") {
|
|
3555
|
+
const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
|
|
3556
|
+
return { ...zeroed, remaining: 0, deferred: counts2.deferred };
|
|
3557
|
+
}
|
|
3558
|
+
const candidates = await this.entryRepo.findUntypedByEntityId(entityId, batchSize, recheckCutoff);
|
|
3559
|
+
if (candidates.length === 0) {
|
|
3560
|
+
const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
|
|
3561
|
+
return { ...zeroed, remaining: counts2.eligible, deferred: counts2.deferred };
|
|
3562
|
+
}
|
|
3563
|
+
const ontologyContext = await ontologyService.buildPromptContext(entityId);
|
|
3564
|
+
const toPromptShape = (f) => ({ id: f.id, title: f.title, body: f.body, tags: f.tags });
|
|
3565
|
+
const buildPrompt = (facts) => this.promptService.buildOntologyBackfillPrompt(
|
|
3566
|
+
facts.map(toPromptShape),
|
|
3567
|
+
options?.promptOverride,
|
|
3568
|
+
ontologyContext
|
|
3569
|
+
);
|
|
3570
|
+
const batch = [candidates[0]];
|
|
3571
|
+
let built = buildPrompt(batch);
|
|
3572
|
+
for (let i = 1; i < candidates.length; i++) {
|
|
3573
|
+
const next = buildPrompt([...batch, candidates[i]]);
|
|
3574
|
+
if (next.systemPrompt.length + next.userPrompt.length > ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS) break;
|
|
3575
|
+
batch.push(candidates[i]);
|
|
3576
|
+
built = next;
|
|
3577
|
+
}
|
|
3578
|
+
const { systemPrompt, userPrompt } = built;
|
|
3579
|
+
const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
|
|
3580
|
+
const parsed = parseJsonResponse(responseText);
|
|
3581
|
+
const classifications = Array.isArray(parsed.classifications) ? parsed.classifications : [];
|
|
3582
|
+
let typed = 0;
|
|
3583
|
+
let failedValidation = 0;
|
|
3584
|
+
let edgesAdded = 0;
|
|
3585
|
+
let abortedOntologyOff = false;
|
|
3586
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
3587
|
+
let { mode: txMode, manifest } = await ontologyService.getEffectiveState(entityId, tx);
|
|
3588
|
+
if (txMode === "off") {
|
|
3589
|
+
abortedOntologyOff = true;
|
|
3590
|
+
return;
|
|
3591
|
+
}
|
|
3592
|
+
if (txMode === "emergent" && parsed.ontology_updates) {
|
|
3593
|
+
manifest = await ontologyService.mergeEmergentUpdates(entityId, parsed.ontology_updates, tx);
|
|
3594
|
+
}
|
|
3595
|
+
const titleRows = await this.entryRepo.findTitleIndexByEntityId(entityId, tx);
|
|
3596
|
+
const titleIndex = /* @__PURE__ */ new Map();
|
|
3597
|
+
for (const row of titleRows) {
|
|
3598
|
+
titleIndex.set(normalizeTitleKey(row.title), { id: row.id, okf_type: row.okf_type });
|
|
3599
|
+
}
|
|
3600
|
+
const batchById = new Map(batch.map((f) => [f.id, f]));
|
|
3601
|
+
const applied = /* @__PURE__ */ new Set();
|
|
3602
|
+
const pendingEdges = [];
|
|
3603
|
+
for (const classification of classifications) {
|
|
3604
|
+
const fact = typeof classification.id === "string" ? batchById.get(classification.id) : void 0;
|
|
3605
|
+
if (!fact || applied.has(fact.id)) {
|
|
3606
|
+
failedValidation++;
|
|
3607
|
+
continue;
|
|
3608
|
+
}
|
|
3609
|
+
applied.add(fact.id);
|
|
3610
|
+
const normalized = ontologyService.validateAndNormalizeFact(
|
|
3611
|
+
{
|
|
3612
|
+
title: fact.title,
|
|
3613
|
+
body: fact.body,
|
|
3614
|
+
tags: fact.tags,
|
|
3615
|
+
confidence: fact.confidence,
|
|
3616
|
+
okf_type: classification.okf_type,
|
|
3617
|
+
edges: classification.edges
|
|
3618
|
+
},
|
|
3619
|
+
manifest
|
|
3620
|
+
);
|
|
3621
|
+
if (!normalized.okf_type) {
|
|
3622
|
+
failedValidation++;
|
|
3623
|
+
continue;
|
|
3624
|
+
}
|
|
3625
|
+
const result = await this.entryRepo.updateOkfType(fact.id, entityId, normalized.okf_type, tx);
|
|
3626
|
+
if (result.changes === 0) continue;
|
|
3627
|
+
typed++;
|
|
3628
|
+
titleIndex.set(normalizeTitleKey(fact.title), { id: fact.id, okf_type: normalized.okf_type });
|
|
3629
|
+
if (normalized.edges.length > 0) {
|
|
3630
|
+
pendingEdges.push({ sourceId: fact.id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
3631
|
+
}
|
|
3632
|
+
}
|
|
3633
|
+
for (const item of pendingEdges) {
|
|
3634
|
+
edgesAdded += await ontologyService.resolveAndPersistEdges(
|
|
3635
|
+
entityId,
|
|
3636
|
+
item.sourceId,
|
|
3637
|
+
item.sourceType,
|
|
3638
|
+
item.edges,
|
|
3639
|
+
manifest,
|
|
3640
|
+
titleIndex,
|
|
3641
|
+
tx,
|
|
3642
|
+
now
|
|
3643
|
+
);
|
|
3644
|
+
}
|
|
3645
|
+
await this.entryRepo.markOntologyChecked(batch.map((f) => f.id), entityId, now, tx);
|
|
3646
|
+
});
|
|
3647
|
+
if (abortedOntologyOff) {
|
|
3648
|
+
const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
|
|
3649
|
+
return { ...zeroed, remaining: 0, deferred: counts2.deferred };
|
|
3650
|
+
}
|
|
3651
|
+
this.searchService.evictCache(entityId);
|
|
3652
|
+
const counts = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
|
|
3653
|
+
return { scanned: batch.length, typed, failedValidation, edgesAdded, remaining: counts.eligible, deferred: counts.deferred };
|
|
3654
|
+
}
|
|
3389
3655
|
_validatePruneDuration(value, name) {
|
|
3390
3656
|
if (value !== null && value !== void 0 && (typeof value !== "number" || !isFinite(value) || value < 0)) {
|
|
3391
3657
|
throw new Error(`Invalid ${name}: must be a non-negative finite number or null`);
|
|
@@ -4603,7 +4869,8 @@ var WriteService = class {
|
|
|
4603
4869
|
var FACT_ONTOLOGY_FIELDS = `
|
|
4604
4870
|
Each fact may optionally include:
|
|
4605
4871
|
- "okf_type": string \u2014 must be one of the node_types in the manifest
|
|
4606
|
-
- "edges": [{ "edge_type": string, "target_title": string }] \u2014 target_title must match another fact's title in this response or existing memory
|
|
4872
|
+
- "edges": [{ "edge_type": string, "target_title": string }] \u2014 target_title must match another fact's title in this response or existing memory
|
|
4873
|
+
- An edge_type may appear in the manifest multiple times with different source_type/target_type; use the row whose types match your fact and target.`;
|
|
4607
4874
|
var EMERGENT_EXTRA = `
|
|
4608
4875
|
You may also return "ontology_updates" to propose new types:
|
|
4609
4876
|
"ontology_updates": {
|
|
@@ -4685,14 +4952,18 @@ var OntologyService = class {
|
|
|
4685
4952
|
return { okf_type: canonical, edges };
|
|
4686
4953
|
}
|
|
4687
4954
|
async resolveAndPersistEdges(entityId, sourceId, sourceType, edges, manifest, titleIndex, tx, now) {
|
|
4688
|
-
if (!sourceType || edges.length === 0) return;
|
|
4955
|
+
if (!sourceType || edges.length === 0) return 0;
|
|
4956
|
+
let persisted = 0;
|
|
4689
4957
|
for (const edge of edges) {
|
|
4690
|
-
const
|
|
4691
|
-
if (
|
|
4958
|
+
const candidates = resolveEdgeDefinitions(edge.edge_type, manifest).filter((d) => d.source_type.toLowerCase() === sourceType.toLowerCase());
|
|
4959
|
+
if (candidates.length === 0) continue;
|
|
4692
4960
|
const targetKey = normalizeTitleKey(edge.target_title);
|
|
4693
4961
|
const target = titleIndex.get(targetKey);
|
|
4694
4962
|
if (!target) continue;
|
|
4695
|
-
|
|
4963
|
+
const def = candidates.find(
|
|
4964
|
+
(d) => d.target_type.toLowerCase() === (target.okf_type ?? "").toLowerCase()
|
|
4965
|
+
);
|
|
4966
|
+
if (!def) continue;
|
|
4696
4967
|
const wikiEdge = {
|
|
4697
4968
|
id: generateId(),
|
|
4698
4969
|
entity_id: entityId,
|
|
@@ -4701,8 +4972,10 @@ var OntologyService = class {
|
|
|
4701
4972
|
edge_type: def.type,
|
|
4702
4973
|
created_at: now
|
|
4703
4974
|
};
|
|
4704
|
-
await this.edgeRepo.addIgnoreDuplicate(wikiEdge, tx);
|
|
4975
|
+
const inserted = await this.edgeRepo.addIgnoreDuplicate(wikiEdge, tx);
|
|
4976
|
+
if (inserted) persisted++;
|
|
4705
4977
|
}
|
|
4978
|
+
return persisted;
|
|
4706
4979
|
}
|
|
4707
4980
|
};
|
|
4708
4981
|
|
|
@@ -4942,6 +5215,24 @@ var WikiMemory = class {
|
|
|
4942
5215
|
async runHeal(entityId, options) {
|
|
4943
5216
|
return this.maintenanceService.runHeal(entityId, options);
|
|
4944
5217
|
}
|
|
5218
|
+
/**
|
|
5219
|
+
* Types already-persisted untyped facts (okf_type IS NULL) in place via one
|
|
5220
|
+
* librarian-style LLM call. Strictly additive: never creates, deletes, or
|
|
5221
|
+
* rewrites facts; never overwrites an existing okf_type. Free when there is
|
|
5222
|
+
* nothing to do (ontology off, or zero eligible untyped facts).
|
|
5223
|
+
*
|
|
5224
|
+
* Hosts own the trigger cadence (e.g. after each sync); loop `while
|
|
5225
|
+
* result.remaining > 0` for convergence. Throws WikiBusyError when another
|
|
5226
|
+
* backfill (or conflicting maintenance op) is running for the entity.
|
|
5227
|
+
*
|
|
5228
|
+
* @param options.promptOverride - Applies only to this call. For persistent
|
|
5229
|
+
* customization set `options.config.prompts.ontologyBackfillSystemPrompt`.
|
|
5230
|
+
* @param options.batchSize - Facts per run (default 25) for providers with
|
|
5231
|
+
* tighter context limits.
|
|
5232
|
+
*/
|
|
5233
|
+
async runOntologyBackfill(entityId, options) {
|
|
5234
|
+
return this.maintenanceService.runOntologyBackfill(entityId, options);
|
|
5235
|
+
}
|
|
4945
5236
|
async runReembed(entityId, opts) {
|
|
4946
5237
|
return this.maintenanceService.runReembed(entityId, opts);
|
|
4947
5238
|
}
|
|
@@ -5781,6 +6072,10 @@ function createWiki(db, options) {
|
|
|
5781
6072
|
|
|
5782
6073
|
exports.DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT = DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT;
|
|
5783
6074
|
exports.HOOK_TIMEOUT_MARKER = HOOK_TIMEOUT_MARKER;
|
|
6075
|
+
exports.ONTOLOGY_BACKFILL_BATCH_SIZE = ONTOLOGY_BACKFILL_BATCH_SIZE;
|
|
6076
|
+
exports.ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS = ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS;
|
|
6077
|
+
exports.ONTOLOGY_BACKFILL_RECHECK_MS = ONTOLOGY_BACKFILL_RECHECK_MS;
|
|
6078
|
+
exports.ONTOLOGY_BACKFILL_SYSTEM_PROMPT = ONTOLOGY_BACKFILL_SYSTEM_PROMPT;
|
|
5784
6079
|
exports.PromptService = PromptService;
|
|
5785
6080
|
exports.PrunePartialFailureError = PrunePartialFailureError;
|
|
5786
6081
|
exports.WikiBusyError = WikiBusyError;
|
|
@@ -5797,5 +6092,6 @@ exports.mapLibrarianOptionsToReadOptions = mapLibrarianOptionsToReadOptions;
|
|
|
5797
6092
|
exports.parseEmbedding = parseEmbedding;
|
|
5798
6093
|
exports.parseOkfBundle = parseOkfBundle;
|
|
5799
6094
|
exports.validateLibrarianPromptTemplate = validateLibrarianPromptTemplate;
|
|
6095
|
+
exports.validateManifest = validateManifest;
|
|
5800
6096
|
//# sourceMappingURL=index.js.map
|
|
5801
6097
|
//# sourceMappingURL=index.js.map
|