@equationalapplications/core-llm-wiki 5.2.1 → 5.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-6HNIOKT4.mjs → chunk-KIJINNYU.mjs} +213 -41
- package/dist/chunk-KIJINNYU.mjs.map +1 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +370 -50
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +162 -16
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-CAk9oFvw.d.mts → testing-zf3Jp_RL.d.mts} +228 -24
- package/dist/{testing-CAk9oFvw.d.ts → testing-zf3Jp_RL.d.ts} +228 -24
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +170 -31
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +2 -2
- package/dist/chunk-6HNIOKT4.mjs.map +0 -1
package/dist/index.d.mts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, O as OntologyManifest, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-
|
|
2
|
-
export { E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HEAL_BATCH_SIZE, i as HEAL_RECHECK_MS, j as HOOK_TIMEOUT_MARKER, k as HealResult, L as LLMProvider, l as ONTOLOGY_BACKFILL_BATCH_SIZE, m as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, n as ONTOLOGY_BACKFILL_RECHECK_MS, o as OntologyBackfillResult, p as OntologyConfig, q as OntologyEdgeType, r as OntologyMode, s as OntologyNodeType, t as OntologyPromptContext, u as OntologyUpdates, P as PromptOverrides, v as PromptService, w as PrunePartialFailureError, V as VectorRanker, x as VectorRankerFallback, y as VectorRankerRankArgs, z as VectorRankerSemanticResult, A as WikiBusyError, B as WikiBusyOperation, C as WikiCheckpoint, D as WikiConfig, I as WikiDuplicateHashError, J as WikiEdge, K as WikiEvent, N as WikiFact, Q as WikiMemoryTestAccess, T as WikiOutboxEvent, U as
|
|
1
|
+
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, O as OntologyManifest, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-zf3Jp_RL.mjs';
|
|
2
|
+
export { E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HEAL_BATCH_SIZE, i as HEAL_RECHECK_MS, j as HOOK_TIMEOUT_MARKER, k as HealResult, L as LLMProvider, l as ONTOLOGY_BACKFILL_BATCH_SIZE, m as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, n as ONTOLOGY_BACKFILL_RECHECK_MS, o as OntologyBackfillResult, p as OntologyConfig, q as OntologyEdgeType, r as OntologyMode, s as OntologyNodeType, t as OntologyPromptContext, u as OntologyUpdates, P as PromptOverrides, v as PromptService, w as PrunePartialFailureError, V as VectorRanker, x as VectorRankerFallback, y as VectorRankerRankArgs, z as VectorRankerSemanticResult, A as WikiBusyError, B as WikiBusyOperation, C as WikiCheckpoint, D as WikiConfig, I as WikiDuplicateHashError, J as WikiEdge, K as WikiEvent, N as WikiFact, Q as WikiMemoryTestAccess, T as WikiOutboxEvent, U as WikiSourceRefHashCollision, X as WikiStrictOntologyViolation, Y as WikiTask, Z as WikiTransactionError } from './testing-zf3Jp_RL.mjs';
|
|
3
3
|
import { OkfFile } from '@equationalapplications/core-okf';
|
|
4
4
|
import 'minisearch';
|
|
5
5
|
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, O as OntologyManifest, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-
|
|
2
|
-
export { E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HEAL_BATCH_SIZE, i as HEAL_RECHECK_MS, j as HOOK_TIMEOUT_MARKER, k as HealResult, L as LLMProvider, l as ONTOLOGY_BACKFILL_BATCH_SIZE, m as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, n as ONTOLOGY_BACKFILL_RECHECK_MS, o as OntologyBackfillResult, p as OntologyConfig, q as OntologyEdgeType, r as OntologyMode, s as OntologyNodeType, t as OntologyPromptContext, u as OntologyUpdates, P as PromptOverrides, v as PromptService, w as PrunePartialFailureError, V as VectorRanker, x as VectorRankerFallback, y as VectorRankerRankArgs, z as VectorRankerSemanticResult, A as WikiBusyError, B as WikiBusyOperation, C as WikiCheckpoint, D as WikiConfig, I as WikiDuplicateHashError, J as WikiEdge, K as WikiEvent, N as WikiFact, Q as WikiMemoryTestAccess, T as WikiOutboxEvent, U as
|
|
1
|
+
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, O as OntologyManifest, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-zf3Jp_RL.js';
|
|
2
|
+
export { E as EntityStatus, d as ExtractedFact, e as ExtractedFactEdge, f as ExtractedFactWithOntology, g as ExtractedTask, h as GraphTraversalOptions, H as HEAL_BATCH_SIZE, i as HEAL_RECHECK_MS, j as HOOK_TIMEOUT_MARKER, k as HealResult, L as LLMProvider, l as ONTOLOGY_BACKFILL_BATCH_SIZE, m as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, n as ONTOLOGY_BACKFILL_RECHECK_MS, o as OntologyBackfillResult, p as OntologyConfig, q as OntologyEdgeType, r as OntologyMode, s as OntologyNodeType, t as OntologyPromptContext, u as OntologyUpdates, P as PromptOverrides, v as PromptService, w as PrunePartialFailureError, V as VectorRanker, x as VectorRankerFallback, y as VectorRankerRankArgs, z as VectorRankerSemanticResult, A as WikiBusyError, B as WikiBusyOperation, C as WikiCheckpoint, D as WikiConfig, I as WikiDuplicateHashError, J as WikiEdge, K as WikiEvent, N as WikiFact, Q as WikiMemoryTestAccess, T as WikiOutboxEvent, U as WikiSourceRefHashCollision, X as WikiStrictOntologyViolation, Y as WikiTask, Z as WikiTransactionError } from './testing-zf3Jp_RL.js';
|
|
3
3
|
import { OkfFile } from '@equationalapplications/core-okf';
|
|
4
4
|
import 'minisearch';
|
|
5
5
|
|
package/dist/index.js
CHANGED
|
@@ -208,6 +208,31 @@ var WikiDuplicateHashError = class extends Error {
|
|
|
208
208
|
this.entityId = params.entityId;
|
|
209
209
|
}
|
|
210
210
|
};
|
|
211
|
+
var WikiStrictOntologyViolation = class extends Error {
|
|
212
|
+
constructor(entityId, kind, type) {
|
|
213
|
+
super(
|
|
214
|
+
`Out-of-manifest ${kind} type "${type}" for entity "${entityId}" under strict mode.`
|
|
215
|
+
);
|
|
216
|
+
this.code = "WIKI_STRICT_ONTOLOGY_VIOLATION";
|
|
217
|
+
this.entityId = entityId;
|
|
218
|
+
this.kind = kind;
|
|
219
|
+
this.type = type;
|
|
220
|
+
this.name = "WikiStrictOntologyViolation";
|
|
221
|
+
}
|
|
222
|
+
};
|
|
223
|
+
var WikiSourceRefHashCollision = class extends Error {
|
|
224
|
+
constructor(params) {
|
|
225
|
+
super(
|
|
226
|
+
`Source hash "${params.sourceHash}" for entity "${params.entityId}" is already mapped to sourceRef "${params.existingSourceRef}"; cannot remap to "${params.attemptedSourceRef}".`
|
|
227
|
+
);
|
|
228
|
+
this.code = "WIKI_SOURCE_REF_HASH_COLLISION";
|
|
229
|
+
this.entityId = params.entityId;
|
|
230
|
+
this.sourceHash = params.sourceHash;
|
|
231
|
+
this.existingSourceRef = params.existingSourceRef;
|
|
232
|
+
this.attemptedSourceRef = params.attemptedSourceRef;
|
|
233
|
+
this.name = "WikiSourceRefHashCollision";
|
|
234
|
+
}
|
|
235
|
+
};
|
|
211
236
|
var WikiTransactionError = class extends Error {
|
|
212
237
|
constructor(message, options) {
|
|
213
238
|
super(message, options);
|
|
@@ -856,12 +881,22 @@ var EntryRepository = class extends BaseRepository {
|
|
|
856
881
|
/**
|
|
857
882
|
* Fetch recent non-deleted entries for an entity (limited), ordered by updated_at DESC.
|
|
858
883
|
* Used by MaintenanceService.doRunLibrarian().
|
|
884
|
+
*
|
|
885
|
+
* When `excludeSourceRef` is supplied, the WHERE clause filters out that
|
|
886
|
+
* sourceRef BEFORE applying LIMIT. The caller in IngestionService is
|
|
887
|
+
* `findRecentByEntityId(entityId, 500, tx)` — it discards same-sourceRef
|
|
888
|
+
* rows after the read to build a cross-sourceRef title index. If those
|
|
889
|
+
* rows are filtered out in SQL, the limit applies to usable rows only
|
|
890
|
+
* and a re-ingest where the caller's sourceRef owns most recent facts
|
|
891
|
+
* still leaves cross-sourceRef entries in the index.
|
|
859
892
|
*/
|
|
860
|
-
async findRecentByEntityId(entityId, limit, tx) {
|
|
893
|
+
async findRecentByEntityId(entityId, limit, tx, excludeSourceRef) {
|
|
861
894
|
const executor = this.getExecutor(tx);
|
|
895
|
+
const excludeClause = excludeSourceRef ? ` AND (source_ref IS NULL OR source_ref != ?)` : "";
|
|
896
|
+
const args = excludeSourceRef ? [entityId, excludeSourceRef, limit] : [entityId, limit];
|
|
862
897
|
const rows = await executor.getAllAsync(
|
|
863
|
-
`SELECT * FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL ORDER BY updated_at DESC LIMIT ?`,
|
|
864
|
-
|
|
898
|
+
`SELECT * FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL${excludeClause} ORDER BY updated_at DESC LIMIT ?`,
|
|
899
|
+
args
|
|
865
900
|
);
|
|
866
901
|
return rows.map(mapRowToFact);
|
|
867
902
|
}
|
|
@@ -2076,6 +2111,34 @@ var EdgeRepository = class extends BaseRepository {
|
|
|
2076
2111
|
const executor = this.getExecutor(tx);
|
|
2077
2112
|
await executor.runAsync(`DELETE FROM ${this.prefix}edges WHERE entity_id = ?`, [entityId]);
|
|
2078
2113
|
}
|
|
2114
|
+
/**
|
|
2115
|
+
* Hard-delete every edge whose `source_id` is in `sourceFactIds`, scoped to
|
|
2116
|
+
* `entityId`. Returns the number of rows deleted. Edges have no soft-delete
|
|
2117
|
+
* concept; this method is named `softDeleteBySourceFactIds` for symmetry with
|
|
2118
|
+
* `EntryRepository.softDeleteBySource` (which IS a soft delete), but its
|
|
2119
|
+
* semantics are immediate presence/absence removal. Use ONLY inside an open
|
|
2120
|
+
* transaction — the method requires `tx` and performs no autocommit.
|
|
2121
|
+
*
|
|
2122
|
+
* Used by `IngestionService.upsertGraphCore` to retire the prior parse's
|
|
2123
|
+
* edges whose source facts are being superseded by `softDeleteBySource`.
|
|
2124
|
+
*/
|
|
2125
|
+
async softDeleteBySourceFactIds(entityId, sourceFactIds, tx) {
|
|
2126
|
+
if (sourceFactIds.length === 0) return 0;
|
|
2127
|
+
const executor = this.getExecutor(tx);
|
|
2128
|
+
const chunkSize = 500;
|
|
2129
|
+
let totalChanges = 0;
|
|
2130
|
+
for (let i = 0; i < sourceFactIds.length; i += chunkSize) {
|
|
2131
|
+
const chunk = sourceFactIds.slice(i, i + chunkSize);
|
|
2132
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
2133
|
+
const result = await executor.runAsync(
|
|
2134
|
+
`DELETE FROM ${this.prefix}edges
|
|
2135
|
+
WHERE entity_id = ? AND source_id IN (${placeholders})`,
|
|
2136
|
+
[entityId, ...chunk]
|
|
2137
|
+
);
|
|
2138
|
+
totalChanges += result.changes;
|
|
2139
|
+
}
|
|
2140
|
+
return totalChanges;
|
|
2141
|
+
}
|
|
2079
2142
|
/**
|
|
2080
2143
|
* Multi-hop traversal from `sourceId` via SQLite `WITH RECURSIVE`. All filtering,
|
|
2081
2144
|
* dead-ending, cycle-guarding, capping, and ordering happens in this one query.
|
|
@@ -2262,14 +2325,25 @@ function mergeOntologyUpdates(current, updates) {
|
|
|
2262
2325
|
}
|
|
2263
2326
|
return { node_types, edge_types };
|
|
2264
2327
|
}
|
|
2265
|
-
function validateInlineEdges(sourceType, _targetType, edges, manifest) {
|
|
2266
|
-
|
|
2328
|
+
function validateInlineEdges(sourceType, _targetType, edges, manifest, opts) {
|
|
2329
|
+
const strict = opts?.strict === true;
|
|
2330
|
+
const entityId = opts?.entityId ?? "";
|
|
2331
|
+
if (!Array.isArray(edges)) {
|
|
2332
|
+
if (strict) throw new WikiStrictOntologyViolation(entityId, "edge", "");
|
|
2333
|
+
return [];
|
|
2334
|
+
}
|
|
2267
2335
|
const valid = [];
|
|
2268
2336
|
for (const edge of edges) {
|
|
2269
|
-
if (typeof edge?.edge_type !== "string" || typeof edge?.target_title !== "string")
|
|
2337
|
+
if (typeof edge?.edge_type !== "string" || typeof edge?.target_title !== "string") {
|
|
2338
|
+
if (strict) throw new WikiStrictOntologyViolation(entityId, "edge", String(edge?.edge_type ?? ""));
|
|
2339
|
+
continue;
|
|
2340
|
+
}
|
|
2270
2341
|
const defs = resolveEdgeDefinitions(edge.edge_type, manifest);
|
|
2271
2342
|
const match = defs.find((d) => d.source_type.toLowerCase() === sourceType.toLowerCase());
|
|
2272
|
-
if (!match)
|
|
2343
|
+
if (!match) {
|
|
2344
|
+
if (strict) throw new WikiStrictOntologyViolation(entityId, "edge", edge.edge_type);
|
|
2345
|
+
continue;
|
|
2346
|
+
}
|
|
2273
2347
|
valid.push({ edge_type: match.type, target_title: edge.target_title });
|
|
2274
2348
|
}
|
|
2275
2349
|
return valid;
|
|
@@ -2370,16 +2444,25 @@ var MetadataRepository = class extends BaseRepository {
|
|
|
2370
2444
|
await this.db.execAsync(`PRAGMA wal_checkpoint(TRUNCATE)`);
|
|
2371
2445
|
await this.db.execAsync(`VACUUM`);
|
|
2372
2446
|
}
|
|
2447
|
+
/**
|
|
2448
|
+
* Returns the set of distinct `entity_id` values across entries/tasks/events.
|
|
2449
|
+
*
|
|
2450
|
+
* Deliberately widened from a prior `WHERE deleted_at IS NULL` form: the
|
|
2451
|
+
* returned set intentionally includes entities whose only rows are
|
|
2452
|
+
* soft-deleted, so the host's maintenance sweep can visit such
|
|
2453
|
+
* "decommissioned-scope" namespaces and reap the soft-deleted rows. Sorting
|
|
2454
|
+
* is `entity_id COLLATE BINARY` for stable, locale-independent ordering.
|
|
2455
|
+
*/
|
|
2373
2456
|
async getDistinctEntityIds(tx) {
|
|
2374
2457
|
const executor = this.getExecutor(tx);
|
|
2375
2458
|
const rows = await executor.getAllAsync(
|
|
2376
2459
|
`SELECT DISTINCT entity_id FROM (
|
|
2377
|
-
SELECT entity_id FROM ${this.prefix}entries
|
|
2460
|
+
SELECT entity_id FROM ${this.prefix}entries
|
|
2378
2461
|
UNION
|
|
2379
|
-
SELECT entity_id FROM ${this.prefix}tasks
|
|
2462
|
+
SELECT entity_id FROM ${this.prefix}tasks
|
|
2380
2463
|
UNION
|
|
2381
2464
|
SELECT entity_id FROM ${this.prefix}events
|
|
2382
|
-
) ORDER BY entity_id`
|
|
2465
|
+
) ORDER BY entity_id COLLATE BINARY`
|
|
2383
2466
|
);
|
|
2384
2467
|
return rows.map((r) => r.entity_id);
|
|
2385
2468
|
}
|
|
@@ -3465,12 +3548,14 @@ var DEFAULT_CHUNK_OVERLAP = 400;
|
|
|
3465
3548
|
|
|
3466
3549
|
// src/services/IngestionService.ts
|
|
3467
3550
|
var IngestionService = class {
|
|
3468
|
-
constructor(db, prefix, options, entryRepo, sourceRefIndexRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
3551
|
+
constructor(db, prefix, options, entryRepo, sourceRefIndexRepo, metadataRepo, edgeRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
3469
3552
|
this.db = db;
|
|
3470
3553
|
this.prefix = prefix;
|
|
3471
3554
|
this.options = options;
|
|
3472
3555
|
this.entryRepo = entryRepo;
|
|
3473
3556
|
this.sourceRefIndexRepo = sourceRefIndexRepo;
|
|
3557
|
+
this.metadataRepo = metadataRepo;
|
|
3558
|
+
this.edgeRepo = edgeRepo;
|
|
3474
3559
|
this.searchService = searchService;
|
|
3475
3560
|
this.jobManager = jobManager;
|
|
3476
3561
|
this.embeddingService = embeddingService;
|
|
@@ -3543,12 +3628,8 @@ var IngestionService = class {
|
|
|
3543
3628
|
try {
|
|
3544
3629
|
await this.db.withTransactionAsync(async (tx) => {
|
|
3545
3630
|
deletedSourceFactIds.push(...await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false));
|
|
3546
|
-
await this.entryRepo.softDeleteBySource(entityId, tx, sourceRef, null);
|
|
3547
|
-
await this.sourceRefIndexRepo.softDeleteByEntityAndSourceRef(entityId, sourceRef, tx);
|
|
3548
|
-
await this.sourceRefIndexRepo.upsert(entityId, sourceHash, sourceRef, tx);
|
|
3549
3631
|
const titleIndex = /* @__PURE__ */ new Map();
|
|
3550
|
-
const
|
|
3551
|
-
const existingFacts = await this.entryRepo.findRecentByEntityId(entityId, 500, tx);
|
|
3632
|
+
const existingFacts = await this.entryRepo.findRecentByEntityId(entityId, 500, tx, sourceRef);
|
|
3552
3633
|
for (const existing of existingFacts) {
|
|
3553
3634
|
titleIndex.set(normalizeTitleKey(existing.title), {
|
|
3554
3635
|
id: existing.id,
|
|
@@ -3557,6 +3638,8 @@ var IngestionService = class {
|
|
|
3557
3638
|
}
|
|
3558
3639
|
let ontologyState = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
3559
3640
|
let { mode, manifest } = ontologyState;
|
|
3641
|
+
const hostNodes = [];
|
|
3642
|
+
const rawEdgeRequests = [];
|
|
3560
3643
|
for (const { facts, ontology_updates } of orderedChunkFacts) {
|
|
3561
3644
|
if (mode === "emergent" && ontology_updates && this.ontologyService) {
|
|
3562
3645
|
manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontology_updates, tx);
|
|
@@ -3565,45 +3648,51 @@ var IngestionService = class {
|
|
|
3565
3648
|
}
|
|
3566
3649
|
for (const fact of facts) {
|
|
3567
3650
|
const ontologyFact = fact;
|
|
3568
|
-
const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest) ?? { okf_type: null, edges: [] };
|
|
3651
|
+
const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false }) ?? { okf_type: null, edges: [] };
|
|
3569
3652
|
const id = generateId("fact_");
|
|
3570
|
-
|
|
3653
|
+
hostNodes.push({
|
|
3571
3654
|
id,
|
|
3572
|
-
|
|
3655
|
+
type: ontologyFact.okf_type ?? "",
|
|
3573
3656
|
title: fact.title,
|
|
3574
3657
|
body: fact.body,
|
|
3658
|
+
// Forward the LLM-extracted tags/confidence through the
|
|
3659
|
+
// extracted-shape upsertGraphCore path so the entries row
|
|
3660
|
+
// stores them (search filterability, heal-candidate
|
|
3661
|
+
// selection, runReembed embedding-text signal). The public
|
|
3662
|
+
// WikiMemory.upsertGraph API doesn't accept tags/confidence
|
|
3663
|
+
// — host-supplied deterministic nodes default to [] and
|
|
3664
|
+
// 'certain' as documented.
|
|
3575
3665
|
tags: fact.tags,
|
|
3576
|
-
confidence: fact.confidence
|
|
3577
|
-
|
|
3578
|
-
source_hash: sourceHash,
|
|
3579
|
-
source_ref: sourceRef,
|
|
3580
|
-
created_at: now,
|
|
3581
|
-
updated_at: now,
|
|
3582
|
-
last_accessed_at: null,
|
|
3583
|
-
access_count: 0,
|
|
3584
|
-
deleted_at: null,
|
|
3585
|
-
okf_type: normalized.okf_type
|
|
3586
|
-
};
|
|
3587
|
-
await this.entryRepo.upsert(wikiFact, tx);
|
|
3666
|
+
confidence: fact.confidence
|
|
3667
|
+
});
|
|
3588
3668
|
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
3589
3669
|
titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
|
|
3590
3670
|
if (normalized.edges.length > 0) {
|
|
3591
|
-
|
|
3671
|
+
rawEdgeRequests.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
3592
3672
|
}
|
|
3593
3673
|
}
|
|
3594
3674
|
}
|
|
3595
|
-
|
|
3596
|
-
|
|
3675
|
+
const hostEdges = [];
|
|
3676
|
+
for (const req of rawEdgeRequests) {
|
|
3677
|
+
const resolved = this.ontologyService?.resolveEdges(
|
|
3597
3678
|
entityId,
|
|
3598
|
-
|
|
3599
|
-
|
|
3600
|
-
|
|
3679
|
+
req.sourceId,
|
|
3680
|
+
req.sourceType,
|
|
3681
|
+
req.edges,
|
|
3601
3682
|
manifest,
|
|
3602
3683
|
titleIndex,
|
|
3603
|
-
tx,
|
|
3604
3684
|
now
|
|
3605
|
-
);
|
|
3685
|
+
) ?? [];
|
|
3686
|
+
for (const e of resolved) {
|
|
3687
|
+
hostEdges.push({ type: e.edge_type, sourceId: e.source_id, targetId: e.target_id });
|
|
3688
|
+
}
|
|
3606
3689
|
}
|
|
3690
|
+
await this.upsertGraphCore(
|
|
3691
|
+
entityId,
|
|
3692
|
+
{ sourceRef, sourceHash, nodes: hostNodes, edges: hostEdges },
|
|
3693
|
+
tx,
|
|
3694
|
+
{ strict: false }
|
|
3695
|
+
);
|
|
3607
3696
|
});
|
|
3608
3697
|
} catch (err) {
|
|
3609
3698
|
const sqliteCode = err instanceof WikiTransactionError ? err.sqliteErrorCode : extractSqliteCode(err);
|
|
@@ -3633,6 +3722,126 @@ var IngestionService = class {
|
|
|
3633
3722
|
releaseIngestLocks();
|
|
3634
3723
|
}
|
|
3635
3724
|
}
|
|
3725
|
+
/**
|
|
3726
|
+
* Implements spec §2 data flow steps a–j with the host-facing
|
|
3727
|
+
* `{sourceRef, sourceHash, nodes, edges}` parameter shape. Shared between
|
|
3728
|
+
* `ingestDocument` (with `{ strict: false }` to preserve current behavior)
|
|
3729
|
+
* and `WikiMemory.upsertGraph` (which lets the persisted ontology mode
|
|
3730
|
+
* decide strictness).
|
|
3731
|
+
*
|
|
3732
|
+
* Runs ENTIRELY inside the supplied `tx`. Does not acquire locks, does not
|
|
3733
|
+
* open a nested transaction, does not perform post-commit work (search
|
|
3734
|
+
* sync, embedding, cache eviction). Those concerns remain in `ingestDocument`.
|
|
3735
|
+
*
|
|
3736
|
+
* The pre-flight validation (steps a–c) lives INSIDE this method per the
|
|
3737
|
+
* spec, so the public `WikiMemory.upsertGraph` can be a thin wrapper that
|
|
3738
|
+
* performs only the C2 probe before delegating.
|
|
3739
|
+
*
|
|
3740
|
+
* @returns Counts: nodesWritten (validated nodes persisted), edgesWritten
|
|
3741
|
+
* (manifest-valid edges persisted), superseded (prior facts soft-deleted
|
|
3742
|
+
* plus prior source-ref edges hard-deleted).
|
|
3743
|
+
*/
|
|
3744
|
+
async upsertGraphCore(entityId, params, tx, opts) {
|
|
3745
|
+
const now = Date.now();
|
|
3746
|
+
const ontologyState = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
3747
|
+
let { mode, manifest } = ontologyState;
|
|
3748
|
+
const strictEffective = opts?.strict === false ? false : opts?.strict === true || mode === "strict";
|
|
3749
|
+
const wikiFacts = [];
|
|
3750
|
+
for (const node of params.nodes) {
|
|
3751
|
+
const factLike = { okf_type: node.type, edges: [] };
|
|
3752
|
+
const normalized = this.ontologyService?.validateAndNormalizeFact(factLike, manifest, {
|
|
3753
|
+
strict: strictEffective,
|
|
3754
|
+
entityId
|
|
3755
|
+
}) ?? { okf_type: null};
|
|
3756
|
+
const wikiFact = {
|
|
3757
|
+
id: node.id,
|
|
3758
|
+
entity_id: entityId,
|
|
3759
|
+
title: node.title,
|
|
3760
|
+
body: node.body ?? "",
|
|
3761
|
+
// Public upsertGraph callers don't supply tags/confidence, so they
|
|
3762
|
+
// default to [] / 'certain' for deterministic AST-grade facts.
|
|
3763
|
+
// ingestDocument routes through this method for the LLM path and
|
|
3764
|
+
// forwards the LLM-supplied tags + confidence, preserving the prior
|
|
3765
|
+
// behavior (search filterability, heal-candidate selection, runReembed
|
|
3766
|
+
// embedding-text signal).
|
|
3767
|
+
tags: node.tags ? Array.from(node.tags) : [],
|
|
3768
|
+
confidence: node.confidence ?? "certain",
|
|
3769
|
+
source_type: "immutable_document",
|
|
3770
|
+
source_hash: params.sourceHash,
|
|
3771
|
+
source_ref: params.sourceRef,
|
|
3772
|
+
created_at: now,
|
|
3773
|
+
updated_at: now,
|
|
3774
|
+
last_accessed_at: null,
|
|
3775
|
+
access_count: 0,
|
|
3776
|
+
deleted_at: null,
|
|
3777
|
+
okf_type: normalized.okf_type
|
|
3778
|
+
};
|
|
3779
|
+
wikiFacts.push(wikiFact);
|
|
3780
|
+
}
|
|
3781
|
+
const sourceIdToType = /* @__PURE__ */ new Map();
|
|
3782
|
+
for (const fact of wikiFacts) sourceIdToType.set(fact.id, fact.okf_type ?? null);
|
|
3783
|
+
const hasConstraints = (manifest.node_types?.length ?? 0) > 0 || (manifest.edge_types?.length ?? 0) > 0;
|
|
3784
|
+
const validEdges = [];
|
|
3785
|
+
for (const edge of params.edges) {
|
|
3786
|
+
if (!hasConstraints) {
|
|
3787
|
+
validEdges.push({
|
|
3788
|
+
id: edge.id ?? generateId(),
|
|
3789
|
+
entity_id: entityId,
|
|
3790
|
+
source_id: edge.sourceId,
|
|
3791
|
+
target_id: edge.targetId,
|
|
3792
|
+
edge_type: edge.type,
|
|
3793
|
+
created_at: now
|
|
3794
|
+
});
|
|
3795
|
+
continue;
|
|
3796
|
+
}
|
|
3797
|
+
if (!sourceIdToType.has(edge.sourceId)) {
|
|
3798
|
+
validEdges.push({
|
|
3799
|
+
id: edge.id ?? generateId(),
|
|
3800
|
+
entity_id: entityId,
|
|
3801
|
+
source_id: edge.sourceId,
|
|
3802
|
+
target_id: edge.targetId,
|
|
3803
|
+
edge_type: edge.type,
|
|
3804
|
+
created_at: now
|
|
3805
|
+
});
|
|
3806
|
+
continue;
|
|
3807
|
+
}
|
|
3808
|
+
const sourceType = sourceIdToType.get(edge.sourceId);
|
|
3809
|
+
const candidates = (manifest.edge_types ?? []).filter(
|
|
3810
|
+
(d) => d.type.toLowerCase() === edge.type.toLowerCase() && d.source_type.toLowerCase() === (sourceType ?? "").toLowerCase()
|
|
3811
|
+
);
|
|
3812
|
+
const match = candidates[0];
|
|
3813
|
+
if (!match) {
|
|
3814
|
+
if (strictEffective) throw new WikiStrictOntologyViolation(entityId, "edge", edge.type);
|
|
3815
|
+
continue;
|
|
3816
|
+
}
|
|
3817
|
+
validEdges.push({
|
|
3818
|
+
id: edge.id ?? generateId(),
|
|
3819
|
+
entity_id: entityId,
|
|
3820
|
+
source_id: edge.sourceId,
|
|
3821
|
+
target_id: edge.targetId,
|
|
3822
|
+
edge_type: match.type,
|
|
3823
|
+
created_at: now
|
|
3824
|
+
});
|
|
3825
|
+
}
|
|
3826
|
+
const deletedFactIds = await this.entryRepo.findIdsBySource(entityId, params.sourceRef, null, tx, false);
|
|
3827
|
+
await this.entryRepo.softDeleteBySource(entityId, tx, params.sourceRef, null);
|
|
3828
|
+
const deletedEdgeCount = await this.edgeRepo.softDeleteBySourceFactIds(entityId, deletedFactIds, tx);
|
|
3829
|
+
await this.sourceRefIndexRepo.softDeleteByEntityAndSourceRef(entityId, params.sourceRef, tx);
|
|
3830
|
+
await this.sourceRefIndexRepo.upsert(entityId, params.sourceHash, params.sourceRef, tx);
|
|
3831
|
+
for (const wikiFact of wikiFacts) {
|
|
3832
|
+
await this.entryRepo.upsert(wikiFact, tx);
|
|
3833
|
+
}
|
|
3834
|
+
let edgesInsertedCount = 0;
|
|
3835
|
+
for (const edge of validEdges) {
|
|
3836
|
+
const inserted = await this.edgeRepo.addIgnoreDuplicate(edge, tx);
|
|
3837
|
+
if (inserted) edgesInsertedCount++;
|
|
3838
|
+
}
|
|
3839
|
+
return {
|
|
3840
|
+
nodesWritten: wikiFacts.length,
|
|
3841
|
+
edgesWritten: edgesInsertedCount,
|
|
3842
|
+
superseded: deletedFactIds.length + deletedEdgeCount
|
|
3843
|
+
};
|
|
3844
|
+
}
|
|
3636
3845
|
};
|
|
3637
3846
|
|
|
3638
3847
|
// src/services/BoundedLlmCall.ts
|
|
@@ -4081,7 +4290,7 @@ var MaintenanceService = class {
|
|
|
4081
4290
|
}
|
|
4082
4291
|
if (skip) continue;
|
|
4083
4292
|
const ontologyFact = fact;
|
|
4084
|
-
const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest) ?? { okf_type: null, edges: [] };
|
|
4293
|
+
const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false }) ?? { okf_type: null, edges: [] };
|
|
4085
4294
|
const id = generateId("fact_");
|
|
4086
4295
|
const factObj = {
|
|
4087
4296
|
id,
|
|
@@ -4466,7 +4675,8 @@ var MaintenanceService = class {
|
|
|
4466
4675
|
okf_type: classification.okf_type,
|
|
4467
4676
|
edges: classification.edges
|
|
4468
4677
|
},
|
|
4469
|
-
manifest
|
|
4678
|
+
manifest,
|
|
4679
|
+
{ strict: false }
|
|
4470
4680
|
);
|
|
4471
4681
|
if (!normalized.okf_type) {
|
|
4472
4682
|
failedValidation++;
|
|
@@ -5856,16 +6066,27 @@ var OntologyService = class {
|
|
|
5856
6066
|
this.invalidateCache(entityId);
|
|
5857
6067
|
return merged;
|
|
5858
6068
|
}
|
|
5859
|
-
validateAndNormalizeFact(fact, manifest) {
|
|
6069
|
+
validateAndNormalizeFact(fact, manifest, opts) {
|
|
5860
6070
|
const rawType = typeof fact.okf_type === "string" ? fact.okf_type : "";
|
|
6071
|
+
const strict = opts?.strict === true;
|
|
5861
6072
|
const canonical = resolveNodeType(rawType, manifest);
|
|
5862
|
-
if (!canonical)
|
|
5863
|
-
|
|
6073
|
+
if (!canonical) {
|
|
6074
|
+
if (strict) throw new WikiStrictOntologyViolation(opts?.entityId ?? "", "node", rawType);
|
|
6075
|
+
return { okf_type: null, edges: [] };
|
|
6076
|
+
}
|
|
6077
|
+
const edges = validateInlineEdges(canonical, null, fact.edges ?? [], manifest, opts);
|
|
5864
6078
|
return { okf_type: canonical, edges };
|
|
5865
6079
|
}
|
|
5866
|
-
|
|
5867
|
-
|
|
5868
|
-
|
|
6080
|
+
/**
|
|
6081
|
+
* Pure resolver: given a source type, LLM-supplied edges, the manifest,
|
|
6082
|
+
* and a title index, return a list of concrete `WikiEdge` objects ready
|
|
6083
|
+
* for persistence. Performs no DB writes. Used by callers that batch edge
|
|
6084
|
+
* writes inside their own transaction (e.g. `IngestionService.ingestDocument`
|
|
6085
|
+
* when delegating to `upsertGraphCore`).
|
|
6086
|
+
*/
|
|
6087
|
+
resolveEdges(entityId, sourceId, sourceType, edges, manifest, titleIndex, now) {
|
|
6088
|
+
if (!sourceType || edges.length === 0) return [];
|
|
6089
|
+
const out = [];
|
|
5869
6090
|
for (const edge of edges) {
|
|
5870
6091
|
const candidates = resolveEdgeDefinitions(edge.edge_type, manifest).filter((d) => d.source_type.toLowerCase() === sourceType.toLowerCase());
|
|
5871
6092
|
if (candidates.length === 0) continue;
|
|
@@ -5876,15 +6097,29 @@ var OntologyService = class {
|
|
|
5876
6097
|
(d) => d.target_type.toLowerCase() === (target.okf_type ?? "").toLowerCase()
|
|
5877
6098
|
);
|
|
5878
6099
|
if (!def) continue;
|
|
5879
|
-
|
|
6100
|
+
out.push({
|
|
5880
6101
|
id: generateId(),
|
|
5881
6102
|
entity_id: entityId,
|
|
5882
6103
|
source_id: sourceId,
|
|
5883
6104
|
target_id: target.id,
|
|
5884
6105
|
edge_type: def.type,
|
|
5885
6106
|
created_at: now
|
|
5886
|
-
};
|
|
5887
|
-
|
|
6107
|
+
});
|
|
6108
|
+
}
|
|
6109
|
+
return out;
|
|
6110
|
+
}
|
|
6111
|
+
/**
|
|
6112
|
+
* Backwards-compatible wrapper: resolves edges via {@link resolveEdges},
|
|
6113
|
+
* then persists each via `edgeRepo.addIgnoreDuplicate` in the supplied
|
|
6114
|
+
* transaction. Returns the number of edges persisted. Used by paths that
|
|
6115
|
+
* want self-contained edge persistence (e.g. MaintenanceService heal /
|
|
6116
|
+
* backfill callers that do not have a separate edge-write step).
|
|
6117
|
+
*/
|
|
6118
|
+
async resolveAndPersistEdges(entityId, sourceId, sourceType, edges, manifest, titleIndex, tx, now) {
|
|
6119
|
+
const resolved = this.resolveEdges(entityId, sourceId, sourceType, edges, manifest, titleIndex, now);
|
|
6120
|
+
let persisted = 0;
|
|
6121
|
+
for (const edge of resolved) {
|
|
6122
|
+
const inserted = await this.edgeRepo.addIgnoreDuplicate(edge, tx);
|
|
5888
6123
|
if (inserted) persisted++;
|
|
5889
6124
|
}
|
|
5890
6125
|
return persisted;
|
|
@@ -5960,6 +6195,8 @@ var WikiMemory = class {
|
|
|
5960
6195
|
this.options,
|
|
5961
6196
|
this.entryRepo,
|
|
5962
6197
|
this.sourceRefIndexRepo,
|
|
6198
|
+
this.metadataRepo,
|
|
6199
|
+
this.edgeRepo,
|
|
5963
6200
|
this.searchService,
|
|
5964
6201
|
this.jobManager,
|
|
5965
6202
|
this.embeddingService,
|
|
@@ -6152,6 +6389,36 @@ var WikiMemory = class {
|
|
|
6152
6389
|
const canonical = await this.sourceRefIndexRepo.findActiveByEntityAndHash(entityId, sourceHash);
|
|
6153
6390
|
return canonical === null ? [] : [canonical];
|
|
6154
6391
|
}
|
|
6392
|
+
/**
|
|
6393
|
+
* Returns the set of `entity_id` values with at least one row in this
|
|
6394
|
+
* database, INCLUDING entities whose only remaining rows are soft-deleted
|
|
6395
|
+
* — sorted ascending `entity_id COLLATE BINARY`. Empty array when the
|
|
6396
|
+
* database has no entities.
|
|
6397
|
+
*
|
|
6398
|
+
* The deliberate inclusion of soft-deleted-only entities closes the
|
|
6399
|
+
* decommissioned-scope leak: a scoped namespace whose documents have all
|
|
6400
|
+
* been forgotten or superseded must still appear so host maintenance
|
|
6401
|
+
* sweeps (`runLibrarian`, `runHeal`, `runOntologyBackfill`, `runPrune`)
|
|
6402
|
+
* can visit it and reap the soft-deleted rows.
|
|
6403
|
+
*
|
|
6404
|
+
* Read-only. Never acquires a lock. Never opens a transaction. Propagates
|
|
6405
|
+
* underlying read errors.
|
|
6406
|
+
*
|
|
6407
|
+
* @param options.prefix - Optional string filter applied as
|
|
6408
|
+
* `id.startsWith(prefix)`. O(n) over distinct ids because the
|
|
6409
|
+
* `(entity_id, source_ref)` index is not seekable on `entity_id`-only
|
|
6410
|
+
* prefix. Empty-string prefix matches every id.
|
|
6411
|
+
*
|
|
6412
|
+
* Note: this widens the coverage of `exportDump` (which delegates to
|
|
6413
|
+
* `MetadataRepository.getDistinctEntityIds`); exported dumps now include
|
|
6414
|
+
* orphaned entities. This is a deliberate widening for backup/migration
|
|
6415
|
+
* coverage, not a breaking change to `exportDump`'s return shape.
|
|
6416
|
+
*/
|
|
6417
|
+
async listEntityIds(options) {
|
|
6418
|
+
const ids = await this.metadataRepo.getDistinctEntityIds();
|
|
6419
|
+
if (options?.prefix === void 0) return ids;
|
|
6420
|
+
return ids.filter((id) => id.startsWith(options.prefix));
|
|
6421
|
+
}
|
|
6155
6422
|
async runPrune(entityId, options) {
|
|
6156
6423
|
return this.maintenanceService.runPrune(entityId, options);
|
|
6157
6424
|
}
|
|
@@ -6248,6 +6515,57 @@ var WikiMemory = class {
|
|
|
6248
6515
|
async ingestDocument(entityId, params, opts) {
|
|
6249
6516
|
return this.ingestionService.ingestDocument(entityId, params, opts);
|
|
6250
6517
|
}
|
|
6518
|
+
/**
|
|
6519
|
+
* Deterministic write path into the graph. Accepts caller-supplied
|
|
6520
|
+
* `(nodes, edges)` and writes them under `(sourceRef, sourceHash)`
|
|
6521
|
+
* semantics identical to `ingestDocument`, minus the LLM extraction step.
|
|
6522
|
+
*
|
|
6523
|
+
* MUST be called from inside an open `db.withTransactionAsync` callback —
|
|
6524
|
+
* the third argument is the caller's `tx`. The method does not open a
|
|
6525
|
+
* nested transaction, does not acquire any lock, and does not perform
|
|
6526
|
+
* post-commit work (search sync, embedding, cache eviction). Hosts that
|
|
6527
|
+
* want embeddings synced should drive the existing maintenance sweep
|
|
6528
|
+
* (`runLibrarian` / `runHeal` / `runOntologyBackfill` / `runPrune`,
|
|
6529
|
+
* scoped via `listEntityIds`).
|
|
6530
|
+
*
|
|
6531
|
+
* Contract (full pre-flight validation lives inside
|
|
6532
|
+
* `IngestionService.upsertGraphCore`, which this method delegates to):
|
|
6533
|
+
* - C2: no-op when `(entityId, sourceHash)` is already mapped to
|
|
6534
|
+
* `params.sourceRef`; throws `WikiSourceRefHashCollision` if mapped to
|
|
6535
|
+
* a different `sourceRef`.
|
|
6536
|
+
* - C3: edges with dangling `targetId` are stored verbatim (no FK, no
|
|
6537
|
+
* resolution).
|
|
6538
|
+
* - C4: under persisted ontology mode `'strict'`, an out-of-manifest node
|
|
6539
|
+
* or edge `type` throws `WikiStrictOntologyViolation` (pre-flight,
|
|
6540
|
+
* all-or-nothing — NONE written on failure).
|
|
6541
|
+
*
|
|
6542
|
+
* @returns Counts: nodesWritten (validated nodes persisted), edgesWritten
|
|
6543
|
+
* (manifest-valid edges persisted), superseded (prior facts soft-deleted
|
|
6544
|
+
* plus prior source-ref edges hard-deleted).
|
|
6545
|
+
*/
|
|
6546
|
+
async upsertGraph(entityId, params, adapter) {
|
|
6547
|
+
const sourceRef = normalizeSourceRef(params.sourceRef);
|
|
6548
|
+
if (!sourceRef) throw new Error("Invalid sourceRef");
|
|
6549
|
+
const sourceHash = normalizeSourceHash(params.sourceHash);
|
|
6550
|
+
if (!sourceHash) throw new Error("Invalid sourceHash (must be 64-char hex string)");
|
|
6551
|
+
const canonical = await this.sourceRefIndexRepo.findActiveByEntityAndHash(entityId, sourceHash, adapter);
|
|
6552
|
+
if (canonical !== null && canonical === sourceRef) {
|
|
6553
|
+
return { nodesWritten: 0, edgesWritten: 0, superseded: 0 };
|
|
6554
|
+
}
|
|
6555
|
+
if (canonical !== null && canonical !== sourceRef) {
|
|
6556
|
+
throw new WikiSourceRefHashCollision({
|
|
6557
|
+
entityId,
|
|
6558
|
+
sourceHash,
|
|
6559
|
+
existingSourceRef: canonical,
|
|
6560
|
+
attemptedSourceRef: sourceRef
|
|
6561
|
+
});
|
|
6562
|
+
}
|
|
6563
|
+
return this.ingestionService.upsertGraphCore(
|
|
6564
|
+
entityId,
|
|
6565
|
+
{ sourceRef, sourceHash, nodes: params.nodes, edges: params.edges },
|
|
6566
|
+
adapter
|
|
6567
|
+
);
|
|
6568
|
+
}
|
|
6251
6569
|
/**
|
|
6252
6570
|
* Returns up to `limit` unprocessed outbox events, oldest first.
|
|
6253
6571
|
* Works regardless of enableOutbox value — allows draining after disabling.
|
|
@@ -7067,6 +7385,8 @@ exports.PrunePartialFailureError = PrunePartialFailureError;
|
|
|
7067
7385
|
exports.WikiBusyError = WikiBusyError;
|
|
7068
7386
|
exports.WikiDuplicateHashError = WikiDuplicateHashError;
|
|
7069
7387
|
exports.WikiMemory = WikiMemory;
|
|
7388
|
+
exports.WikiSourceRefHashCollision = WikiSourceRefHashCollision;
|
|
7389
|
+
exports.WikiStrictOntologyViolation = WikiStrictOntologyViolation;
|
|
7070
7390
|
exports.WikiTransactionError = WikiTransactionError;
|
|
7071
7391
|
exports.chunkText = chunkText;
|
|
7072
7392
|
exports.configureRandomSource = configureRandomSource;
|