@equationalapplications/core-llm-wiki 5.2.0 → 5.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -0
- package/dist/{chunk-J3N3WRK7.mjs → chunk-KIJINNYU.mjs} +367 -84
- package/dist/chunk-KIJINNYU.mjs.map +1 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +664 -103
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +300 -24
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-DyXISRwS.d.mts → testing-zf3Jp_RL.d.mts} +336 -29
- package/dist/{testing-DyXISRwS.d.ts → testing-zf3Jp_RL.d.ts} +336 -29
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +378 -74
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +2 -2
- package/dist/chunk-J3N3WRK7.mjs.map +0 -1
package/dist/index.js
CHANGED
|
@@ -45,6 +45,24 @@ async function setupDatabase(db, prefix) {
|
|
|
45
45
|
CREATE INDEX IF NOT EXISTS ${prefix}entries_source_hash_idx ON ${prefix}entries(entity_id, source_hash) WHERE source_hash IS NOT NULL;
|
|
46
46
|
CREATE INDEX IF NOT EXISTS ${prefix}entries_updated_idx ON ${prefix}entries(updated_at DESC);
|
|
47
47
|
|
|
48
|
+
-- source_ref_index: per-(entity, source_hash) record of the canonical sourceRef
|
|
49
|
+
-- currently holding that hash. The partial UNIQUE index on (entity_id, source_hash)
|
|
50
|
+
-- WHERE deleted_at IS NULL enforces the sourceRef-level TOCTOU-race invariant;
|
|
51
|
+
-- entries-level uniqueness cannot express it because a single ingestDocument call
|
|
52
|
+
-- writes N facts that all share (entity_id, source_ref, source_hash). See
|
|
53
|
+
-- docs/superpowers/specs/2026-08-07-dependabot-concurrency-release-hygiene-design.md \xA7B1.
|
|
54
|
+
CREATE TABLE IF NOT EXISTS ${prefix}source_ref_index (
|
|
55
|
+
id TEXT PRIMARY KEY,
|
|
56
|
+
entity_id TEXT NOT NULL,
|
|
57
|
+
source_hash TEXT NOT NULL,
|
|
58
|
+
source_ref TEXT NOT NULL,
|
|
59
|
+
created_at INTEGER NOT NULL,
|
|
60
|
+
deleted_at INTEGER
|
|
61
|
+
);
|
|
62
|
+
CREATE UNIQUE INDEX IF NOT EXISTS ${prefix}idx_source_ref_hash
|
|
63
|
+
ON ${prefix}source_ref_index (entity_id, source_hash)
|
|
64
|
+
WHERE deleted_at IS NULL;
|
|
65
|
+
|
|
48
66
|
CREATE TABLE IF NOT EXISTS ${prefix}tasks (
|
|
49
67
|
id TEXT PRIMARY KEY,
|
|
50
68
|
entity_id TEXT NOT NULL,
|
|
@@ -149,6 +167,12 @@ var SQLITE_RESULT_CODE_NAMES = {
|
|
|
149
167
|
26: "SQLITE_NOTADB",
|
|
150
168
|
27: "SQLITE_NOTICE",
|
|
151
169
|
28: "SQLITE_WARNING",
|
|
170
|
+
// Extended result codes (https://sqlite.org/rescode.html#extended_result_code_list).
|
|
171
|
+
// 2067 = SQLITE_CONSTRAINT_UNIQUE — raised when a UNIQUE index/PRIMARY KEY
|
|
172
|
+
// rejects an INSERT/UPDATE; expo-sqlite reports this as a numeric "Error code N:"
|
|
173
|
+
// prefix and we normalize it back to the symbolic name so callers can branch on
|
|
174
|
+
// the same string the rest of the ecosystem uses.
|
|
175
|
+
2067: "SQLITE_CONSTRAINT_UNIQUE",
|
|
152
176
|
100: "SQLITE_ROW",
|
|
153
177
|
101: "SQLITE_DONE"
|
|
154
178
|
};
|
|
@@ -184,6 +208,31 @@ var WikiDuplicateHashError = class extends Error {
|
|
|
184
208
|
this.entityId = params.entityId;
|
|
185
209
|
}
|
|
186
210
|
};
|
|
211
|
+
var WikiStrictOntologyViolation = class extends Error {
|
|
212
|
+
constructor(entityId, kind, type) {
|
|
213
|
+
super(
|
|
214
|
+
`Out-of-manifest ${kind} type "${type}" for entity "${entityId}" under strict mode.`
|
|
215
|
+
);
|
|
216
|
+
this.code = "WIKI_STRICT_ONTOLOGY_VIOLATION";
|
|
217
|
+
this.entityId = entityId;
|
|
218
|
+
this.kind = kind;
|
|
219
|
+
this.type = type;
|
|
220
|
+
this.name = "WikiStrictOntologyViolation";
|
|
221
|
+
}
|
|
222
|
+
};
|
|
223
|
+
var WikiSourceRefHashCollision = class extends Error {
|
|
224
|
+
constructor(params) {
|
|
225
|
+
super(
|
|
226
|
+
`Source hash "${params.sourceHash}" for entity "${params.entityId}" is already mapped to sourceRef "${params.existingSourceRef}"; cannot remap to "${params.attemptedSourceRef}".`
|
|
227
|
+
);
|
|
228
|
+
this.code = "WIKI_SOURCE_REF_HASH_COLLISION";
|
|
229
|
+
this.entityId = params.entityId;
|
|
230
|
+
this.sourceHash = params.sourceHash;
|
|
231
|
+
this.existingSourceRef = params.existingSourceRef;
|
|
232
|
+
this.attemptedSourceRef = params.attemptedSourceRef;
|
|
233
|
+
this.name = "WikiSourceRefHashCollision";
|
|
234
|
+
}
|
|
235
|
+
};
|
|
187
236
|
var WikiTransactionError = class extends Error {
|
|
188
237
|
constructor(message, options) {
|
|
189
238
|
super(message, options);
|
|
@@ -377,6 +426,61 @@ var MIGRATIONS = [
|
|
|
377
426
|
);
|
|
378
427
|
}
|
|
379
428
|
}
|
|
429
|
+
},
|
|
430
|
+
{
|
|
431
|
+
version: 9,
|
|
432
|
+
description: "add_source_ref_index",
|
|
433
|
+
run: async (db, prefix) => {
|
|
434
|
+
const duplicates = await db.getAllAsync(
|
|
435
|
+
`SELECT entity_id, source_hash, COUNT(DISTINCT source_ref) AS n_refs
|
|
436
|
+
FROM ${prefix}entries
|
|
437
|
+
WHERE deleted_at IS NULL AND source_hash IS NOT NULL
|
|
438
|
+
GROUP BY entity_id, source_hash
|
|
439
|
+
HAVING COUNT(DISTINCT source_ref) > 1`
|
|
440
|
+
);
|
|
441
|
+
if (duplicates.length > 0) {
|
|
442
|
+
const sample = duplicates.slice(0, 5).map((d) => `(entity_id=${d.entity_id}, source_hash=${d.source_hash.slice(0, 12)}\u2026, n_refs=${d.n_refs})`).join(", ");
|
|
443
|
+
throw new Error(
|
|
444
|
+
`Migration v9 (add_source_ref_index) failed: existing live rows have multiple sourceRefs sharing a hash. Found ${duplicates.length} duplicate (entity_id, source_hash) groups. First ${Math.min(5, duplicates.length)}: ${sample}. Resolve each by calling forget({ sourceRef: <loser> }) for the offending sourceRef, then re-run setup.`
|
|
445
|
+
);
|
|
446
|
+
}
|
|
447
|
+
await db.execAsync(
|
|
448
|
+
`CREATE TABLE IF NOT EXISTS ${prefix}source_ref_index (
|
|
449
|
+
id TEXT PRIMARY KEY,
|
|
450
|
+
entity_id TEXT NOT NULL,
|
|
451
|
+
source_hash TEXT NOT NULL,
|
|
452
|
+
source_ref TEXT NOT NULL,
|
|
453
|
+
created_at INTEGER NOT NULL,
|
|
454
|
+
deleted_at INTEGER
|
|
455
|
+
);
|
|
456
|
+
CREATE UNIQUE INDEX IF NOT EXISTS ${prefix}idx_source_ref_hash
|
|
457
|
+
ON ${prefix}source_ref_index (entity_id, source_hash)
|
|
458
|
+
WHERE deleted_at IS NULL;`
|
|
459
|
+
);
|
|
460
|
+
await db.execAsync(
|
|
461
|
+
`INSERT OR IGNORE INTO ${prefix}source_ref_index (id, entity_id, source_hash, source_ref, created_at, deleted_at)
|
|
462
|
+
SELECT
|
|
463
|
+
'sri:' || entity_id || ':' || source_hash,
|
|
464
|
+
entity_id,
|
|
465
|
+
source_hash,
|
|
466
|
+
source_ref,
|
|
467
|
+
updated_at,
|
|
468
|
+
NULL
|
|
469
|
+
FROM (
|
|
470
|
+
SELECT
|
|
471
|
+
entity_id, source_hash, source_ref, updated_at,
|
|
472
|
+
ROW_NUMBER() OVER (
|
|
473
|
+
PARTITION BY entity_id, source_hash
|
|
474
|
+
ORDER BY updated_at ASC, id ASC
|
|
475
|
+
) AS rn
|
|
476
|
+
FROM ${prefix}entries
|
|
477
|
+
WHERE deleted_at IS NULL
|
|
478
|
+
AND source_hash IS NOT NULL
|
|
479
|
+
AND source_ref IS NOT NULL
|
|
480
|
+
) ranked
|
|
481
|
+
WHERE rn = 1;`
|
|
482
|
+
);
|
|
483
|
+
}
|
|
380
484
|
}
|
|
381
485
|
];
|
|
382
486
|
for (let i = 1; i < MIGRATIONS.length; i++) {
|
|
@@ -777,12 +881,22 @@ var EntryRepository = class extends BaseRepository {
|
|
|
777
881
|
/**
|
|
778
882
|
* Fetch recent non-deleted entries for an entity (limited), ordered by updated_at DESC.
|
|
779
883
|
* Used by MaintenanceService.doRunLibrarian().
|
|
884
|
+
*
|
|
885
|
+
* When `excludeSourceRef` is supplied, the WHERE clause filters out that
|
|
886
|
+
* sourceRef BEFORE applying LIMIT. The caller in IngestionService is
|
|
887
|
+
* `findRecentByEntityId(entityId, 500, tx)` — it discards same-sourceRef
|
|
888
|
+
* rows after the read to build a cross-sourceRef title index. If those
|
|
889
|
+
* rows are filtered out in SQL, the limit applies to usable rows only
|
|
890
|
+
* and a re-ingest where the caller's sourceRef owns most recent facts
|
|
891
|
+
* still leaves cross-sourceRef entries in the index.
|
|
780
892
|
*/
|
|
781
|
-
async findRecentByEntityId(entityId, limit, tx) {
|
|
893
|
+
async findRecentByEntityId(entityId, limit, tx, excludeSourceRef) {
|
|
782
894
|
const executor = this.getExecutor(tx);
|
|
895
|
+
const excludeClause = excludeSourceRef ? ` AND (source_ref IS NULL OR source_ref != ?)` : "";
|
|
896
|
+
const args = excludeSourceRef ? [entityId, excludeSourceRef, limit] : [entityId, limit];
|
|
783
897
|
const rows = await executor.getAllAsync(
|
|
784
|
-
`SELECT * FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL ORDER BY updated_at DESC LIMIT ?`,
|
|
785
|
-
|
|
898
|
+
`SELECT * FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL${excludeClause} ORDER BY updated_at DESC LIMIT ?`,
|
|
899
|
+
args
|
|
786
900
|
);
|
|
787
901
|
return rows.map(mapRowToFact);
|
|
788
902
|
}
|
|
@@ -1532,6 +1646,57 @@ var OutboxRepository = class extends BaseRepository {
|
|
|
1532
1646
|
}
|
|
1533
1647
|
};
|
|
1534
1648
|
|
|
1649
|
+
// src/repositories/SourceRefIndexRepository.ts
|
|
1650
|
+
var SourceRefIndexRepository = class extends BaseRepository {
|
|
1651
|
+
/**
|
|
1652
|
+
* Idempotent insert: the partial UNIQUE index catches concurrent inserts for
|
|
1653
|
+
* the same (entity_id, source_hash). The caller (IngestionService) catches
|
|
1654
|
+
* the resulting SQLITE_CONSTRAINT_UNIQUE and translates it to the per-mode
|
|
1655
|
+
* duplicate-hash outcome. Runtime IDs use the `sri_` prefix to avoid
|
|
1656
|
+
* collision with the deterministic `sri:<entity>:<hash>` IDs used by the v9
|
|
1657
|
+
* backfill (the colon separator keeps the two ID spaces disjoint).
|
|
1658
|
+
*/
|
|
1659
|
+
async upsert(entityId, sourceHash, sourceRef, tx) {
|
|
1660
|
+
const executor = this.getExecutor(tx);
|
|
1661
|
+
await executor.runAsync(
|
|
1662
|
+
`INSERT INTO ${this.prefix}source_ref_index (id, entity_id, source_hash, source_ref, created_at, deleted_at)
|
|
1663
|
+
VALUES (?, ?, ?, ?, ?, NULL)`,
|
|
1664
|
+
[generateId("sri_"), entityId, sourceHash, sourceRef, Date.now()]
|
|
1665
|
+
);
|
|
1666
|
+
}
|
|
1667
|
+
/**
|
|
1668
|
+
* Idempotent soft-delete of the live row for (entity_id, source_ref).
|
|
1669
|
+
* Called at the start of every ingestDocument to remove the prior run's
|
|
1670
|
+
* index row, so the new upsert doesn't collide with itself. No-op when the
|
|
1671
|
+
* row is already soft-deleted or never existed.
|
|
1672
|
+
*/
|
|
1673
|
+
async softDeleteByEntityAndSourceRef(entityId, sourceRef, tx) {
|
|
1674
|
+
const executor = this.getExecutor(tx);
|
|
1675
|
+
const now = Date.now();
|
|
1676
|
+
const result = await executor.runAsync(
|
|
1677
|
+
`UPDATE ${this.prefix}source_ref_index
|
|
1678
|
+
SET deleted_at = ?, created_at = ?
|
|
1679
|
+
WHERE entity_id = ? AND source_ref = ? AND deleted_at IS NULL`,
|
|
1680
|
+
[now, now, entityId, sourceRef]
|
|
1681
|
+
);
|
|
1682
|
+
return result.changes;
|
|
1683
|
+
}
|
|
1684
|
+
/**
|
|
1685
|
+
* Returns the live sourceRef holding the given hash, or null when no live
|
|
1686
|
+
* row exists. Used by the IngestionService pre-check (line 82) and the
|
|
1687
|
+
* catch-and-translate canonical lookup (line 212).
|
|
1688
|
+
*/
|
|
1689
|
+
async findActiveByEntityAndHash(entityId, sourceHash, tx) {
|
|
1690
|
+
const executor = this.getExecutor(tx);
|
|
1691
|
+
const row = await executor.getFirstAsync(
|
|
1692
|
+
`SELECT source_ref FROM ${this.prefix}source_ref_index
|
|
1693
|
+
WHERE entity_id = ? AND source_hash = ? AND deleted_at IS NULL`,
|
|
1694
|
+
[entityId, sourceHash]
|
|
1695
|
+
);
|
|
1696
|
+
return row?.source_ref ?? null;
|
|
1697
|
+
}
|
|
1698
|
+
};
|
|
1699
|
+
|
|
1535
1700
|
// src/repositories/TaskRepository.ts
|
|
1536
1701
|
function mapRowToTask(row) {
|
|
1537
1702
|
return {
|
|
@@ -1946,6 +2111,34 @@ var EdgeRepository = class extends BaseRepository {
|
|
|
1946
2111
|
const executor = this.getExecutor(tx);
|
|
1947
2112
|
await executor.runAsync(`DELETE FROM ${this.prefix}edges WHERE entity_id = ?`, [entityId]);
|
|
1948
2113
|
}
|
|
2114
|
+
/**
|
|
2115
|
+
* Hard-delete every edge whose `source_id` is in `sourceFactIds`, scoped to
|
|
2116
|
+
* `entityId`. Returns the number of rows deleted. Edges have no soft-delete
|
|
2117
|
+
* concept; this method is named `softDeleteBySourceFactIds` for symmetry with
|
|
2118
|
+
* `EntryRepository.softDeleteBySource` (which IS a soft delete), but its
|
|
2119
|
+
* semantics are immediate presence/absence removal. Use ONLY inside an open
|
|
2120
|
+
* transaction — the method requires `tx` and performs no autocommit.
|
|
2121
|
+
*
|
|
2122
|
+
* Used by `IngestionService.upsertGraphCore` to retire the prior parse's
|
|
2123
|
+
* edges whose source facts are being superseded by `softDeleteBySource`.
|
|
2124
|
+
*/
|
|
2125
|
+
async softDeleteBySourceFactIds(entityId, sourceFactIds, tx) {
|
|
2126
|
+
if (sourceFactIds.length === 0) return 0;
|
|
2127
|
+
const executor = this.getExecutor(tx);
|
|
2128
|
+
const chunkSize = 500;
|
|
2129
|
+
let totalChanges = 0;
|
|
2130
|
+
for (let i = 0; i < sourceFactIds.length; i += chunkSize) {
|
|
2131
|
+
const chunk = sourceFactIds.slice(i, i + chunkSize);
|
|
2132
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
2133
|
+
const result = await executor.runAsync(
|
|
2134
|
+
`DELETE FROM ${this.prefix}edges
|
|
2135
|
+
WHERE entity_id = ? AND source_id IN (${placeholders})`,
|
|
2136
|
+
[entityId, ...chunk]
|
|
2137
|
+
);
|
|
2138
|
+
totalChanges += result.changes;
|
|
2139
|
+
}
|
|
2140
|
+
return totalChanges;
|
|
2141
|
+
}
|
|
1949
2142
|
/**
|
|
1950
2143
|
* Multi-hop traversal from `sourceId` via SQLite `WITH RECURSIVE`. All filtering,
|
|
1951
2144
|
* dead-ending, cycle-guarding, capping, and ordering happens in this one query.
|
|
@@ -2132,14 +2325,25 @@ function mergeOntologyUpdates(current, updates) {
|
|
|
2132
2325
|
}
|
|
2133
2326
|
return { node_types, edge_types };
|
|
2134
2327
|
}
|
|
2135
|
-
function validateInlineEdges(sourceType, _targetType, edges, manifest) {
|
|
2136
|
-
|
|
2328
|
+
function validateInlineEdges(sourceType, _targetType, edges, manifest, opts) {
|
|
2329
|
+
const strict = opts?.strict === true;
|
|
2330
|
+
const entityId = opts?.entityId ?? "";
|
|
2331
|
+
if (!Array.isArray(edges)) {
|
|
2332
|
+
if (strict) throw new WikiStrictOntologyViolation(entityId, "edge", "");
|
|
2333
|
+
return [];
|
|
2334
|
+
}
|
|
2137
2335
|
const valid = [];
|
|
2138
2336
|
for (const edge of edges) {
|
|
2139
|
-
if (typeof edge?.edge_type !== "string" || typeof edge?.target_title !== "string")
|
|
2337
|
+
if (typeof edge?.edge_type !== "string" || typeof edge?.target_title !== "string") {
|
|
2338
|
+
if (strict) throw new WikiStrictOntologyViolation(entityId, "edge", String(edge?.edge_type ?? ""));
|
|
2339
|
+
continue;
|
|
2340
|
+
}
|
|
2140
2341
|
const defs = resolveEdgeDefinitions(edge.edge_type, manifest);
|
|
2141
2342
|
const match = defs.find((d) => d.source_type.toLowerCase() === sourceType.toLowerCase());
|
|
2142
|
-
if (!match)
|
|
2343
|
+
if (!match) {
|
|
2344
|
+
if (strict) throw new WikiStrictOntologyViolation(entityId, "edge", edge.edge_type);
|
|
2345
|
+
continue;
|
|
2346
|
+
}
|
|
2143
2347
|
valid.push({ edge_type: match.type, target_title: edge.target_title });
|
|
2144
2348
|
}
|
|
2145
2349
|
return valid;
|
|
@@ -2240,16 +2444,25 @@ var MetadataRepository = class extends BaseRepository {
|
|
|
2240
2444
|
await this.db.execAsync(`PRAGMA wal_checkpoint(TRUNCATE)`);
|
|
2241
2445
|
await this.db.execAsync(`VACUUM`);
|
|
2242
2446
|
}
|
|
2447
|
+
/**
|
|
2448
|
+
* Returns the set of distinct `entity_id` values across entries/tasks/events.
|
|
2449
|
+
*
|
|
2450
|
+
* Deliberately widened from a prior `WHERE deleted_at IS NULL` form: the
|
|
2451
|
+
* returned set intentionally includes entities whose only rows are
|
|
2452
|
+
* soft-deleted, so the host's maintenance sweep can visit such
|
|
2453
|
+
* "decommissioned-scope" namespaces and reap the soft-deleted rows. Sorting
|
|
2454
|
+
* is `entity_id COLLATE BINARY` for stable, locale-independent ordering.
|
|
2455
|
+
*/
|
|
2243
2456
|
async getDistinctEntityIds(tx) {
|
|
2244
2457
|
const executor = this.getExecutor(tx);
|
|
2245
2458
|
const rows = await executor.getAllAsync(
|
|
2246
2459
|
`SELECT DISTINCT entity_id FROM (
|
|
2247
|
-
SELECT entity_id FROM ${this.prefix}entries
|
|
2460
|
+
SELECT entity_id FROM ${this.prefix}entries
|
|
2248
2461
|
UNION
|
|
2249
|
-
SELECT entity_id FROM ${this.prefix}tasks
|
|
2462
|
+
SELECT entity_id FROM ${this.prefix}tasks
|
|
2250
2463
|
UNION
|
|
2251
2464
|
SELECT entity_id FROM ${this.prefix}events
|
|
2252
|
-
) ORDER BY entity_id`
|
|
2465
|
+
) ORDER BY entity_id COLLATE BINARY`
|
|
2253
2466
|
);
|
|
2254
2467
|
return rows.map((r) => r.entity_id);
|
|
2255
2468
|
}
|
|
@@ -2575,6 +2788,19 @@ var JobManager = class {
|
|
|
2575
2788
|
this.prefix = prefix;
|
|
2576
2789
|
this.activeMaintenanceJobs = /* @__PURE__ */ new Set();
|
|
2577
2790
|
this.activeIngestJobs = /* @__PURE__ */ new Map();
|
|
2791
|
+
/**
|
|
2792
|
+
* Per-(entityId, sourceHash) promise-chain lock. Serializes ingest races
|
|
2793
|
+
* at the application level (the v9 partial UNIQUE index enforces it at
|
|
2794
|
+
* the DB level as defense-in-depth). Two callers holding the same hash
|
|
2795
|
+
* see FIFO order; different hashes never block one another.
|
|
2796
|
+
*
|
|
2797
|
+
* Keyed `${entityId}\0${sourceHash}` so distinct (entityId, sourceHash)
|
|
2798
|
+
* pairs can never share a lock. A raw concatenation would let
|
|
2799
|
+
* ("ab", "c…") collide with ("a", "bc…"); the NUL separator makes that
|
|
2800
|
+
* impossible because neither component can contain NUL (entityIds are
|
|
2801
|
+
* user-supplied identifiers; sourceHashes are hex digests).
|
|
2802
|
+
*/
|
|
2803
|
+
this.hashLocks = /* @__PURE__ */ new Map();
|
|
2578
2804
|
this.statusSubscribers = /* @__PURE__ */ new Map();
|
|
2579
2805
|
/**
|
|
2580
2806
|
* Lookup table for acquireLock/releaseLock's dynamic-dispatch branch.
|
|
@@ -2592,6 +2818,51 @@ var JobManager = class {
|
|
|
2592
2818
|
ontologyBackfill: (id) => this._ontologyBackfillKey(id)
|
|
2593
2819
|
};
|
|
2594
2820
|
}
|
|
2821
|
+
_hashLockKey(entityId, sourceHash) {
|
|
2822
|
+
return `${entityId}\0${sourceHash}`;
|
|
2823
|
+
}
|
|
2824
|
+
/**
|
|
2825
|
+
* Acquire a promise-chain lock scoped to one `(entityId, sourceHash)` pair.
|
|
2826
|
+
* Returns a zero-argument release closure that removes the current tail
|
|
2827
|
+
* from the chain. Releasing is idempotent so a holder that double-fires
|
|
2828
|
+
* (e.g. on a finally that races an explicit release) cannot unblock
|
|
2829
|
+
* the next holder prematurely.
|
|
2830
|
+
*
|
|
2831
|
+
* Documented ordering: ingest acquires the hash lock FIRST, then the
|
|
2832
|
+
* synchronous sourceRef ingest lock, and the release closure unwinds in
|
|
2833
|
+
* the opposite order (sourceRef first, hash second). Hash-then-sourceRef
|
|
2834
|
+
* is required because the v9 partial UNIQUE index conflicts on the
|
|
2835
|
+
* (entity_id, source_hash) pair regardless of source_ref, so two callers
|
|
2836
|
+
* with DIFFERENT source_refs racing the same hash are the exact case the
|
|
2837
|
+
* TOCTOU race fix needs to serialize.
|
|
2838
|
+
*
|
|
2839
|
+
* Mechanics: each holder owns a private `released` promise that resolves
|
|
2840
|
+
* only when THIS holder calls release(). The map stores the holder's
|
|
2841
|
+
* `released` promise as the "tail" — the NEXT caller awaits that tail
|
|
2842
|
+
* via `previous = this.hashLocks.get(key)`. The current caller gets
|
|
2843
|
+
* their unique release function once `previous` (the prior holder's
|
|
2844
|
+
* release signal) settles, so each holder is guaranteed a UNIQUE release
|
|
2845
|
+
* closure bound to their own `released` promise.
|
|
2846
|
+
*/
|
|
2847
|
+
acquireHashLock(entityId, sourceHash) {
|
|
2848
|
+
const key = this._hashLockKey(entityId, sourceHash);
|
|
2849
|
+
const previous = this.hashLocks.get(key) ?? Promise.resolve();
|
|
2850
|
+
let resolveReleased;
|
|
2851
|
+
const released = new Promise((resolve) => {
|
|
2852
|
+
resolveReleased = resolve;
|
|
2853
|
+
});
|
|
2854
|
+
let fired = false;
|
|
2855
|
+
const release = () => {
|
|
2856
|
+
if (fired) return;
|
|
2857
|
+
fired = true;
|
|
2858
|
+
if (this.hashLocks.get(key) === released) {
|
|
2859
|
+
this.hashLocks.delete(key);
|
|
2860
|
+
}
|
|
2861
|
+
resolveReleased();
|
|
2862
|
+
};
|
|
2863
|
+
this.hashLocks.set(key, released);
|
|
2864
|
+
return previous.then(() => release);
|
|
2865
|
+
}
|
|
2595
2866
|
_pruneKey(entityId) {
|
|
2596
2867
|
return `${this.prefix}:${entityId}:prune`;
|
|
2597
2868
|
}
|
|
@@ -2805,6 +3076,38 @@ var JobManager = class {
|
|
|
2805
3076
|
this._notifyStatusSubscribers(entityId);
|
|
2806
3077
|
return true;
|
|
2807
3078
|
}
|
|
3079
|
+
/**
|
|
3080
|
+
* Centralized ingest lock acquisition. Acquires the hash lock FIRST
|
|
3081
|
+
* (FIFO across callers racing the same hash), then the synchronous
|
|
3082
|
+
* sourceRef ingest lock (rejects cross-entity conflicts and busy
|
|
3083
|
+
* operations). Returns a single zero-argument release closure that
|
|
3084
|
+
* unwinds in the opposite order — sourceRef first, hash second — so
|
|
3085
|
+
* the sourceRef lock is freed the instant its work is done and only
|
|
3086
|
+
* the hash lock keeps serializing the duplicate-content race window.
|
|
3087
|
+
*
|
|
3088
|
+
* On a sourceRef conflict (WikiBusyError thrown synchronously by the
|
|
3089
|
+
* underlying acquireLock) the hash lock is released BEFORE the error
|
|
3090
|
+
* propagates, so the next caller in the hash chain is not held up by
|
|
3091
|
+
* a request that never reached the DB.
|
|
3092
|
+
*/
|
|
3093
|
+
async acquireIngestLocks(entityId, sourceRef, sourceHash) {
|
|
3094
|
+
const releaseHash = await this.acquireHashLock(entityId, sourceHash);
|
|
3095
|
+
let sourceLockAcquired = false;
|
|
3096
|
+
try {
|
|
3097
|
+
this.acquireLock("ingest", entityId, sourceRef);
|
|
3098
|
+
sourceLockAcquired = true;
|
|
3099
|
+
} catch (err) {
|
|
3100
|
+
releaseHash();
|
|
3101
|
+
throw err;
|
|
3102
|
+
}
|
|
3103
|
+
return () => {
|
|
3104
|
+
try {
|
|
3105
|
+
if (sourceLockAcquired) this.releaseLock("ingest", entityId, sourceRef);
|
|
3106
|
+
} finally {
|
|
3107
|
+
releaseHash();
|
|
3108
|
+
}
|
|
3109
|
+
};
|
|
3110
|
+
}
|
|
2808
3111
|
/**
|
|
2809
3112
|
* Validates then acquires global + per-entity import locks atomically.
|
|
2810
3113
|
* Validates all entities before acquiring any lock (same as current importDump semantics).
|
|
@@ -3245,11 +3548,14 @@ var DEFAULT_CHUNK_OVERLAP = 400;
|
|
|
3245
3548
|
|
|
3246
3549
|
// src/services/IngestionService.ts
|
|
3247
3550
|
var IngestionService = class {
|
|
3248
|
-
constructor(db, prefix, options, entryRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
3551
|
+
constructor(db, prefix, options, entryRepo, sourceRefIndexRepo, metadataRepo, edgeRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
3249
3552
|
this.db = db;
|
|
3250
3553
|
this.prefix = prefix;
|
|
3251
3554
|
this.options = options;
|
|
3252
3555
|
this.entryRepo = entryRepo;
|
|
3556
|
+
this.sourceRefIndexRepo = sourceRefIndexRepo;
|
|
3557
|
+
this.metadataRepo = metadataRepo;
|
|
3558
|
+
this.edgeRepo = edgeRepo;
|
|
3253
3559
|
this.searchService = searchService;
|
|
3254
3560
|
this.jobManager = jobManager;
|
|
3255
3561
|
this.embeddingService = embeddingService;
|
|
@@ -3261,18 +3567,6 @@ var IngestionService = class {
|
|
|
3261
3567
|
if (!sourceRef) throw new Error("Invalid sourceRef");
|
|
3262
3568
|
const sourceHash = normalizeSourceHash(params.sourceHash);
|
|
3263
3569
|
if (!sourceHash) throw new Error("Invalid sourceHash (must be 64-char hex string)");
|
|
3264
|
-
const onDuplicateHash = opts?.onDuplicateHash ?? "ingest";
|
|
3265
|
-
if (onDuplicateHash !== "ingest") {
|
|
3266
|
-
const refs = await this.entryRepo.findSourceRefsByHash(entityId, sourceHash);
|
|
3267
|
-
const others = refs.filter((r) => r !== sourceRef);
|
|
3268
|
-
if (others.length > 0) {
|
|
3269
|
-
const canonical = others[0];
|
|
3270
|
-
if (onDuplicateHash === "throw") {
|
|
3271
|
-
throw new WikiDuplicateHashError({ canonical, sourceHash, entityId });
|
|
3272
|
-
}
|
|
3273
|
-
return { truncated: false, chunks: 0, duplicateOf: canonical };
|
|
3274
|
-
}
|
|
3275
|
-
}
|
|
3276
3570
|
const maxChunkLength = params.maxChunkLength ?? this.options.config?.maxChunkLength ?? DEFAULT_MAX_CHUNK_LENGTH;
|
|
3277
3571
|
const rawOverlap = params.chunkOverlap ?? this.options.config?.chunkOverlap ?? DEFAULT_CHUNK_OVERLAP;
|
|
3278
3572
|
const chunkOverlap = Math.min(
|
|
@@ -3284,8 +3578,18 @@ var IngestionService = class {
|
|
|
3284
3578
|
if (typeof params.documentChunk !== "string") {
|
|
3285
3579
|
throw new Error(`documentChunk must be a string, received ${typeof params.documentChunk}`);
|
|
3286
3580
|
}
|
|
3287
|
-
this.jobManager.
|
|
3581
|
+
const releaseIngestLocks = await this.jobManager.acquireIngestLocks(entityId, sourceRef, sourceHash);
|
|
3288
3582
|
try {
|
|
3583
|
+
const onDuplicateHash = opts?.onDuplicateHash ?? "ingest";
|
|
3584
|
+
if (onDuplicateHash !== "ingest") {
|
|
3585
|
+
const canonical = await this.sourceRefIndexRepo.findActiveByEntityAndHash(entityId, sourceHash);
|
|
3586
|
+
if (canonical !== null && canonical !== sourceRef) {
|
|
3587
|
+
if (onDuplicateHash === "throw") {
|
|
3588
|
+
throw new WikiDuplicateHashError({ canonical, sourceHash, entityId });
|
|
3589
|
+
}
|
|
3590
|
+
return { truncated: false, chunks: 0, duplicateOf: canonical };
|
|
3591
|
+
}
|
|
3592
|
+
}
|
|
3289
3593
|
const { chunks, truncated } = chunkText(params.documentChunk, maxChunkLength, chunkOverlap);
|
|
3290
3594
|
if (chunks.length === 0) return { truncated: false, chunks: 0 };
|
|
3291
3595
|
const chunkResults = await withConcurrency(
|
|
@@ -3321,68 +3625,85 @@ var IngestionService = class {
|
|
|
3321
3625
|
const now = Date.now();
|
|
3322
3626
|
const insertedFacts = [];
|
|
3323
3627
|
const deletedSourceFactIds = [];
|
|
3324
|
-
|
|
3325
|
-
|
|
3326
|
-
|
|
3327
|
-
|
|
3328
|
-
|
|
3329
|
-
|
|
3330
|
-
|
|
3331
|
-
|
|
3332
|
-
|
|
3333
|
-
|
|
3334
|
-
}
|
|
3335
|
-
|
|
3336
|
-
|
|
3337
|
-
|
|
3338
|
-
|
|
3339
|
-
|
|
3340
|
-
|
|
3341
|
-
|
|
3342
|
-
|
|
3628
|
+
try {
|
|
3629
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
3630
|
+
deletedSourceFactIds.push(...await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false));
|
|
3631
|
+
const titleIndex = /* @__PURE__ */ new Map();
|
|
3632
|
+
const existingFacts = await this.entryRepo.findRecentByEntityId(entityId, 500, tx, sourceRef);
|
|
3633
|
+
for (const existing of existingFacts) {
|
|
3634
|
+
titleIndex.set(normalizeTitleKey(existing.title), {
|
|
3635
|
+
id: existing.id,
|
|
3636
|
+
okf_type: existing.okf_type ?? null
|
|
3637
|
+
});
|
|
3638
|
+
}
|
|
3639
|
+
let ontologyState = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
3640
|
+
let { mode, manifest } = ontologyState;
|
|
3641
|
+
const hostNodes = [];
|
|
3642
|
+
const rawEdgeRequests = [];
|
|
3643
|
+
for (const { facts, ontology_updates } of orderedChunkFacts) {
|
|
3644
|
+
if (mode === "emergent" && ontology_updates && this.ontologyService) {
|
|
3645
|
+
manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontology_updates, tx);
|
|
3646
|
+
ontologyState = await this.ontologyService.getEffectiveState(entityId, tx);
|
|
3647
|
+
mode = ontologyState.mode;
|
|
3648
|
+
}
|
|
3649
|
+
for (const fact of facts) {
|
|
3650
|
+
const ontologyFact = fact;
|
|
3651
|
+
const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false }) ?? { okf_type: null, edges: [] };
|
|
3652
|
+
const id = generateId("fact_");
|
|
3653
|
+
hostNodes.push({
|
|
3654
|
+
id,
|
|
3655
|
+
type: ontologyFact.okf_type ?? "",
|
|
3656
|
+
title: fact.title,
|
|
3657
|
+
body: fact.body,
|
|
3658
|
+
// Forward the LLM-extracted tags/confidence through the
|
|
3659
|
+
// extracted-shape upsertGraphCore path so the entries row
|
|
3660
|
+
// stores them (search filterability, heal-candidate
|
|
3661
|
+
// selection, runReembed embedding-text signal). The public
|
|
3662
|
+
// WikiMemory.upsertGraph API doesn't accept tags/confidence
|
|
3663
|
+
// — host-supplied deterministic nodes default to [] and
|
|
3664
|
+
// 'certain' as documented.
|
|
3665
|
+
tags: fact.tags,
|
|
3666
|
+
confidence: fact.confidence
|
|
3667
|
+
});
|
|
3668
|
+
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
3669
|
+
titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
|
|
3670
|
+
if (normalized.edges.length > 0) {
|
|
3671
|
+
rawEdgeRequests.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
3672
|
+
}
|
|
3673
|
+
}
|
|
3343
3674
|
}
|
|
3344
|
-
|
|
3345
|
-
|
|
3346
|
-
const
|
|
3347
|
-
|
|
3348
|
-
|
|
3349
|
-
|
|
3350
|
-
|
|
3351
|
-
|
|
3352
|
-
|
|
3353
|
-
|
|
3354
|
-
|
|
3355
|
-
|
|
3356
|
-
|
|
3357
|
-
source_ref: sourceRef,
|
|
3358
|
-
created_at: now,
|
|
3359
|
-
updated_at: now,
|
|
3360
|
-
last_accessed_at: null,
|
|
3361
|
-
access_count: 0,
|
|
3362
|
-
deleted_at: null,
|
|
3363
|
-
okf_type: normalized.okf_type
|
|
3364
|
-
};
|
|
3365
|
-
await this.entryRepo.upsert(wikiFact, tx);
|
|
3366
|
-
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
3367
|
-
titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
|
|
3368
|
-
if (normalized.edges.length > 0) {
|
|
3369
|
-
pendingEdges.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
3675
|
+
const hostEdges = [];
|
|
3676
|
+
for (const req of rawEdgeRequests) {
|
|
3677
|
+
const resolved = this.ontologyService?.resolveEdges(
|
|
3678
|
+
entityId,
|
|
3679
|
+
req.sourceId,
|
|
3680
|
+
req.sourceType,
|
|
3681
|
+
req.edges,
|
|
3682
|
+
manifest,
|
|
3683
|
+
titleIndex,
|
|
3684
|
+
now
|
|
3685
|
+
) ?? [];
|
|
3686
|
+
for (const e of resolved) {
|
|
3687
|
+
hostEdges.push({ type: e.edge_type, sourceId: e.source_id, targetId: e.target_id });
|
|
3370
3688
|
}
|
|
3371
3689
|
}
|
|
3372
|
-
|
|
3373
|
-
for (const item of pendingEdges) {
|
|
3374
|
-
await this.ontologyService?.resolveAndPersistEdges(
|
|
3690
|
+
await this.upsertGraphCore(
|
|
3375
3691
|
entityId,
|
|
3376
|
-
|
|
3377
|
-
item.sourceType,
|
|
3378
|
-
item.edges ?? [],
|
|
3379
|
-
manifest,
|
|
3380
|
-
titleIndex,
|
|
3692
|
+
{ sourceRef, sourceHash, nodes: hostNodes, edges: hostEdges },
|
|
3381
3693
|
tx,
|
|
3382
|
-
|
|
3694
|
+
{ strict: false }
|
|
3383
3695
|
);
|
|
3696
|
+
});
|
|
3697
|
+
} catch (err) {
|
|
3698
|
+
const sqliteCode = err instanceof WikiTransactionError ? err.sqliteErrorCode : extractSqliteCode(err);
|
|
3699
|
+
if (sqliteCode !== "SQLITE_CONSTRAINT_UNIQUE") throw err;
|
|
3700
|
+
const canonical = await this.sourceRefIndexRepo.findActiveByEntityAndHash(entityId, sourceHash);
|
|
3701
|
+
if (canonical === null) throw err;
|
|
3702
|
+
if (onDuplicateHash === "throw" || onDuplicateHash === "ingest") {
|
|
3703
|
+
throw new WikiDuplicateHashError({ canonical, sourceHash, entityId });
|
|
3384
3704
|
}
|
|
3385
|
-
|
|
3705
|
+
return { truncated: false, chunks: 0, duplicateOf: canonical };
|
|
3706
|
+
}
|
|
3386
3707
|
await this.searchService.sync(entityId);
|
|
3387
3708
|
const uniqueDeletedSourceFactIds = Array.from(new Set(deletedSourceFactIds));
|
|
3388
3709
|
for (const factId of uniqueDeletedSourceFactIds) {
|
|
@@ -3398,9 +3719,129 @@ var IngestionService = class {
|
|
|
3398
3719
|
this.searchService.evictCache(entityId);
|
|
3399
3720
|
return { truncated, chunks: chunks.length };
|
|
3400
3721
|
} finally {
|
|
3401
|
-
|
|
3722
|
+
releaseIngestLocks();
|
|
3402
3723
|
}
|
|
3403
3724
|
}
|
|
3725
|
+
/**
|
|
3726
|
+
* Implements spec §2 data flow steps a–j with the host-facing
|
|
3727
|
+
* `{sourceRef, sourceHash, nodes, edges}` parameter shape. Shared between
|
|
3728
|
+
* `ingestDocument` (with `{ strict: false }` to preserve current behavior)
|
|
3729
|
+
* and `WikiMemory.upsertGraph` (which lets the persisted ontology mode
|
|
3730
|
+
* decide strictness).
|
|
3731
|
+
*
|
|
3732
|
+
* Runs ENTIRELY inside the supplied `tx`. Does not acquire locks, does not
|
|
3733
|
+
* open a nested transaction, does not perform post-commit work (search
|
|
3734
|
+
* sync, embedding, cache eviction). Those concerns remain in `ingestDocument`.
|
|
3735
|
+
*
|
|
3736
|
+
* The pre-flight validation (steps a–c) lives INSIDE this method per the
|
|
3737
|
+
* spec, so the public `WikiMemory.upsertGraph` can be a thin wrapper that
|
|
3738
|
+
* performs only the C2 probe before delegating.
|
|
3739
|
+
*
|
|
3740
|
+
* @returns Counts: nodesWritten (validated nodes persisted), edgesWritten
|
|
3741
|
+
* (manifest-valid edges persisted), superseded (prior facts soft-deleted
|
|
3742
|
+
* plus prior source-ref edges hard-deleted).
|
|
3743
|
+
*/
|
|
3744
|
+
async upsertGraphCore(entityId, params, tx, opts) {
|
|
3745
|
+
const now = Date.now();
|
|
3746
|
+
const ontologyState = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
3747
|
+
let { mode, manifest } = ontologyState;
|
|
3748
|
+
const strictEffective = opts?.strict === false ? false : opts?.strict === true || mode === "strict";
|
|
3749
|
+
const wikiFacts = [];
|
|
3750
|
+
for (const node of params.nodes) {
|
|
3751
|
+
const factLike = { okf_type: node.type, edges: [] };
|
|
3752
|
+
const normalized = this.ontologyService?.validateAndNormalizeFact(factLike, manifest, {
|
|
3753
|
+
strict: strictEffective,
|
|
3754
|
+
entityId
|
|
3755
|
+
}) ?? { okf_type: null};
|
|
3756
|
+
const wikiFact = {
|
|
3757
|
+
id: node.id,
|
|
3758
|
+
entity_id: entityId,
|
|
3759
|
+
title: node.title,
|
|
3760
|
+
body: node.body ?? "",
|
|
3761
|
+
// Public upsertGraph callers don't supply tags/confidence, so they
|
|
3762
|
+
// default to [] / 'certain' for deterministic AST-grade facts.
|
|
3763
|
+
// ingestDocument routes through this method for the LLM path and
|
|
3764
|
+
// forwards the LLM-supplied tags + confidence, preserving the prior
|
|
3765
|
+
// behavior (search filterability, heal-candidate selection, runReembed
|
|
3766
|
+
// embedding-text signal).
|
|
3767
|
+
tags: node.tags ? Array.from(node.tags) : [],
|
|
3768
|
+
confidence: node.confidence ?? "certain",
|
|
3769
|
+
source_type: "immutable_document",
|
|
3770
|
+
source_hash: params.sourceHash,
|
|
3771
|
+
source_ref: params.sourceRef,
|
|
3772
|
+
created_at: now,
|
|
3773
|
+
updated_at: now,
|
|
3774
|
+
last_accessed_at: null,
|
|
3775
|
+
access_count: 0,
|
|
3776
|
+
deleted_at: null,
|
|
3777
|
+
okf_type: normalized.okf_type
|
|
3778
|
+
};
|
|
3779
|
+
wikiFacts.push(wikiFact);
|
|
3780
|
+
}
|
|
3781
|
+
const sourceIdToType = /* @__PURE__ */ new Map();
|
|
3782
|
+
for (const fact of wikiFacts) sourceIdToType.set(fact.id, fact.okf_type ?? null);
|
|
3783
|
+
const hasConstraints = (manifest.node_types?.length ?? 0) > 0 || (manifest.edge_types?.length ?? 0) > 0;
|
|
3784
|
+
const validEdges = [];
|
|
3785
|
+
for (const edge of params.edges) {
|
|
3786
|
+
if (!hasConstraints) {
|
|
3787
|
+
validEdges.push({
|
|
3788
|
+
id: edge.id ?? generateId(),
|
|
3789
|
+
entity_id: entityId,
|
|
3790
|
+
source_id: edge.sourceId,
|
|
3791
|
+
target_id: edge.targetId,
|
|
3792
|
+
edge_type: edge.type,
|
|
3793
|
+
created_at: now
|
|
3794
|
+
});
|
|
3795
|
+
continue;
|
|
3796
|
+
}
|
|
3797
|
+
if (!sourceIdToType.has(edge.sourceId)) {
|
|
3798
|
+
validEdges.push({
|
|
3799
|
+
id: edge.id ?? generateId(),
|
|
3800
|
+
entity_id: entityId,
|
|
3801
|
+
source_id: edge.sourceId,
|
|
3802
|
+
target_id: edge.targetId,
|
|
3803
|
+
edge_type: edge.type,
|
|
3804
|
+
created_at: now
|
|
3805
|
+
});
|
|
3806
|
+
continue;
|
|
3807
|
+
}
|
|
3808
|
+
const sourceType = sourceIdToType.get(edge.sourceId);
|
|
3809
|
+
const candidates = (manifest.edge_types ?? []).filter(
|
|
3810
|
+
(d) => d.type.toLowerCase() === edge.type.toLowerCase() && d.source_type.toLowerCase() === (sourceType ?? "").toLowerCase()
|
|
3811
|
+
);
|
|
3812
|
+
const match = candidates[0];
|
|
3813
|
+
if (!match) {
|
|
3814
|
+
if (strictEffective) throw new WikiStrictOntologyViolation(entityId, "edge", edge.type);
|
|
3815
|
+
continue;
|
|
3816
|
+
}
|
|
3817
|
+
validEdges.push({
|
|
3818
|
+
id: edge.id ?? generateId(),
|
|
3819
|
+
entity_id: entityId,
|
|
3820
|
+
source_id: edge.sourceId,
|
|
3821
|
+
target_id: edge.targetId,
|
|
3822
|
+
edge_type: match.type,
|
|
3823
|
+
created_at: now
|
|
3824
|
+
});
|
|
3825
|
+
}
|
|
3826
|
+
const deletedFactIds = await this.entryRepo.findIdsBySource(entityId, params.sourceRef, null, tx, false);
|
|
3827
|
+
await this.entryRepo.softDeleteBySource(entityId, tx, params.sourceRef, null);
|
|
3828
|
+
const deletedEdgeCount = await this.edgeRepo.softDeleteBySourceFactIds(entityId, deletedFactIds, tx);
|
|
3829
|
+
await this.sourceRefIndexRepo.softDeleteByEntityAndSourceRef(entityId, params.sourceRef, tx);
|
|
3830
|
+
await this.sourceRefIndexRepo.upsert(entityId, params.sourceHash, params.sourceRef, tx);
|
|
3831
|
+
for (const wikiFact of wikiFacts) {
|
|
3832
|
+
await this.entryRepo.upsert(wikiFact, tx);
|
|
3833
|
+
}
|
|
3834
|
+
let edgesInsertedCount = 0;
|
|
3835
|
+
for (const edge of validEdges) {
|
|
3836
|
+
const inserted = await this.edgeRepo.addIgnoreDuplicate(edge, tx);
|
|
3837
|
+
if (inserted) edgesInsertedCount++;
|
|
3838
|
+
}
|
|
3839
|
+
return {
|
|
3840
|
+
nodesWritten: wikiFacts.length,
|
|
3841
|
+
edgesWritten: edgesInsertedCount,
|
|
3842
|
+
superseded: deletedFactIds.length + deletedEdgeCount
|
|
3843
|
+
};
|
|
3844
|
+
}
|
|
3404
3845
|
};
|
|
3405
3846
|
|
|
3406
3847
|
// src/services/BoundedLlmCall.ts
|
|
@@ -3522,11 +3963,12 @@ var HEAL_MAX_PROMPT_CHARS = 4e4;
|
|
|
3522
3963
|
var HEAL_BATCH_SIZE = 25;
|
|
3523
3964
|
var HEAL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
3524
3965
|
var MaintenanceService = class {
|
|
3525
|
-
constructor(db, prefix, options, entryRepo, taskRepo, eventRepo, metadataRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
3966
|
+
constructor(db, prefix, options, entryRepo, sourceRefIndexRepo, taskRepo, eventRepo, metadataRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
3526
3967
|
this.db = db;
|
|
3527
3968
|
this.prefix = prefix;
|
|
3528
3969
|
this.options = options;
|
|
3529
3970
|
this.entryRepo = entryRepo;
|
|
3971
|
+
this.sourceRefIndexRepo = sourceRefIndexRepo;
|
|
3530
3972
|
this.taskRepo = taskRepo;
|
|
3531
3973
|
this.eventRepo = eventRepo;
|
|
3532
3974
|
this.metadataRepo = metadataRepo;
|
|
@@ -3725,10 +4167,12 @@ var MaintenanceService = class {
|
|
|
3725
4167
|
const entryPromise = params.entryId ? this.entryRepo.softDelete(params.entryId, entityId, tx).then((r) => r.changes > 0) : null;
|
|
3726
4168
|
const taskDeletedPromise = params.taskId ? this.taskRepo.softDeleteById(params.taskId, entityId, tx).then((r) => r.changes > 0) : null;
|
|
3727
4169
|
const refPromise = sourceRef || sourceHash ? this.entryRepo.softDeleteBySource(entityId, tx, sourceRef, sourceHash) : null;
|
|
4170
|
+
const sourceRefIndexPromise = sourceRef ? this.sourceRefIndexRepo.softDeleteByEntityAndSourceRef(entityId, sourceRef, tx).then(() => void 0) : null;
|
|
3728
4171
|
const [entryResult, taskResult, refResult] = await Promise.all([
|
|
3729
4172
|
entryPromise ?? Promise.resolve(false),
|
|
3730
4173
|
taskDeletedPromise ?? Promise.resolve(false),
|
|
3731
|
-
refPromise ?? Promise.resolve(0)
|
|
4174
|
+
refPromise ?? Promise.resolve(0),
|
|
4175
|
+
sourceRefIndexPromise ?? Promise.resolve(void 0)
|
|
3732
4176
|
]);
|
|
3733
4177
|
if (entryResult) deletedEntries++;
|
|
3734
4178
|
if (taskResult) deletedTasks++;
|
|
@@ -3846,7 +4290,7 @@ var MaintenanceService = class {
|
|
|
3846
4290
|
}
|
|
3847
4291
|
if (skip) continue;
|
|
3848
4292
|
const ontologyFact = fact;
|
|
3849
|
-
const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest) ?? { okf_type: null, edges: [] };
|
|
4293
|
+
const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false }) ?? { okf_type: null, edges: [] };
|
|
3850
4294
|
const id = generateId("fact_");
|
|
3851
4295
|
const factObj = {
|
|
3852
4296
|
id,
|
|
@@ -4231,7 +4675,8 @@ var MaintenanceService = class {
|
|
|
4231
4675
|
okf_type: classification.okf_type,
|
|
4232
4676
|
edges: classification.edges
|
|
4233
4677
|
},
|
|
4234
|
-
manifest
|
|
4678
|
+
manifest,
|
|
4679
|
+
{ strict: false }
|
|
4235
4680
|
);
|
|
4236
4681
|
if (!normalized.okf_type) {
|
|
4237
4682
|
failedValidation++;
|
|
@@ -5621,16 +6066,27 @@ var OntologyService = class {
|
|
|
5621
6066
|
this.invalidateCache(entityId);
|
|
5622
6067
|
return merged;
|
|
5623
6068
|
}
|
|
5624
|
-
validateAndNormalizeFact(fact, manifest) {
|
|
6069
|
+
validateAndNormalizeFact(fact, manifest, opts) {
|
|
5625
6070
|
const rawType = typeof fact.okf_type === "string" ? fact.okf_type : "";
|
|
6071
|
+
const strict = opts?.strict === true;
|
|
5626
6072
|
const canonical = resolveNodeType(rawType, manifest);
|
|
5627
|
-
if (!canonical)
|
|
5628
|
-
|
|
6073
|
+
if (!canonical) {
|
|
6074
|
+
if (strict) throw new WikiStrictOntologyViolation(opts?.entityId ?? "", "node", rawType);
|
|
6075
|
+
return { okf_type: null, edges: [] };
|
|
6076
|
+
}
|
|
6077
|
+
const edges = validateInlineEdges(canonical, null, fact.edges ?? [], manifest, opts);
|
|
5629
6078
|
return { okf_type: canonical, edges };
|
|
5630
6079
|
}
|
|
5631
|
-
|
|
5632
|
-
|
|
5633
|
-
|
|
6080
|
+
/**
|
|
6081
|
+
* Pure resolver: given a source type, LLM-supplied edges, the manifest,
|
|
6082
|
+
* and a title index, return a list of concrete `WikiEdge` objects ready
|
|
6083
|
+
* for persistence. Performs no DB writes. Used by callers that batch edge
|
|
6084
|
+
* writes inside their own transaction (e.g. `IngestionService.ingestDocument`
|
|
6085
|
+
* when delegating to `upsertGraphCore`).
|
|
6086
|
+
*/
|
|
6087
|
+
resolveEdges(entityId, sourceId, sourceType, edges, manifest, titleIndex, now) {
|
|
6088
|
+
if (!sourceType || edges.length === 0) return [];
|
|
6089
|
+
const out = [];
|
|
5634
6090
|
for (const edge of edges) {
|
|
5635
6091
|
const candidates = resolveEdgeDefinitions(edge.edge_type, manifest).filter((d) => d.source_type.toLowerCase() === sourceType.toLowerCase());
|
|
5636
6092
|
if (candidates.length === 0) continue;
|
|
@@ -5641,15 +6097,29 @@ var OntologyService = class {
|
|
|
5641
6097
|
(d) => d.target_type.toLowerCase() === (target.okf_type ?? "").toLowerCase()
|
|
5642
6098
|
);
|
|
5643
6099
|
if (!def) continue;
|
|
5644
|
-
|
|
6100
|
+
out.push({
|
|
5645
6101
|
id: generateId(),
|
|
5646
6102
|
entity_id: entityId,
|
|
5647
6103
|
source_id: sourceId,
|
|
5648
6104
|
target_id: target.id,
|
|
5649
6105
|
edge_type: def.type,
|
|
5650
6106
|
created_at: now
|
|
5651
|
-
};
|
|
5652
|
-
|
|
6107
|
+
});
|
|
6108
|
+
}
|
|
6109
|
+
return out;
|
|
6110
|
+
}
|
|
6111
|
+
/**
|
|
6112
|
+
* Backwards-compatible wrapper: resolves edges via {@link resolveEdges},
|
|
6113
|
+
* then persists each via `edgeRepo.addIgnoreDuplicate` in the supplied
|
|
6114
|
+
* transaction. Returns the number of edges persisted. Used by paths that
|
|
6115
|
+
* want self-contained edge persistence (e.g. MaintenanceService heal /
|
|
6116
|
+
* backfill callers that do not have a separate edge-write step).
|
|
6117
|
+
*/
|
|
6118
|
+
async resolveAndPersistEdges(entityId, sourceId, sourceType, edges, manifest, titleIndex, tx, now) {
|
|
6119
|
+
const resolved = this.resolveEdges(entityId, sourceId, sourceType, edges, manifest, titleIndex, now);
|
|
6120
|
+
let persisted = 0;
|
|
6121
|
+
for (const edge of resolved) {
|
|
6122
|
+
const inserted = await this.edgeRepo.addIgnoreDuplicate(edge, tx);
|
|
5653
6123
|
if (inserted) persisted++;
|
|
5654
6124
|
}
|
|
5655
6125
|
return persisted;
|
|
@@ -5705,6 +6175,7 @@ var WikiMemory = class {
|
|
|
5705
6175
|
}
|
|
5706
6176
|
this.outboxRepo = new OutboxRepository(this.db, this.prefix, !!options.config?.enableOutbox);
|
|
5707
6177
|
this.entryRepo = new EntryRepository(this.db, this.prefix, this.outboxRepo);
|
|
6178
|
+
this.sourceRefIndexRepo = new SourceRefIndexRepository(this.db, this.prefix);
|
|
5708
6179
|
this.taskRepo = new TaskRepository(this.db, this.prefix, this.outboxRepo);
|
|
5709
6180
|
this.eventRepo = new EventRepository(this.db, this.prefix);
|
|
5710
6181
|
this.edgeRepo = new EdgeRepository(this.db, this.prefix);
|
|
@@ -5723,6 +6194,9 @@ var WikiMemory = class {
|
|
|
5723
6194
|
this.prefix,
|
|
5724
6195
|
this.options,
|
|
5725
6196
|
this.entryRepo,
|
|
6197
|
+
this.sourceRefIndexRepo,
|
|
6198
|
+
this.metadataRepo,
|
|
6199
|
+
this.edgeRepo,
|
|
5726
6200
|
this.searchService,
|
|
5727
6201
|
this.jobManager,
|
|
5728
6202
|
this.embeddingService,
|
|
@@ -5734,6 +6208,7 @@ var WikiMemory = class {
|
|
|
5734
6208
|
this.prefix,
|
|
5735
6209
|
this.options,
|
|
5736
6210
|
this.entryRepo,
|
|
6211
|
+
this.sourceRefIndexRepo,
|
|
5737
6212
|
this.taskRepo,
|
|
5738
6213
|
this.eventRepo,
|
|
5739
6214
|
this.metadataRepo,
|
|
@@ -5798,6 +6273,7 @@ var WikiMemory = class {
|
|
|
5798
6273
|
promptService: this.promptService,
|
|
5799
6274
|
graphTraversalService: this.graphTraversalService,
|
|
5800
6275
|
entryRepo: this.entryRepo,
|
|
6276
|
+
sourceRefIndexRepo: this.sourceRefIndexRepo,
|
|
5801
6277
|
metadataRepo: this.metadataRepo,
|
|
5802
6278
|
jobManager: this.jobManager
|
|
5803
6279
|
};
|
|
@@ -5874,7 +6350,7 @@ var WikiMemory = class {
|
|
|
5874
6350
|
const latestHashes = await this.entryRepo.findLatestSourceHashes(entityId, normalized.map((e) => e.sourceRef));
|
|
5875
6351
|
const distinctHashes = Array.from(new Set(normalized.map((e) => e.sourceHash)));
|
|
5876
6352
|
const dupRefs = await Promise.all(
|
|
5877
|
-
distinctHashes.map((h) => this.
|
|
6353
|
+
distinctHashes.map((h) => this.sourceRefIndexRepo.findActiveByEntityAndHash(entityId, h))
|
|
5878
6354
|
);
|
|
5879
6355
|
const dupMap = /* @__PURE__ */ new Map();
|
|
5880
6356
|
for (let i = 0; i < distinctHashes.length; i++) {
|
|
@@ -5883,12 +6359,10 @@ var WikiMemory = class {
|
|
|
5883
6359
|
return normalized.map((e) => {
|
|
5884
6360
|
const stored = latestHashes.get(e.sourceRef);
|
|
5885
6361
|
const changed = stored === void 0 || stored === null || normalizeSourceHash(stored) !== e.sourceHash;
|
|
5886
|
-
const
|
|
5887
|
-
|
|
5888
|
-
if (others.length === 0) {
|
|
6362
|
+
const canonical = dupMap.get(e.sourceHash) ?? null;
|
|
6363
|
+
if (canonical === null || canonical === e.sourceRef) {
|
|
5889
6364
|
return { sourceRef: e.rawSourceRef, changed };
|
|
5890
6365
|
}
|
|
5891
|
-
const canonical = others[0];
|
|
5892
6366
|
return { sourceRef: e.rawSourceRef, changed, duplicateOf: canonical };
|
|
5893
6367
|
});
|
|
5894
6368
|
}
|
|
@@ -5903,13 +6377,47 @@ var WikiMemory = class {
|
|
|
5903
6377
|
return this.entryRepo.listSourceRefs(entityId);
|
|
5904
6378
|
}
|
|
5905
6379
|
/**
|
|
5906
|
-
* Returns the live source_refs for an entity that hold the given source_hash
|
|
5907
|
-
*
|
|
5908
|
-
*
|
|
5909
|
-
*
|
|
6380
|
+
* Returns the live source_refs for an entity that hold the given source_hash.
|
|
6381
|
+
* With v9, source_ref_index is the source of truth for the sourceRef-level
|
|
6382
|
+
* TOCTOU-race invariant: at most one sourceRef can hold a given
|
|
6383
|
+
* (entity_id, source_hash). The result is either a single-element array
|
|
6384
|
+
* (one canonical ref) or empty (no live ref holds the hash). Returned as
|
|
6385
|
+
* an array to preserve the existing public-API shape used by hosts
|
|
6386
|
+
* auditing duplicate-content collisions.
|
|
5910
6387
|
*/
|
|
5911
6388
|
async findSourceRefsByHash(entityId, sourceHash) {
|
|
5912
|
-
|
|
6389
|
+
const canonical = await this.sourceRefIndexRepo.findActiveByEntityAndHash(entityId, sourceHash);
|
|
6390
|
+
return canonical === null ? [] : [canonical];
|
|
6391
|
+
}
|
|
6392
|
+
/**
|
|
6393
|
+
* Returns the set of `entity_id` values with at least one row in this
|
|
6394
|
+
* database, INCLUDING entities whose only remaining rows are soft-deleted
|
|
6395
|
+
* — sorted ascending `entity_id COLLATE BINARY`. Empty array when the
|
|
6396
|
+
* database has no entities.
|
|
6397
|
+
*
|
|
6398
|
+
* The deliberate inclusion of soft-deleted-only entities closes the
|
|
6399
|
+
* decommissioned-scope leak: a scoped namespace whose documents have all
|
|
6400
|
+
* been forgotten or superseded must still appear so host maintenance
|
|
6401
|
+
* sweeps (`runLibrarian`, `runHeal`, `runOntologyBackfill`, `runPrune`)
|
|
6402
|
+
* can visit it and reap the soft-deleted rows.
|
|
6403
|
+
*
|
|
6404
|
+
* Read-only. Never acquires a lock. Never opens a transaction. Propagates
|
|
6405
|
+
* underlying read errors.
|
|
6406
|
+
*
|
|
6407
|
+
* @param options.prefix - Optional string filter applied as
|
|
6408
|
+
* `id.startsWith(prefix)`. O(n) over distinct ids because the
|
|
6409
|
+
* `(entity_id, source_ref)` index is not seekable on `entity_id`-only
|
|
6410
|
+
* prefix. Empty-string prefix matches every id.
|
|
6411
|
+
*
|
|
6412
|
+
* Note: this widens the coverage of `exportDump` (which delegates to
|
|
6413
|
+
* `MetadataRepository.getDistinctEntityIds`); exported dumps now include
|
|
6414
|
+
* orphaned entities. This is a deliberate widening for backup/migration
|
|
6415
|
+
* coverage, not a breaking change to `exportDump`'s return shape.
|
|
6416
|
+
*/
|
|
6417
|
+
async listEntityIds(options) {
|
|
6418
|
+
const ids = await this.metadataRepo.getDistinctEntityIds();
|
|
6419
|
+
if (options?.prefix === void 0) return ids;
|
|
6420
|
+
return ids.filter((id) => id.startsWith(options.prefix));
|
|
5913
6421
|
}
|
|
5914
6422
|
async runPrune(entityId, options) {
|
|
5915
6423
|
return this.maintenanceService.runPrune(entityId, options);
|
|
@@ -6007,6 +6515,57 @@ var WikiMemory = class {
|
|
|
6007
6515
|
async ingestDocument(entityId, params, opts) {
|
|
6008
6516
|
return this.ingestionService.ingestDocument(entityId, params, opts);
|
|
6009
6517
|
}
|
|
6518
|
+
/**
|
|
6519
|
+
* Deterministic write path into the graph. Accepts caller-supplied
|
|
6520
|
+
* `(nodes, edges)` and writes them under `(sourceRef, sourceHash)`
|
|
6521
|
+
* semantics identical to `ingestDocument`, minus the LLM extraction step.
|
|
6522
|
+
*
|
|
6523
|
+
* MUST be called from inside an open `db.withTransactionAsync` callback —
|
|
6524
|
+
* the third argument is the caller's `tx`. The method does not open a
|
|
6525
|
+
* nested transaction, does not acquire any lock, and does not perform
|
|
6526
|
+
* post-commit work (search sync, embedding, cache eviction). Hosts that
|
|
6527
|
+
* want embeddings synced should drive the existing maintenance sweep
|
|
6528
|
+
* (`runLibrarian` / `runHeal` / `runOntologyBackfill` / `runPrune`,
|
|
6529
|
+
* scoped via `listEntityIds`).
|
|
6530
|
+
*
|
|
6531
|
+
* Contract (full pre-flight validation lives inside
|
|
6532
|
+
* `IngestionService.upsertGraphCore`, which this method delegates to):
|
|
6533
|
+
* - C2: no-op when `(entityId, sourceHash)` is already mapped to
|
|
6534
|
+
* `params.sourceRef`; throws `WikiSourceRefHashCollision` if mapped to
|
|
6535
|
+
* a different `sourceRef`.
|
|
6536
|
+
* - C3: edges with dangling `targetId` are stored verbatim (no FK, no
|
|
6537
|
+
* resolution).
|
|
6538
|
+
* - C4: under persisted ontology mode `'strict'`, an out-of-manifest node
|
|
6539
|
+
* or edge `type` throws `WikiStrictOntologyViolation` (pre-flight,
|
|
6540
|
+
* all-or-nothing — NONE written on failure).
|
|
6541
|
+
*
|
|
6542
|
+
* @returns Counts: nodesWritten (validated nodes persisted), edgesWritten
|
|
6543
|
+
* (manifest-valid edges persisted), superseded (prior facts soft-deleted
|
|
6544
|
+
* plus prior source-ref edges hard-deleted).
|
|
6545
|
+
*/
|
|
6546
|
+
async upsertGraph(entityId, params, adapter) {
|
|
6547
|
+
const sourceRef = normalizeSourceRef(params.sourceRef);
|
|
6548
|
+
if (!sourceRef) throw new Error("Invalid sourceRef");
|
|
6549
|
+
const sourceHash = normalizeSourceHash(params.sourceHash);
|
|
6550
|
+
if (!sourceHash) throw new Error("Invalid sourceHash (must be 64-char hex string)");
|
|
6551
|
+
const canonical = await this.sourceRefIndexRepo.findActiveByEntityAndHash(entityId, sourceHash, adapter);
|
|
6552
|
+
if (canonical !== null && canonical === sourceRef) {
|
|
6553
|
+
return { nodesWritten: 0, edgesWritten: 0, superseded: 0 };
|
|
6554
|
+
}
|
|
6555
|
+
if (canonical !== null && canonical !== sourceRef) {
|
|
6556
|
+
throw new WikiSourceRefHashCollision({
|
|
6557
|
+
entityId,
|
|
6558
|
+
sourceHash,
|
|
6559
|
+
existingSourceRef: canonical,
|
|
6560
|
+
attemptedSourceRef: sourceRef
|
|
6561
|
+
});
|
|
6562
|
+
}
|
|
6563
|
+
return this.ingestionService.upsertGraphCore(
|
|
6564
|
+
entityId,
|
|
6565
|
+
{ sourceRef, sourceHash, nodes: params.nodes, edges: params.edges },
|
|
6566
|
+
adapter
|
|
6567
|
+
);
|
|
6568
|
+
}
|
|
6010
6569
|
/**
|
|
6011
6570
|
* Returns up to `limit` unprocessed outbox events, oldest first.
|
|
6012
6571
|
* Works regardless of enableOutbox value — allows draining after disabling.
|
|
@@ -6826,6 +7385,8 @@ exports.PrunePartialFailureError = PrunePartialFailureError;
|
|
|
6826
7385
|
exports.WikiBusyError = WikiBusyError;
|
|
6827
7386
|
exports.WikiDuplicateHashError = WikiDuplicateHashError;
|
|
6828
7387
|
exports.WikiMemory = WikiMemory;
|
|
7388
|
+
exports.WikiSourceRefHashCollision = WikiSourceRefHashCollision;
|
|
7389
|
+
exports.WikiStrictOntologyViolation = WikiStrictOntologyViolation;
|
|
6829
7390
|
exports.WikiTransactionError = WikiTransactionError;
|
|
6830
7391
|
exports.chunkText = chunkText;
|
|
6831
7392
|
exports.configureRandomSource = configureRandomSource;
|