@equationalapplications/core-llm-wiki 5.2.0 → 5.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -0
- package/dist/{chunk-J3N3WRK7.mjs → chunk-KIJINNYU.mjs} +367 -84
- package/dist/chunk-KIJINNYU.mjs.map +1 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +664 -103
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +300 -24
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-DyXISRwS.d.mts → testing-zf3Jp_RL.d.mts} +336 -29
- package/dist/{testing-DyXISRwS.d.ts → testing-zf3Jp_RL.d.ts} +336 -29
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +378 -74
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +2 -2
- package/dist/chunk-J3N3WRK7.mjs.map +0 -1
package/dist/index.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { __privateAdd, MetadataRepository, EmbeddingService, SearchService, JobManager, PromptService, IngestionService, MaintenanceService, ImportExportService, RetrievalService, WriteService, __privateGet, __privateSet, normalizeSourceRef, normalizeSourceHash, entitySummaryMetaKey, generateId, BaseRepository, emptyManifest, resolveNodeType, validateInlineEdges, resolveEdgeDefinitions, normalizeTitleKey, WikiTransactionError, extractSqliteCode } from './chunk-
|
|
2
|
-
export { DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, HEAL_BATCH_SIZE, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, WikiBusyError, WikiDuplicateHashError, WikiTransactionError, chunkText, configureRandomSource, parseEmbedding, safeSlice, validateManifest } from './chunk-
|
|
1
|
+
import { __privateAdd, MetadataRepository, EmbeddingService, SearchService, JobManager, PromptService, IngestionService, MaintenanceService, ImportExportService, RetrievalService, WriteService, __privateGet, __privateSet, normalizeSourceRef, normalizeSourceHash, entitySummaryMetaKey, WikiSourceRefHashCollision, generateId, BaseRepository, emptyManifest, resolveNodeType, WikiStrictOntologyViolation, validateInlineEdges, resolveEdgeDefinitions, normalizeTitleKey, WikiTransactionError, extractSqliteCode } from './chunk-KIJINNYU.mjs';
|
|
2
|
+
export { DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, HEAL_BATCH_SIZE, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, WikiBusyError, WikiDuplicateHashError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiTransactionError, chunkText, configureRandomSource, parseEmbedding, safeSlice, validateManifest } from './chunk-KIJINNYU.mjs';
|
|
3
3
|
import { appendRelatedSection, buildConceptDocument, buildLogMd, buildEntityIndexMd, buildRootIndexMd, isAllowedOkfPath, parseRootIndexMd, parseEntityIndexMd, parseConcept, splitRelatedSection, extractMarkdownLinks, parseLogMd, parseEventIdComment, appendEventIdComment } from '@equationalapplications/core-okf';
|
|
4
4
|
|
|
5
5
|
// src/db/schema.ts
|
|
@@ -32,6 +32,24 @@ async function setupDatabase(db, prefix) {
|
|
|
32
32
|
CREATE INDEX IF NOT EXISTS ${prefix}entries_source_hash_idx ON ${prefix}entries(entity_id, source_hash) WHERE source_hash IS NOT NULL;
|
|
33
33
|
CREATE INDEX IF NOT EXISTS ${prefix}entries_updated_idx ON ${prefix}entries(updated_at DESC);
|
|
34
34
|
|
|
35
|
+
-- source_ref_index: per-(entity, source_hash) record of the canonical sourceRef
|
|
36
|
+
-- currently holding that hash. The partial UNIQUE index on (entity_id, source_hash)
|
|
37
|
+
-- WHERE deleted_at IS NULL enforces the sourceRef-level TOCTOU-race invariant;
|
|
38
|
+
-- entries-level uniqueness cannot express it because a single ingestDocument call
|
|
39
|
+
-- writes N facts that all share (entity_id, source_ref, source_hash). See
|
|
40
|
+
-- docs/superpowers/specs/2026-08-07-dependabot-concurrency-release-hygiene-design.md \xA7B1.
|
|
41
|
+
CREATE TABLE IF NOT EXISTS ${prefix}source_ref_index (
|
|
42
|
+
id TEXT PRIMARY KEY,
|
|
43
|
+
entity_id TEXT NOT NULL,
|
|
44
|
+
source_hash TEXT NOT NULL,
|
|
45
|
+
source_ref TEXT NOT NULL,
|
|
46
|
+
created_at INTEGER NOT NULL,
|
|
47
|
+
deleted_at INTEGER
|
|
48
|
+
);
|
|
49
|
+
CREATE UNIQUE INDEX IF NOT EXISTS ${prefix}idx_source_ref_hash
|
|
50
|
+
ON ${prefix}source_ref_index (entity_id, source_hash)
|
|
51
|
+
WHERE deleted_at IS NULL;
|
|
52
|
+
|
|
35
53
|
CREATE TABLE IF NOT EXISTS ${prefix}tasks (
|
|
36
54
|
id TEXT PRIMARY KEY,
|
|
37
55
|
entity_id TEXT NOT NULL,
|
|
@@ -278,6 +296,61 @@ var MIGRATIONS = [
|
|
|
278
296
|
);
|
|
279
297
|
}
|
|
280
298
|
}
|
|
299
|
+
},
|
|
300
|
+
{
|
|
301
|
+
version: 9,
|
|
302
|
+
description: "add_source_ref_index",
|
|
303
|
+
run: async (db, prefix) => {
|
|
304
|
+
const duplicates = await db.getAllAsync(
|
|
305
|
+
`SELECT entity_id, source_hash, COUNT(DISTINCT source_ref) AS n_refs
|
|
306
|
+
FROM ${prefix}entries
|
|
307
|
+
WHERE deleted_at IS NULL AND source_hash IS NOT NULL
|
|
308
|
+
GROUP BY entity_id, source_hash
|
|
309
|
+
HAVING COUNT(DISTINCT source_ref) > 1`
|
|
310
|
+
);
|
|
311
|
+
if (duplicates.length > 0) {
|
|
312
|
+
const sample = duplicates.slice(0, 5).map((d) => `(entity_id=${d.entity_id}, source_hash=${d.source_hash.slice(0, 12)}\u2026, n_refs=${d.n_refs})`).join(", ");
|
|
313
|
+
throw new Error(
|
|
314
|
+
`Migration v9 (add_source_ref_index) failed: existing live rows have multiple sourceRefs sharing a hash. Found ${duplicates.length} duplicate (entity_id, source_hash) groups. First ${Math.min(5, duplicates.length)}: ${sample}. Resolve each by calling forget({ sourceRef: <loser> }) for the offending sourceRef, then re-run setup.`
|
|
315
|
+
);
|
|
316
|
+
}
|
|
317
|
+
await db.execAsync(
|
|
318
|
+
`CREATE TABLE IF NOT EXISTS ${prefix}source_ref_index (
|
|
319
|
+
id TEXT PRIMARY KEY,
|
|
320
|
+
entity_id TEXT NOT NULL,
|
|
321
|
+
source_hash TEXT NOT NULL,
|
|
322
|
+
source_ref TEXT NOT NULL,
|
|
323
|
+
created_at INTEGER NOT NULL,
|
|
324
|
+
deleted_at INTEGER
|
|
325
|
+
);
|
|
326
|
+
CREATE UNIQUE INDEX IF NOT EXISTS ${prefix}idx_source_ref_hash
|
|
327
|
+
ON ${prefix}source_ref_index (entity_id, source_hash)
|
|
328
|
+
WHERE deleted_at IS NULL;`
|
|
329
|
+
);
|
|
330
|
+
await db.execAsync(
|
|
331
|
+
`INSERT OR IGNORE INTO ${prefix}source_ref_index (id, entity_id, source_hash, source_ref, created_at, deleted_at)
|
|
332
|
+
SELECT
|
|
333
|
+
'sri:' || entity_id || ':' || source_hash,
|
|
334
|
+
entity_id,
|
|
335
|
+
source_hash,
|
|
336
|
+
source_ref,
|
|
337
|
+
updated_at,
|
|
338
|
+
NULL
|
|
339
|
+
FROM (
|
|
340
|
+
SELECT
|
|
341
|
+
entity_id, source_hash, source_ref, updated_at,
|
|
342
|
+
ROW_NUMBER() OVER (
|
|
343
|
+
PARTITION BY entity_id, source_hash
|
|
344
|
+
ORDER BY updated_at ASC, id ASC
|
|
345
|
+
) AS rn
|
|
346
|
+
FROM ${prefix}entries
|
|
347
|
+
WHERE deleted_at IS NULL
|
|
348
|
+
AND source_hash IS NOT NULL
|
|
349
|
+
AND source_ref IS NOT NULL
|
|
350
|
+
) ranked
|
|
351
|
+
WHERE rn = 1;`
|
|
352
|
+
);
|
|
353
|
+
}
|
|
281
354
|
}
|
|
282
355
|
];
|
|
283
356
|
for (let i = 1; i < MIGRATIONS.length; i++) {
|
|
@@ -663,12 +736,22 @@ var EntryRepository = class extends BaseRepository {
|
|
|
663
736
|
/**
|
|
664
737
|
* Fetch recent non-deleted entries for an entity (limited), ordered by updated_at DESC.
|
|
665
738
|
* Used by MaintenanceService.doRunLibrarian().
|
|
739
|
+
*
|
|
740
|
+
* When `excludeSourceRef` is supplied, the WHERE clause filters out that
|
|
741
|
+
* sourceRef BEFORE applying LIMIT. The caller in IngestionService is
|
|
742
|
+
* `findRecentByEntityId(entityId, 500, tx)` — it discards same-sourceRef
|
|
743
|
+
* rows after the read to build a cross-sourceRef title index. If those
|
|
744
|
+
* rows are filtered out in SQL, the limit applies to usable rows only
|
|
745
|
+
* and a re-ingest where the caller's sourceRef owns most recent facts
|
|
746
|
+
* still leaves cross-sourceRef entries in the index.
|
|
666
747
|
*/
|
|
667
|
-
async findRecentByEntityId(entityId, limit, tx) {
|
|
748
|
+
async findRecentByEntityId(entityId, limit, tx, excludeSourceRef) {
|
|
668
749
|
const executor = this.getExecutor(tx);
|
|
750
|
+
const excludeClause = excludeSourceRef ? ` AND (source_ref IS NULL OR source_ref != ?)` : "";
|
|
751
|
+
const args = excludeSourceRef ? [entityId, excludeSourceRef, limit] : [entityId, limit];
|
|
669
752
|
const rows = await executor.getAllAsync(
|
|
670
|
-
`SELECT * FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL ORDER BY updated_at DESC LIMIT ?`,
|
|
671
|
-
|
|
753
|
+
`SELECT * FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL${excludeClause} ORDER BY updated_at DESC LIMIT ?`,
|
|
754
|
+
args
|
|
672
755
|
);
|
|
673
756
|
return rows.map(mapRowToFact);
|
|
674
757
|
}
|
|
@@ -1394,6 +1477,57 @@ var OutboxRepository = class extends BaseRepository {
|
|
|
1394
1477
|
}
|
|
1395
1478
|
};
|
|
1396
1479
|
|
|
1480
|
+
// src/repositories/SourceRefIndexRepository.ts
|
|
1481
|
+
var SourceRefIndexRepository = class extends BaseRepository {
|
|
1482
|
+
/**
|
|
1483
|
+
* Idempotent insert: the partial UNIQUE index catches concurrent inserts for
|
|
1484
|
+
* the same (entity_id, source_hash). The caller (IngestionService) catches
|
|
1485
|
+
* the resulting SQLITE_CONSTRAINT_UNIQUE and translates it to the per-mode
|
|
1486
|
+
* duplicate-hash outcome. Runtime IDs use the `sri_` prefix to avoid
|
|
1487
|
+
* collision with the deterministic `sri:<entity>:<hash>` IDs used by the v9
|
|
1488
|
+
* backfill (the colon separator keeps the two ID spaces disjoint).
|
|
1489
|
+
*/
|
|
1490
|
+
async upsert(entityId, sourceHash, sourceRef, tx) {
|
|
1491
|
+
const executor = this.getExecutor(tx);
|
|
1492
|
+
await executor.runAsync(
|
|
1493
|
+
`INSERT INTO ${this.prefix}source_ref_index (id, entity_id, source_hash, source_ref, created_at, deleted_at)
|
|
1494
|
+
VALUES (?, ?, ?, ?, ?, NULL)`,
|
|
1495
|
+
[generateId("sri_"), entityId, sourceHash, sourceRef, Date.now()]
|
|
1496
|
+
);
|
|
1497
|
+
}
|
|
1498
|
+
/**
|
|
1499
|
+
* Idempotent soft-delete of the live row for (entity_id, source_ref).
|
|
1500
|
+
* Called at the start of every ingestDocument to remove the prior run's
|
|
1501
|
+
* index row, so the new upsert doesn't collide with itself. No-op when the
|
|
1502
|
+
* row is already soft-deleted or never existed.
|
|
1503
|
+
*/
|
|
1504
|
+
async softDeleteByEntityAndSourceRef(entityId, sourceRef, tx) {
|
|
1505
|
+
const executor = this.getExecutor(tx);
|
|
1506
|
+
const now = Date.now();
|
|
1507
|
+
const result = await executor.runAsync(
|
|
1508
|
+
`UPDATE ${this.prefix}source_ref_index
|
|
1509
|
+
SET deleted_at = ?, created_at = ?
|
|
1510
|
+
WHERE entity_id = ? AND source_ref = ? AND deleted_at IS NULL`,
|
|
1511
|
+
[now, now, entityId, sourceRef]
|
|
1512
|
+
);
|
|
1513
|
+
return result.changes;
|
|
1514
|
+
}
|
|
1515
|
+
/**
|
|
1516
|
+
* Returns the live sourceRef holding the given hash, or null when no live
|
|
1517
|
+
* row exists. Used by the IngestionService pre-check (line 82) and the
|
|
1518
|
+
* catch-and-translate canonical lookup (line 212).
|
|
1519
|
+
*/
|
|
1520
|
+
async findActiveByEntityAndHash(entityId, sourceHash, tx) {
|
|
1521
|
+
const executor = this.getExecutor(tx);
|
|
1522
|
+
const row = await executor.getFirstAsync(
|
|
1523
|
+
`SELECT source_ref FROM ${this.prefix}source_ref_index
|
|
1524
|
+
WHERE entity_id = ? AND source_hash = ? AND deleted_at IS NULL`,
|
|
1525
|
+
[entityId, sourceHash]
|
|
1526
|
+
);
|
|
1527
|
+
return row?.source_ref ?? null;
|
|
1528
|
+
}
|
|
1529
|
+
};
|
|
1530
|
+
|
|
1397
1531
|
// src/repositories/TaskRepository.ts
|
|
1398
1532
|
function mapRowToTask(row) {
|
|
1399
1533
|
return {
|
|
@@ -1808,6 +1942,34 @@ var EdgeRepository = class extends BaseRepository {
|
|
|
1808
1942
|
const executor = this.getExecutor(tx);
|
|
1809
1943
|
await executor.runAsync(`DELETE FROM ${this.prefix}edges WHERE entity_id = ?`, [entityId]);
|
|
1810
1944
|
}
|
|
1945
|
+
/**
|
|
1946
|
+
* Hard-delete every edge whose `source_id` is in `sourceFactIds`, scoped to
|
|
1947
|
+
* `entityId`. Returns the number of rows deleted. Edges have no soft-delete
|
|
1948
|
+
* concept; this method is named `softDeleteBySourceFactIds` for symmetry with
|
|
1949
|
+
* `EntryRepository.softDeleteBySource` (which IS a soft delete), but its
|
|
1950
|
+
* semantics are immediate presence/absence removal. Use ONLY inside an open
|
|
1951
|
+
* transaction — the method requires `tx` and performs no autocommit.
|
|
1952
|
+
*
|
|
1953
|
+
* Used by `IngestionService.upsertGraphCore` to retire the prior parse's
|
|
1954
|
+
* edges whose source facts are being superseded by `softDeleteBySource`.
|
|
1955
|
+
*/
|
|
1956
|
+
async softDeleteBySourceFactIds(entityId, sourceFactIds, tx) {
|
|
1957
|
+
if (sourceFactIds.length === 0) return 0;
|
|
1958
|
+
const executor = this.getExecutor(tx);
|
|
1959
|
+
const chunkSize = 500;
|
|
1960
|
+
let totalChanges = 0;
|
|
1961
|
+
for (let i = 0; i < sourceFactIds.length; i += chunkSize) {
|
|
1962
|
+
const chunk = sourceFactIds.slice(i, i + chunkSize);
|
|
1963
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
1964
|
+
const result = await executor.runAsync(
|
|
1965
|
+
`DELETE FROM ${this.prefix}edges
|
|
1966
|
+
WHERE entity_id = ? AND source_id IN (${placeholders})`,
|
|
1967
|
+
[entityId, ...chunk]
|
|
1968
|
+
);
|
|
1969
|
+
totalChanges += result.changes;
|
|
1970
|
+
}
|
|
1971
|
+
return totalChanges;
|
|
1972
|
+
}
|
|
1811
1973
|
/**
|
|
1812
1974
|
* Multi-hop traversal from `sourceId` via SQLite `WITH RECURSIVE`. All filtering,
|
|
1813
1975
|
* dead-ending, cycle-guarding, capping, and ordering happens in this one query.
|
|
@@ -1986,16 +2148,27 @@ var OntologyService = class {
|
|
|
1986
2148
|
this.invalidateCache(entityId);
|
|
1987
2149
|
return merged;
|
|
1988
2150
|
}
|
|
1989
|
-
validateAndNormalizeFact(fact, manifest) {
|
|
2151
|
+
validateAndNormalizeFact(fact, manifest, opts) {
|
|
1990
2152
|
const rawType = typeof fact.okf_type === "string" ? fact.okf_type : "";
|
|
2153
|
+
const strict = opts?.strict === true;
|
|
1991
2154
|
const canonical = resolveNodeType(rawType, manifest);
|
|
1992
|
-
if (!canonical)
|
|
1993
|
-
|
|
2155
|
+
if (!canonical) {
|
|
2156
|
+
if (strict) throw new WikiStrictOntologyViolation(opts?.entityId ?? "", "node", rawType);
|
|
2157
|
+
return { okf_type: null, edges: [] };
|
|
2158
|
+
}
|
|
2159
|
+
const edges = validateInlineEdges(canonical, null, fact.edges ?? [], manifest, opts);
|
|
1994
2160
|
return { okf_type: canonical, edges };
|
|
1995
2161
|
}
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
|
|
2162
|
+
/**
|
|
2163
|
+
* Pure resolver: given a source type, LLM-supplied edges, the manifest,
|
|
2164
|
+
* and a title index, return a list of concrete `WikiEdge` objects ready
|
|
2165
|
+
* for persistence. Performs no DB writes. Used by callers that batch edge
|
|
2166
|
+
* writes inside their own transaction (e.g. `IngestionService.ingestDocument`
|
|
2167
|
+
* when delegating to `upsertGraphCore`).
|
|
2168
|
+
*/
|
|
2169
|
+
resolveEdges(entityId, sourceId, sourceType, edges, manifest, titleIndex, now) {
|
|
2170
|
+
if (!sourceType || edges.length === 0) return [];
|
|
2171
|
+
const out = [];
|
|
1999
2172
|
for (const edge of edges) {
|
|
2000
2173
|
const candidates = resolveEdgeDefinitions(edge.edge_type, manifest).filter((d) => d.source_type.toLowerCase() === sourceType.toLowerCase());
|
|
2001
2174
|
if (candidates.length === 0) continue;
|
|
@@ -2006,15 +2179,29 @@ var OntologyService = class {
|
|
|
2006
2179
|
(d) => d.target_type.toLowerCase() === (target.okf_type ?? "").toLowerCase()
|
|
2007
2180
|
);
|
|
2008
2181
|
if (!def) continue;
|
|
2009
|
-
|
|
2182
|
+
out.push({
|
|
2010
2183
|
id: generateId(),
|
|
2011
2184
|
entity_id: entityId,
|
|
2012
2185
|
source_id: sourceId,
|
|
2013
2186
|
target_id: target.id,
|
|
2014
2187
|
edge_type: def.type,
|
|
2015
2188
|
created_at: now
|
|
2016
|
-
};
|
|
2017
|
-
|
|
2189
|
+
});
|
|
2190
|
+
}
|
|
2191
|
+
return out;
|
|
2192
|
+
}
|
|
2193
|
+
/**
|
|
2194
|
+
* Backwards-compatible wrapper: resolves edges via {@link resolveEdges},
|
|
2195
|
+
* then persists each via `edgeRepo.addIgnoreDuplicate` in the supplied
|
|
2196
|
+
* transaction. Returns the number of edges persisted. Used by paths that
|
|
2197
|
+
* want self-contained edge persistence (e.g. MaintenanceService heal /
|
|
2198
|
+
* backfill callers that do not have a separate edge-write step).
|
|
2199
|
+
*/
|
|
2200
|
+
async resolveAndPersistEdges(entityId, sourceId, sourceType, edges, manifest, titleIndex, tx, now) {
|
|
2201
|
+
const resolved = this.resolveEdges(entityId, sourceId, sourceType, edges, manifest, titleIndex, now);
|
|
2202
|
+
let persisted = 0;
|
|
2203
|
+
for (const edge of resolved) {
|
|
2204
|
+
const inserted = await this.edgeRepo.addIgnoreDuplicate(edge, tx);
|
|
2018
2205
|
if (inserted) persisted++;
|
|
2019
2206
|
}
|
|
2020
2207
|
return persisted;
|
|
@@ -2070,6 +2257,7 @@ var WikiMemory = class {
|
|
|
2070
2257
|
}
|
|
2071
2258
|
this.outboxRepo = new OutboxRepository(this.db, this.prefix, !!options.config?.enableOutbox);
|
|
2072
2259
|
this.entryRepo = new EntryRepository(this.db, this.prefix, this.outboxRepo);
|
|
2260
|
+
this.sourceRefIndexRepo = new SourceRefIndexRepository(this.db, this.prefix);
|
|
2073
2261
|
this.taskRepo = new TaskRepository(this.db, this.prefix, this.outboxRepo);
|
|
2074
2262
|
this.eventRepo = new EventRepository(this.db, this.prefix);
|
|
2075
2263
|
this.edgeRepo = new EdgeRepository(this.db, this.prefix);
|
|
@@ -2088,6 +2276,9 @@ var WikiMemory = class {
|
|
|
2088
2276
|
this.prefix,
|
|
2089
2277
|
this.options,
|
|
2090
2278
|
this.entryRepo,
|
|
2279
|
+
this.sourceRefIndexRepo,
|
|
2280
|
+
this.metadataRepo,
|
|
2281
|
+
this.edgeRepo,
|
|
2091
2282
|
this.searchService,
|
|
2092
2283
|
this.jobManager,
|
|
2093
2284
|
this.embeddingService,
|
|
@@ -2099,6 +2290,7 @@ var WikiMemory = class {
|
|
|
2099
2290
|
this.prefix,
|
|
2100
2291
|
this.options,
|
|
2101
2292
|
this.entryRepo,
|
|
2293
|
+
this.sourceRefIndexRepo,
|
|
2102
2294
|
this.taskRepo,
|
|
2103
2295
|
this.eventRepo,
|
|
2104
2296
|
this.metadataRepo,
|
|
@@ -2163,6 +2355,7 @@ var WikiMemory = class {
|
|
|
2163
2355
|
promptService: this.promptService,
|
|
2164
2356
|
graphTraversalService: this.graphTraversalService,
|
|
2165
2357
|
entryRepo: this.entryRepo,
|
|
2358
|
+
sourceRefIndexRepo: this.sourceRefIndexRepo,
|
|
2166
2359
|
metadataRepo: this.metadataRepo,
|
|
2167
2360
|
jobManager: this.jobManager
|
|
2168
2361
|
};
|
|
@@ -2239,7 +2432,7 @@ var WikiMemory = class {
|
|
|
2239
2432
|
const latestHashes = await this.entryRepo.findLatestSourceHashes(entityId, normalized.map((e) => e.sourceRef));
|
|
2240
2433
|
const distinctHashes = Array.from(new Set(normalized.map((e) => e.sourceHash)));
|
|
2241
2434
|
const dupRefs = await Promise.all(
|
|
2242
|
-
distinctHashes.map((h) => this.
|
|
2435
|
+
distinctHashes.map((h) => this.sourceRefIndexRepo.findActiveByEntityAndHash(entityId, h))
|
|
2243
2436
|
);
|
|
2244
2437
|
const dupMap = /* @__PURE__ */ new Map();
|
|
2245
2438
|
for (let i = 0; i < distinctHashes.length; i++) {
|
|
@@ -2248,12 +2441,10 @@ var WikiMemory = class {
|
|
|
2248
2441
|
return normalized.map((e) => {
|
|
2249
2442
|
const stored = latestHashes.get(e.sourceRef);
|
|
2250
2443
|
const changed = stored === void 0 || stored === null || normalizeSourceHash(stored) !== e.sourceHash;
|
|
2251
|
-
const
|
|
2252
|
-
|
|
2253
|
-
if (others.length === 0) {
|
|
2444
|
+
const canonical = dupMap.get(e.sourceHash) ?? null;
|
|
2445
|
+
if (canonical === null || canonical === e.sourceRef) {
|
|
2254
2446
|
return { sourceRef: e.rawSourceRef, changed };
|
|
2255
2447
|
}
|
|
2256
|
-
const canonical = others[0];
|
|
2257
2448
|
return { sourceRef: e.rawSourceRef, changed, duplicateOf: canonical };
|
|
2258
2449
|
});
|
|
2259
2450
|
}
|
|
@@ -2268,13 +2459,47 @@ var WikiMemory = class {
|
|
|
2268
2459
|
return this.entryRepo.listSourceRefs(entityId);
|
|
2269
2460
|
}
|
|
2270
2461
|
/**
|
|
2271
|
-
* Returns the live source_refs for an entity that hold the given source_hash
|
|
2272
|
-
*
|
|
2273
|
-
*
|
|
2274
|
-
*
|
|
2462
|
+
* Returns the live source_refs for an entity that hold the given source_hash.
|
|
2463
|
+
* With v9, source_ref_index is the source of truth for the sourceRef-level
|
|
2464
|
+
* TOCTOU-race invariant: at most one sourceRef can hold a given
|
|
2465
|
+
* (entity_id, source_hash). The result is either a single-element array
|
|
2466
|
+
* (one canonical ref) or empty (no live ref holds the hash). Returned as
|
|
2467
|
+
* an array to preserve the existing public-API shape used by hosts
|
|
2468
|
+
* auditing duplicate-content collisions.
|
|
2275
2469
|
*/
|
|
2276
2470
|
async findSourceRefsByHash(entityId, sourceHash) {
|
|
2277
|
-
|
|
2471
|
+
const canonical = await this.sourceRefIndexRepo.findActiveByEntityAndHash(entityId, sourceHash);
|
|
2472
|
+
return canonical === null ? [] : [canonical];
|
|
2473
|
+
}
|
|
2474
|
+
/**
|
|
2475
|
+
* Returns the set of `entity_id` values with at least one row in this
|
|
2476
|
+
* database, INCLUDING entities whose only remaining rows are soft-deleted
|
|
2477
|
+
* — sorted ascending `entity_id COLLATE BINARY`. Empty array when the
|
|
2478
|
+
* database has no entities.
|
|
2479
|
+
*
|
|
2480
|
+
* The deliberate inclusion of soft-deleted-only entities closes the
|
|
2481
|
+
* decommissioned-scope leak: a scoped namespace whose documents have all
|
|
2482
|
+
* been forgotten or superseded must still appear so host maintenance
|
|
2483
|
+
* sweeps (`runLibrarian`, `runHeal`, `runOntologyBackfill`, `runPrune`)
|
|
2484
|
+
* can visit it and reap the soft-deleted rows.
|
|
2485
|
+
*
|
|
2486
|
+
* Read-only. Never acquires a lock. Never opens a transaction. Propagates
|
|
2487
|
+
* underlying read errors.
|
|
2488
|
+
*
|
|
2489
|
+
* @param options.prefix - Optional string filter applied as
|
|
2490
|
+
* `id.startsWith(prefix)`. O(n) over distinct ids because the
|
|
2491
|
+
* `(entity_id, source_ref)` index is not seekable on `entity_id`-only
|
|
2492
|
+
* prefix. Empty-string prefix matches every id.
|
|
2493
|
+
*
|
|
2494
|
+
* Note: this widens the coverage of `exportDump` (which delegates to
|
|
2495
|
+
* `MetadataRepository.getDistinctEntityIds`); exported dumps now include
|
|
2496
|
+
* orphaned entities. This is a deliberate widening for backup/migration
|
|
2497
|
+
* coverage, not a breaking change to `exportDump`'s return shape.
|
|
2498
|
+
*/
|
|
2499
|
+
async listEntityIds(options) {
|
|
2500
|
+
const ids = await this.metadataRepo.getDistinctEntityIds();
|
|
2501
|
+
if (options?.prefix === void 0) return ids;
|
|
2502
|
+
return ids.filter((id) => id.startsWith(options.prefix));
|
|
2278
2503
|
}
|
|
2279
2504
|
async runPrune(entityId, options) {
|
|
2280
2505
|
return this.maintenanceService.runPrune(entityId, options);
|
|
@@ -2372,6 +2597,57 @@ var WikiMemory = class {
|
|
|
2372
2597
|
async ingestDocument(entityId, params, opts) {
|
|
2373
2598
|
return this.ingestionService.ingestDocument(entityId, params, opts);
|
|
2374
2599
|
}
|
|
2600
|
+
/**
|
|
2601
|
+
* Deterministic write path into the graph. Accepts caller-supplied
|
|
2602
|
+
* `(nodes, edges)` and writes them under `(sourceRef, sourceHash)`
|
|
2603
|
+
* semantics identical to `ingestDocument`, minus the LLM extraction step.
|
|
2604
|
+
*
|
|
2605
|
+
* MUST be called from inside an open `db.withTransactionAsync` callback —
|
|
2606
|
+
* the third argument is the caller's `tx`. The method does not open a
|
|
2607
|
+
* nested transaction, does not acquire any lock, and does not perform
|
|
2608
|
+
* post-commit work (search sync, embedding, cache eviction). Hosts that
|
|
2609
|
+
* want embeddings synced should drive the existing maintenance sweep
|
|
2610
|
+
* (`runLibrarian` / `runHeal` / `runOntologyBackfill` / `runPrune`,
|
|
2611
|
+
* scoped via `listEntityIds`).
|
|
2612
|
+
*
|
|
2613
|
+
* Contract (full pre-flight validation lives inside
|
|
2614
|
+
* `IngestionService.upsertGraphCore`, which this method delegates to):
|
|
2615
|
+
* - C2: no-op when `(entityId, sourceHash)` is already mapped to
|
|
2616
|
+
* `params.sourceRef`; throws `WikiSourceRefHashCollision` if mapped to
|
|
2617
|
+
* a different `sourceRef`.
|
|
2618
|
+
* - C3: edges with dangling `targetId` are stored verbatim (no FK, no
|
|
2619
|
+
* resolution).
|
|
2620
|
+
* - C4: under persisted ontology mode `'strict'`, an out-of-manifest node
|
|
2621
|
+
* or edge `type` throws `WikiStrictOntologyViolation` (pre-flight,
|
|
2622
|
+
* all-or-nothing — NONE written on failure).
|
|
2623
|
+
*
|
|
2624
|
+
* @returns Counts: nodesWritten (validated nodes persisted), edgesWritten
|
|
2625
|
+
* (manifest-valid edges persisted), superseded (prior facts soft-deleted
|
|
2626
|
+
* plus prior source-ref edges hard-deleted).
|
|
2627
|
+
*/
|
|
2628
|
+
async upsertGraph(entityId, params, adapter) {
|
|
2629
|
+
const sourceRef = normalizeSourceRef(params.sourceRef);
|
|
2630
|
+
if (!sourceRef) throw new Error("Invalid sourceRef");
|
|
2631
|
+
const sourceHash = normalizeSourceHash(params.sourceHash);
|
|
2632
|
+
if (!sourceHash) throw new Error("Invalid sourceHash (must be 64-char hex string)");
|
|
2633
|
+
const canonical = await this.sourceRefIndexRepo.findActiveByEntityAndHash(entityId, sourceHash, adapter);
|
|
2634
|
+
if (canonical !== null && canonical === sourceRef) {
|
|
2635
|
+
return { nodesWritten: 0, edgesWritten: 0, superseded: 0 };
|
|
2636
|
+
}
|
|
2637
|
+
if (canonical !== null && canonical !== sourceRef) {
|
|
2638
|
+
throw new WikiSourceRefHashCollision({
|
|
2639
|
+
entityId,
|
|
2640
|
+
sourceHash,
|
|
2641
|
+
existingSourceRef: canonical,
|
|
2642
|
+
attemptedSourceRef: sourceRef
|
|
2643
|
+
});
|
|
2644
|
+
}
|
|
2645
|
+
return this.ingestionService.upsertGraphCore(
|
|
2646
|
+
entityId,
|
|
2647
|
+
{ sourceRef, sourceHash, nodes: params.nodes, edges: params.edges },
|
|
2648
|
+
adapter
|
|
2649
|
+
);
|
|
2650
|
+
}
|
|
2375
2651
|
/**
|
|
2376
2652
|
* Returns up to `limit` unprocessed outbox events, oldest first.
|
|
2377
2653
|
* Works regardless of enableOutbox value — allows draining after disabling.
|