@equationalapplications/core-llm-wiki 7.12.0 → 7.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-KN4XSLLB.mjs → chunk-PCRYIUV5.mjs} +232 -18
- package/dist/chunk-PCRYIUV5.mjs.map +1 -0
- package/dist/index.d.mts +10 -3
- package/dist/index.d.ts +10 -3
- package/dist/index.js +730 -146
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +502 -133
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-7d76Ptlq.d.mts → testing-CqsXv2_l.d.mts} +111 -18
- package/dist/{testing-7d76Ptlq.d.ts → testing-CqsXv2_l.d.ts} +111 -18
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +80 -15
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +3 -3
- package/dist/chunk-KN4XSLLB.mjs.map +0 -1
package/dist/index.js
CHANGED
|
@@ -75,6 +75,8 @@ var SQLITE_RESULT_CODE_NAMES = {
|
|
|
75
75
|
// prefix and we normalize it back to the symbolic name so callers can branch on
|
|
76
76
|
// the same string the rest of the ecosystem uses.
|
|
77
77
|
2067: "SQLITE_CONSTRAINT_UNIQUE",
|
|
78
|
+
// 267 = SQLITE_CORRUPT_VTAB — a corrupt FTS5 shadow table (ftsCorruptionClass).
|
|
79
|
+
267: "SQLITE_CORRUPT_VTAB",
|
|
78
80
|
100: "SQLITE_ROW",
|
|
79
81
|
101: "SQLITE_DONE"
|
|
80
82
|
};
|
|
@@ -935,6 +937,143 @@ async function createTemporalIndexesIfColumnsExist(db, prefix) {
|
|
|
935
937
|
}
|
|
936
938
|
}
|
|
937
939
|
|
|
940
|
+
// src/services/search/fts5Sql.ts
|
|
941
|
+
var FTS5_STATE_KEY = "fts5_index_state";
|
|
942
|
+
var FTS5_DRAIN_CHUNK = 500;
|
|
943
|
+
var FTS5_SWEEP_CHUNK = 500;
|
|
944
|
+
var FTS5_RETIRED_TABLES_KEY = "fts5_retired_tables";
|
|
945
|
+
var FTS5_RETIRED_COUNTER_KEY = "fts5_retired_counter";
|
|
946
|
+
var FTS5_RECLAIM_MERGE_PAGES = 500;
|
|
947
|
+
function retiredTableName(p, n) {
|
|
948
|
+
return `${p}entries_fts_retired_${n}`;
|
|
949
|
+
}
|
|
950
|
+
function isRetiredTableName(p, name) {
|
|
951
|
+
return typeof name === "string" && name.startsWith(`${p}entries_fts_retired_`) && /^\d+$/.test(name.slice(`${p}entries_fts_retired_`.length));
|
|
952
|
+
}
|
|
953
|
+
var FTS5_REBUILD_WM_KEY = "fts5_rebuild_wm";
|
|
954
|
+
var FTS5_REBUILD_SNAPSHOT_KEY = "fts5_rebuild_snapshot";
|
|
955
|
+
var FTS5_SWEEP_CURSOR_KEY = "fts5_orphan_sweep_cursor";
|
|
956
|
+
function fts5TriggerNames(prefix) {
|
|
957
|
+
return [`${prefix}entries_fts_ai`, `${prefix}entries_fts_au`, `${prefix}entries_fts_ad`];
|
|
958
|
+
}
|
|
959
|
+
function tagsExpr(col) {
|
|
960
|
+
return `CASE WHEN json_valid(${col}) THEN
|
|
961
|
+
CASE WHEN json_type(${col}) = 'array' THEN
|
|
962
|
+
(SELECT group_concat(CASE type WHEN 'true' THEN 'true' WHEN 'false' THEN 'false'
|
|
963
|
+
WHEN 'null' THEN '' ELSE value END, ' ')
|
|
964
|
+
FROM json_each(${col}))
|
|
965
|
+
ELSE ${col} END
|
|
966
|
+
ELSE ${col} END`;
|
|
967
|
+
}
|
|
968
|
+
function fts5TablesDdl(p) {
|
|
969
|
+
return `
|
|
970
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS ${p}entries_fts USING fts5(
|
|
971
|
+
id UNINDEXED, entity_id UNINDEXED, title, body, tags,
|
|
972
|
+
tokenize = 'porter unicode61'
|
|
973
|
+
);
|
|
974
|
+
CREATE TABLE IF NOT EXISTS ${p}fts_map (
|
|
975
|
+
fts_rowid INTEGER PRIMARY KEY,
|
|
976
|
+
id TEXT NOT NULL UNIQUE,
|
|
977
|
+
entity_id TEXT NOT NULL
|
|
978
|
+
);
|
|
979
|
+
CREATE INDEX IF NOT EXISTS ${p}fts_map_entity ON ${p}fts_map(entity_id);
|
|
980
|
+
CREATE TABLE IF NOT EXISTS ${p}fts_pending (
|
|
981
|
+
seq INTEGER PRIMARY KEY,
|
|
982
|
+
entity_id TEXT NOT NULL,
|
|
983
|
+
id TEXT NOT NULL
|
|
984
|
+
);`;
|
|
985
|
+
}
|
|
986
|
+
function fts5TriggersDdl(p) {
|
|
987
|
+
const [ai, au, ad] = fts5TriggerNames(p);
|
|
988
|
+
return `
|
|
989
|
+
CREATE TRIGGER IF NOT EXISTS ${ai} AFTER INSERT ON ${p}entries BEGIN
|
|
990
|
+
INSERT INTO ${p}fts_pending (entity_id, id) VALUES (NEW.entity_id, NEW.id);
|
|
991
|
+
END;
|
|
992
|
+
CREATE TRIGGER IF NOT EXISTS ${au}
|
|
993
|
+
AFTER UPDATE OF id, entity_id, title, body, tags, deleted_at ON ${p}entries BEGIN
|
|
994
|
+
INSERT INTO ${p}fts_pending (entity_id, id) VALUES (OLD.entity_id, OLD.id);
|
|
995
|
+
INSERT INTO ${p}fts_pending (entity_id, id)
|
|
996
|
+
SELECT NEW.entity_id, NEW.id WHERE NEW.id <> OLD.id OR NEW.entity_id <> OLD.entity_id;
|
|
997
|
+
END;
|
|
998
|
+
CREATE TRIGGER IF NOT EXISTS ${ad} AFTER DELETE ON ${p}entries BEGIN
|
|
999
|
+
INSERT INTO ${p}fts_pending (entity_id, id) VALUES (OLD.entity_id, OLD.id);
|
|
1000
|
+
END;`;
|
|
1001
|
+
}
|
|
1002
|
+
function drainHiSql(p) {
|
|
1003
|
+
return `SELECT max(seq) AS hi FROM (SELECT seq FROM ${p}fts_pending ORDER BY seq LIMIT ${FTS5_DRAIN_CHUNK})`;
|
|
1004
|
+
}
|
|
1005
|
+
function ledgerChunkSql(p, chunkIds, ledgerWhere) {
|
|
1006
|
+
return [
|
|
1007
|
+
`DELETE FROM ${p}entries_fts WHERE rowid IN (SELECT m.fts_rowid FROM ${p}fts_map m WHERE m.id IN (${chunkIds}))`,
|
|
1008
|
+
`DELETE FROM ${p}fts_map WHERE id IN (${chunkIds})`,
|
|
1009
|
+
`INSERT INTO ${p}fts_map (id, entity_id)
|
|
1010
|
+
SELECT e.id, e.entity_id FROM ${p}entries e WHERE e.id IN (${chunkIds}) AND e.deleted_at IS NULL`,
|
|
1011
|
+
`INSERT INTO ${p}entries_fts (rowid, id, entity_id, title, body, tags)
|
|
1012
|
+
SELECT m.fts_rowid, e.id, e.entity_id, e.title, e.body, ${tagsExpr("e.tags")}
|
|
1013
|
+
FROM ${p}fts_map m JOIN ${p}entries e ON e.id = m.id
|
|
1014
|
+
WHERE m.id IN (${chunkIds})`,
|
|
1015
|
+
`DELETE FROM ${p}fts_pending WHERE ${ledgerWhere}`
|
|
1016
|
+
];
|
|
1017
|
+
}
|
|
1018
|
+
function drainChunkSql(p) {
|
|
1019
|
+
return ledgerChunkSql(p, `SELECT id FROM ${p}fts_pending WHERE seq <= ?`, "seq <= ?");
|
|
1020
|
+
}
|
|
1021
|
+
function fts5PendingIndexesDdl(p) {
|
|
1022
|
+
return `
|
|
1023
|
+
CREATE INDEX IF NOT EXISTS ${p}fts_pending_id ON ${p}fts_pending(id);
|
|
1024
|
+
CREATE INDEX IF NOT EXISTS ${p}fts_pending_entity ON ${p}fts_pending(entity_id);`;
|
|
1025
|
+
}
|
|
1026
|
+
function idDrainChunkSql(p) {
|
|
1027
|
+
return ledgerChunkSql(
|
|
1028
|
+
p,
|
|
1029
|
+
`SELECT id FROM ${p}fts_pending WHERE id IN (SELECT value FROM json_each(?))`,
|
|
1030
|
+
"id IN (SELECT value FROM json_each(?))"
|
|
1031
|
+
);
|
|
1032
|
+
}
|
|
1033
|
+
function idReindexChunkSql(p) {
|
|
1034
|
+
return ledgerChunkSql(p, "SELECT value FROM json_each(?)", "id IN (SELECT value FROM json_each(?))");
|
|
1035
|
+
}
|
|
1036
|
+
function entityDrainHiSql(p) {
|
|
1037
|
+
return `SELECT max(seq) AS hi FROM (SELECT seq FROM ${p}fts_pending WHERE entity_id = ? ORDER BY seq LIMIT ${FTS5_DRAIN_CHUNK})`;
|
|
1038
|
+
}
|
|
1039
|
+
function entityDrainChunkSql(p) {
|
|
1040
|
+
return ledgerChunkSql(p, `SELECT id FROM ${p}fts_pending WHERE entity_id = ? AND seq <= ?`, "entity_id = ? AND seq <= ?");
|
|
1041
|
+
}
|
|
1042
|
+
function fts5LedgerDropSql(p) {
|
|
1043
|
+
return [
|
|
1044
|
+
...fts5TriggerNames(p).map((n) => `DROP TRIGGER IF EXISTS ${n};`),
|
|
1045
|
+
`DROP TABLE IF EXISTS ${p}fts_pending;`,
|
|
1046
|
+
`DROP TABLE IF EXISTS ${p}fts_map;`
|
|
1047
|
+
].join("\n");
|
|
1048
|
+
}
|
|
1049
|
+
function windowHiSql(p) {
|
|
1050
|
+
return `SELECT max(rowid) AS hi FROM (SELECT rowid FROM ${p}entries WHERE rowid > ? AND rowid <= ? ORDER BY rowid LIMIT ${FTS5_DRAIN_CHUNK})`;
|
|
1051
|
+
}
|
|
1052
|
+
function windowChunkSql(p) {
|
|
1053
|
+
const chunkIds = `SELECT id FROM ${p}entries WHERE rowid > ? AND rowid <= ?`;
|
|
1054
|
+
return [
|
|
1055
|
+
`DELETE FROM ${p}entries_fts WHERE rowid IN (SELECT m.fts_rowid FROM ${p}fts_map m WHERE m.id IN (${chunkIds}))`,
|
|
1056
|
+
`DELETE FROM ${p}fts_map WHERE id IN (${chunkIds})`,
|
|
1057
|
+
`INSERT INTO ${p}fts_map (id, entity_id)
|
|
1058
|
+
SELECT e.id, e.entity_id FROM ${p}entries e WHERE e.rowid > ? AND e.rowid <= ? AND e.deleted_at IS NULL`,
|
|
1059
|
+
`INSERT INTO ${p}entries_fts (rowid, id, entity_id, title, body, tags)
|
|
1060
|
+
SELECT m.fts_rowid, e.id, e.entity_id, e.title, e.body, ${tagsExpr("e.tags")}
|
|
1061
|
+
FROM ${p}entries e JOIN ${p}fts_map m ON m.id = e.id
|
|
1062
|
+
WHERE e.rowid > ? AND e.rowid <= ?`
|
|
1063
|
+
];
|
|
1064
|
+
}
|
|
1065
|
+
function sweepHiSql(p) {
|
|
1066
|
+
return `SELECT max(fts_rowid) AS hi FROM (SELECT fts_rowid FROM ${p}fts_map WHERE fts_rowid > ? AND fts_rowid <= ? ORDER BY fts_rowid LIMIT ${FTS5_SWEEP_CHUNK})`;
|
|
1067
|
+
}
|
|
1068
|
+
function sweepChunkSql(p) {
|
|
1069
|
+
const orphans = `SELECT m.fts_rowid FROM ${p}fts_map m
|
|
1070
|
+
WHERE m.fts_rowid > ? AND m.fts_rowid <= ? AND NOT EXISTS (SELECT 1 FROM ${p}entries e WHERE e.id = m.id)`;
|
|
1071
|
+
return [
|
|
1072
|
+
`DELETE FROM ${p}entries_fts WHERE rowid IN (${orphans})`,
|
|
1073
|
+
`DELETE FROM ${p}fts_map WHERE fts_rowid IN (${orphans})`
|
|
1074
|
+
];
|
|
1075
|
+
}
|
|
1076
|
+
|
|
938
1077
|
// src/db/migrations.ts
|
|
939
1078
|
var MIGRATIONS = [
|
|
940
1079
|
{
|
|
@@ -1215,6 +1354,17 @@ var MIGRATIONS = [
|
|
|
1215
1354
|
run: async (db, prefix) => {
|
|
1216
1355
|
await createLegacySourceTypeIndex(db, prefix);
|
|
1217
1356
|
}
|
|
1357
|
+
},
|
|
1358
|
+
{
|
|
1359
|
+
version: 15,
|
|
1360
|
+
description: "fts_pending(id) and fts_pending(entity_id) indexes for the ids-first and entity-targeted drains",
|
|
1361
|
+
run: async (db, prefix) => {
|
|
1362
|
+
const row = await db.getFirstAsync(
|
|
1363
|
+
`SELECT count(*) AS n FROM sqlite_master WHERE type = 'table' AND name = ?`,
|
|
1364
|
+
[`${prefix}fts_pending`]
|
|
1365
|
+
);
|
|
1366
|
+
if (row?.n) await db.execAsync(fts5PendingIndexesDdl(prefix));
|
|
1367
|
+
}
|
|
1218
1368
|
}
|
|
1219
1369
|
];
|
|
1220
1370
|
async function createLegacySourceTypeIndex(db, prefix) {
|
|
@@ -1400,6 +1550,44 @@ var BaseRepository = class {
|
|
|
1400
1550
|
}
|
|
1401
1551
|
};
|
|
1402
1552
|
|
|
1553
|
+
// src/services/search/fts5Progress.ts
|
|
1554
|
+
async function writeRebuildStart(tx, prefix, meta) {
|
|
1555
|
+
const e = await tx.getFirstAsync(`SELECT coalesce(max(rowid), 0) AS hi FROM ${prefix}entries`);
|
|
1556
|
+
const m = await tx.getFirstAsync(`SELECT coalesce(max(fts_rowid), 0) AS hi FROM ${prefix}fts_map`);
|
|
1557
|
+
await meta.setMeta(FTS5_REBUILD_WM_KEY, JSON.stringify({ rowid: 0, id: null }), tx);
|
|
1558
|
+
await meta.setMeta(FTS5_REBUILD_SNAPSHOT_KEY, JSON.stringify({ entries: e?.hi ?? 0, map: m?.hi ?? 0 }), tx);
|
|
1559
|
+
await meta.setMeta(FTS5_SWEEP_CURSOR_KEY, "0", tx);
|
|
1560
|
+
}
|
|
1561
|
+
async function readRebuildProgress(meta, tx) {
|
|
1562
|
+
const wm = parseJsonObject(await meta.getMeta(FTS5_REBUILD_WM_KEY, tx));
|
|
1563
|
+
const snap = parseJsonObject(await meta.getMeta(FTS5_REBUILD_SNAPSHOT_KEY, tx));
|
|
1564
|
+
const rawCursor = await meta.getMeta(FTS5_SWEEP_CURSOR_KEY, tx);
|
|
1565
|
+
const cursor = rawCursor === null ? Number.NaN : Number(rawCursor);
|
|
1566
|
+
if (typeof wm?.rowid !== "number" || !(typeof wm.id === "string" || wm.id === null)) return null;
|
|
1567
|
+
if (typeof snap?.entries !== "number" || typeof snap.map !== "number" || !Number.isFinite(cursor)) return null;
|
|
1568
|
+
return { wm: { rowid: wm.rowid, id: wm.id }, snapshot: { entries: snap.entries, map: snap.map }, cursor };
|
|
1569
|
+
}
|
|
1570
|
+
async function reanchorRebuildFingerprint(tx, prefix, deletedIds) {
|
|
1571
|
+
if (deletedIds.length === 0) return;
|
|
1572
|
+
const metaValue = async (key) => (await tx.getFirstAsync(`SELECT value FROM ${prefix}meta WHERE key = ?`, [key]))?.value ?? null;
|
|
1573
|
+
if (await metaValue(FTS5_STATE_KEY) !== "rebuilding") return;
|
|
1574
|
+
const wm = parseJsonObject(await metaValue(FTS5_REBUILD_WM_KEY));
|
|
1575
|
+
if (typeof wm?.rowid !== "number" || typeof wm.id !== "string" || !deletedIds.includes(wm.id)) return;
|
|
1576
|
+
if ((await tx.getFirstAsync(`SELECT id FROM ${prefix}entries WHERE rowid = ?`, [wm.rowid]))?.id === wm.id) return;
|
|
1577
|
+
const below = await tx.getFirstAsync(
|
|
1578
|
+
`SELECT rowid, id FROM ${prefix}entries WHERE rowid < ? ORDER BY rowid DESC LIMIT 1`,
|
|
1579
|
+
[wm.rowid]
|
|
1580
|
+
);
|
|
1581
|
+
const next = below ? { rowid: below.rowid, id: below.id } : { rowid: 0, id: null };
|
|
1582
|
+
await tx.runAsync(`UPDATE ${prefix}meta SET value = ? WHERE key = ?`, [JSON.stringify(next), FTS5_REBUILD_WM_KEY]);
|
|
1583
|
+
}
|
|
1584
|
+
async function restartRebuildAfterVacuum(db, prefix, meta) {
|
|
1585
|
+
await db.withTransactionAsync(async (tx) => {
|
|
1586
|
+
if (await meta.getMeta(FTS5_STATE_KEY, tx) !== "rebuilding") return;
|
|
1587
|
+
await writeRebuildStart(tx, prefix, meta);
|
|
1588
|
+
});
|
|
1589
|
+
}
|
|
1590
|
+
|
|
1403
1591
|
// src/utils/temporal.ts
|
|
1404
1592
|
function liveAtSql(mode, alias = "") {
|
|
1405
1593
|
const from = mode === "asOf" ? `COALESCE(${alias}valid_from, ${alias}created_at) <= ?` : `(${alias}valid_from IS NULL OR ${alias}valid_from <= ?)`;
|
|
@@ -2047,6 +2235,7 @@ var _EntryRepository = class _EntryRepository extends BaseRepository {
|
|
|
2047
2235
|
}
|
|
2048
2236
|
}
|
|
2049
2237
|
}
|
|
2238
|
+
if (totalChanges > 0) await reanchorRebuildFingerprint(executor, this.prefix, ids);
|
|
2050
2239
|
return totalChanges;
|
|
2051
2240
|
}
|
|
2052
2241
|
/**
|
|
@@ -4153,7 +4342,7 @@ var UnavailableIndexStrategy = class {
|
|
|
4153
4342
|
hasIndexedEntity(_entityId) {
|
|
4154
4343
|
return false;
|
|
4155
4344
|
}
|
|
4156
|
-
async drain() {
|
|
4345
|
+
async drain(_opts) {
|
|
4157
4346
|
}
|
|
4158
4347
|
};
|
|
4159
4348
|
|
|
@@ -4199,24 +4388,31 @@ var _SearchService = class _SearchService {
|
|
|
4199
4388
|
isKeywordAvailable() {
|
|
4200
4389
|
return !(this.indexStrategy instanceof UnavailableIndexStrategy);
|
|
4201
4390
|
}
|
|
4391
|
+
/** True when the installed strategy drains a durable ledger (FTS5, or the degraded stand-in). */
|
|
4392
|
+
hasLedger() {
|
|
4393
|
+
return this.indexStrategy.drain !== void 0;
|
|
4394
|
+
}
|
|
4202
4395
|
/**
|
|
4203
4396
|
* One serialized drain turn for strategies with a durable ledger. Like
|
|
4204
4397
|
* sync(), it never rejects: the keyword index is rebuildable, and a failed
|
|
4205
|
-
* drain leaves the ledger rows in place for the next turn.
|
|
4398
|
+
* drain leaves the ledger rows in place for the next turn. Resolves false
|
|
4399
|
+
* when the drain failed.
|
|
4206
4400
|
*/
|
|
4207
|
-
drainTurn(entityId) {
|
|
4401
|
+
drainTurn(entityId, opts) {
|
|
4208
4402
|
const work = this.syncChain.then(async () => {
|
|
4209
4403
|
try {
|
|
4210
4404
|
try {
|
|
4211
|
-
await this.indexStrategy.drain();
|
|
4405
|
+
await this.indexStrategy.drain(opts);
|
|
4212
4406
|
} finally {
|
|
4213
4407
|
this.evictCache(entityId);
|
|
4214
4408
|
}
|
|
4409
|
+
return true;
|
|
4215
4410
|
} catch (err) {
|
|
4216
4411
|
console.warn(`[WikiMemory] search index drain failed for ${entityId ?? "*"}:`, err);
|
|
4412
|
+
return false;
|
|
4217
4413
|
}
|
|
4218
4414
|
});
|
|
4219
|
-
this.syncChain = work;
|
|
4415
|
+
this.syncChain = work.then(() => void 0);
|
|
4220
4416
|
return work;
|
|
4221
4417
|
}
|
|
4222
4418
|
/**
|
|
@@ -4226,9 +4422,18 @@ var _SearchService = class _SearchService {
|
|
|
4226
4422
|
* Rebuilds are serialized per instance and never reject: the keyword index
|
|
4227
4423
|
* is a rebuildable cache over SQLite, so degraded keyword search is the
|
|
4228
4424
|
* correct failure mode and killing the host process is not.
|
|
4229
|
-
|
|
4230
|
-
|
|
4231
|
-
|
|
4425
|
+
*
|
|
4426
|
+
* With a ledger strategy, `sync()` runs one full quantum bounded by
|
|
4427
|
+
* `opts.deadline`; `sync(entityId)` is a write turn (ledger only).
|
|
4428
|
+
* `opts.ids` are the caller's touched ids: a ledger strategy drains them
|
|
4429
|
+
* first, to completion; MiniSearch ignores them and rebuilds the entity in
|
|
4430
|
+
* one scan.
|
|
4431
|
+
*/
|
|
4432
|
+
async sync(entityId, opts) {
|
|
4433
|
+
if (this.indexStrategy.drain) {
|
|
4434
|
+
await this.drainTurn(entityId, entityId === void 0 ? { mode: "full", deadline: opts?.deadline, ids: opts?.ids } : { mode: "ledger", entityId, ids: opts?.ids });
|
|
4435
|
+
return;
|
|
4436
|
+
}
|
|
4232
4437
|
const work = this.syncChain.then(async () => {
|
|
4233
4438
|
try {
|
|
4234
4439
|
try {
|
|
@@ -4269,13 +4474,17 @@ var _SearchService = class _SearchService {
|
|
|
4269
4474
|
* — including on an empty `ids` list, which is a no-op only for an entity
|
|
4270
4475
|
* the index already tracks, and even then one that waits for rebuilds
|
|
4271
4476
|
* already queued on the chain.
|
|
4477
|
+
*
|
|
4478
|
+
* Resolves false only when a ledger drain failed: the caller's ids were not
|
|
4479
|
+
* applied (their ledger rows remain), so a caller holding them retries.
|
|
4272
4480
|
*/
|
|
4273
4481
|
async syncEntries(entityId, ids) {
|
|
4274
|
-
if (this.indexStrategy.drain) return this.drainTurn(entityId);
|
|
4482
|
+
if (this.indexStrategy.drain) return this.drainTurn(entityId, { mode: "ledger", entityId, ids: [...new Set(ids)] });
|
|
4275
4483
|
const uniqueIds = [...new Set(ids)];
|
|
4276
4484
|
const indexed = this.indexStrategy.hasIndexedEntity?.(entityId);
|
|
4277
4485
|
if (uniqueIds.length === 0 && !this.staleEntities.has(entityId) && indexed === true) {
|
|
4278
|
-
|
|
4486
|
+
await this.syncChain;
|
|
4487
|
+
return true;
|
|
4279
4488
|
}
|
|
4280
4489
|
const work = this.syncChain.then(async () => {
|
|
4281
4490
|
try {
|
|
@@ -4304,7 +4513,8 @@ var _SearchService = class _SearchService {
|
|
|
4304
4513
|
}
|
|
4305
4514
|
});
|
|
4306
4515
|
this.syncChain = work;
|
|
4307
|
-
|
|
4516
|
+
await work;
|
|
4517
|
+
return true;
|
|
4308
4518
|
}
|
|
4309
4519
|
/**
|
|
4310
4520
|
* Forces the entity's next syncEntries to rebuild it in full. For writes core
|
|
@@ -4320,11 +4530,13 @@ var _SearchService = class _SearchService {
|
|
|
4320
4530
|
* so only those entities are rebuilt. Each rebuild still takes its own turn
|
|
4321
4531
|
* on the sync chain, and like syncEntries this never rejects. With nothing
|
|
4322
4532
|
* stale it resolves at once, without waiting on work already on the chain.
|
|
4533
|
+
* Resolves false only when a ledger drain failed (see syncEntries).
|
|
4323
4534
|
*/
|
|
4324
|
-
async syncStale() {
|
|
4325
|
-
if (this.indexStrategy.drain) return this.drainTurn();
|
|
4535
|
+
async syncStale(opts) {
|
|
4536
|
+
if (this.indexStrategy.drain) return this.drainTurn(void 0, { mode: "full", ids: opts?.ids });
|
|
4326
4537
|
const ids = [...this.staleEntities];
|
|
4327
4538
|
await Promise.all(ids.map((id) => this.syncEntries(id, [])));
|
|
4539
|
+
return true;
|
|
4328
4540
|
}
|
|
4329
4541
|
/**
|
|
4330
4542
|
* Clears the parsed vector cache. Useful for mid-loop flush guarantees
|
|
@@ -4665,87 +4877,11 @@ function assertOwnedBy(entityId, documents) {
|
|
|
4665
4877
|
}
|
|
4666
4878
|
}
|
|
4667
4879
|
|
|
4668
|
-
// src/
|
|
4669
|
-
var
|
|
4670
|
-
|
|
4671
|
-
|
|
4672
|
-
|
|
4673
|
-
}
|
|
4674
|
-
function tagsExpr(col) {
|
|
4675
|
-
return `CASE WHEN json_valid(${col}) THEN
|
|
4676
|
-
CASE WHEN json_type(${col}) = 'array' THEN
|
|
4677
|
-
(SELECT group_concat(CASE type WHEN 'true' THEN 'true' WHEN 'false' THEN 'false'
|
|
4678
|
-
WHEN 'null' THEN '' ELSE value END, ' ')
|
|
4679
|
-
FROM json_each(${col}))
|
|
4680
|
-
ELSE ${col} END
|
|
4681
|
-
ELSE ${col} END`;
|
|
4682
|
-
}
|
|
4683
|
-
function fts5TablesDdl(p) {
|
|
4684
|
-
return `
|
|
4685
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS ${p}entries_fts USING fts5(
|
|
4686
|
-
id UNINDEXED, entity_id UNINDEXED, title, body, tags,
|
|
4687
|
-
tokenize = 'porter unicode61'
|
|
4688
|
-
);
|
|
4689
|
-
CREATE TABLE IF NOT EXISTS ${p}fts_map (
|
|
4690
|
-
fts_rowid INTEGER PRIMARY KEY,
|
|
4691
|
-
id TEXT NOT NULL UNIQUE,
|
|
4692
|
-
entity_id TEXT NOT NULL
|
|
4693
|
-
);
|
|
4694
|
-
CREATE INDEX IF NOT EXISTS ${p}fts_map_entity ON ${p}fts_map(entity_id);
|
|
4695
|
-
CREATE TABLE IF NOT EXISTS ${p}fts_pending (
|
|
4696
|
-
seq INTEGER PRIMARY KEY,
|
|
4697
|
-
entity_id TEXT NOT NULL,
|
|
4698
|
-
id TEXT NOT NULL
|
|
4699
|
-
);`;
|
|
4700
|
-
}
|
|
4701
|
-
function fts5TriggersDdl(p) {
|
|
4702
|
-
const [ai, au, ad] = fts5TriggerNames(p);
|
|
4703
|
-
return `
|
|
4704
|
-
CREATE TRIGGER IF NOT EXISTS ${ai} AFTER INSERT ON ${p}entries BEGIN
|
|
4705
|
-
INSERT INTO ${p}fts_pending (entity_id, id) VALUES (NEW.entity_id, NEW.id);
|
|
4706
|
-
END;
|
|
4707
|
-
CREATE TRIGGER IF NOT EXISTS ${au}
|
|
4708
|
-
AFTER UPDATE OF id, entity_id, title, body, tags, deleted_at ON ${p}entries BEGIN
|
|
4709
|
-
INSERT INTO ${p}fts_pending (entity_id, id) VALUES (OLD.entity_id, OLD.id);
|
|
4710
|
-
INSERT INTO ${p}fts_pending (entity_id, id)
|
|
4711
|
-
SELECT NEW.entity_id, NEW.id WHERE NEW.id <> OLD.id OR NEW.entity_id <> OLD.entity_id;
|
|
4712
|
-
END;
|
|
4713
|
-
CREATE TRIGGER IF NOT EXISTS ${ad} AFTER DELETE ON ${p}entries BEGIN
|
|
4714
|
-
INSERT INTO ${p}fts_pending (entity_id, id) VALUES (OLD.entity_id, OLD.id);
|
|
4715
|
-
END;`;
|
|
4716
|
-
}
|
|
4717
|
-
function drainHiSql(p) {
|
|
4718
|
-
return `SELECT max(seq) AS hi FROM (SELECT seq FROM ${p}fts_pending ORDER BY seq LIMIT ${FTS5_DRAIN_CHUNK})`;
|
|
4719
|
-
}
|
|
4720
|
-
function drainChunkSql(p) {
|
|
4721
|
-
const chunkIds = `SELECT id FROM ${p}fts_pending WHERE seq <= ?`;
|
|
4722
|
-
return [
|
|
4723
|
-
`DELETE FROM ${p}entries_fts WHERE rowid IN (SELECT m.fts_rowid FROM ${p}fts_map m WHERE m.id IN (${chunkIds}))`,
|
|
4724
|
-
`DELETE FROM ${p}fts_map WHERE id IN (${chunkIds})`,
|
|
4725
|
-
`INSERT INTO ${p}fts_map (id, entity_id)
|
|
4726
|
-
SELECT e.id, e.entity_id FROM ${p}entries e WHERE e.id IN (${chunkIds}) AND e.deleted_at IS NULL`,
|
|
4727
|
-
`INSERT INTO ${p}entries_fts (rowid, id, entity_id, title, body, tags)
|
|
4728
|
-
SELECT m.fts_rowid, e.id, e.entity_id, e.title, e.body, ${tagsExpr("e.tags")}
|
|
4729
|
-
FROM ${p}fts_map m JOIN ${p}entries e ON e.id = m.id
|
|
4730
|
-
WHERE m.id IN (${chunkIds})`,
|
|
4731
|
-
`DELETE FROM ${p}fts_pending WHERE seq <= ?`
|
|
4732
|
-
];
|
|
4733
|
-
}
|
|
4734
|
-
function rebuildSql(p) {
|
|
4735
|
-
return [
|
|
4736
|
-
`INSERT INTO ${p}fts_map (id, entity_id) SELECT id, entity_id FROM ${p}entries WHERE deleted_at IS NULL`,
|
|
4737
|
-
`INSERT INTO ${p}entries_fts (rowid, id, entity_id, title, body, tags)
|
|
4738
|
-
SELECT m.fts_rowid, e.id, e.entity_id, e.title, e.body, ${tagsExpr("e.tags")}
|
|
4739
|
-
FROM ${p}fts_map m JOIN ${p}entries e ON e.id = m.id`
|
|
4740
|
-
];
|
|
4741
|
-
}
|
|
4742
|
-
function fts5LedgerDropSql(p) {
|
|
4743
|
-
return [
|
|
4744
|
-
...fts5TriggerNames(p).map((n) => `DROP TRIGGER IF EXISTS ${n};`),
|
|
4745
|
-
`DROP TABLE IF EXISTS ${p}fts_pending;`,
|
|
4746
|
-
`DROP TABLE IF EXISTS ${p}fts_map;`
|
|
4747
|
-
].join("\n");
|
|
4748
|
-
}
|
|
4880
|
+
// src/utils/monotonicNow.ts
|
|
4881
|
+
var monotonicNow = () => typeof performance !== "undefined" && typeof performance.now === "function" ? performance.now() : Date.now();
|
|
4882
|
+
|
|
4883
|
+
// src/utils/setupDefaults.ts
|
|
4884
|
+
var DEFAULT_SETUP_BUDGET_MS = 1500;
|
|
4749
4885
|
|
|
4750
4886
|
// src/services/search/ftsQuery.ts
|
|
4751
4887
|
var TOKEN = /[\p{L}\p{N}]+/gu;
|
|
@@ -4758,41 +4894,363 @@ function buildFtsMatchQuery(query) {
|
|
|
4758
4894
|
|
|
4759
4895
|
// src/services/search/Fts5IndexStrategy.ts
|
|
4760
4896
|
var Fts5IndexStrategy = class {
|
|
4761
|
-
constructor(db, prefix, metadataRepo) {
|
|
4897
|
+
constructor(db, prefix, metadataRepo, budgetMs = DEFAULT_SETUP_BUDGET_MS) {
|
|
4762
4898
|
this.db = db;
|
|
4763
4899
|
this.prefix = prefix;
|
|
4764
4900
|
this.metadataRepo = metadataRepo;
|
|
4901
|
+
this.budgetMs = budgetMs;
|
|
4765
4902
|
}
|
|
4766
|
-
|
|
4903
|
+
/**
|
|
4904
|
+
* Resolves the index state (spec #280 Change 3 transition table). Every
|
|
4905
|
+
* read-only probe runs before the one transition transaction, so a probe
|
|
4906
|
+
* failure leaves zero mutation (Change 4 ordering).
|
|
4907
|
+
*/
|
|
4908
|
+
async init(opts = {}) {
|
|
4909
|
+
try {
|
|
4910
|
+
await this.initOnce(opts);
|
|
4911
|
+
} catch (err) {
|
|
4912
|
+
const wrapped = unwrapFtsOnly(err);
|
|
4913
|
+
const cls = ftsCorruptionClass(wrapped.cause);
|
|
4914
|
+
if (cls === "vtab" || cls === "corrupt" && wrapped.ftsOnly) {
|
|
4915
|
+
await this.resetAndRearm();
|
|
4916
|
+
return;
|
|
4917
|
+
}
|
|
4918
|
+
throw wrapped.cause;
|
|
4919
|
+
}
|
|
4920
|
+
}
|
|
4921
|
+
/** Marks errors from statements that touch only FTS tables (entries_fts + shadows). */
|
|
4922
|
+
async ftsOnly(fn) {
|
|
4923
|
+
try {
|
|
4924
|
+
return await fn();
|
|
4925
|
+
} catch (err) {
|
|
4926
|
+
throw new FtsOnlyStatementError(err);
|
|
4927
|
+
}
|
|
4928
|
+
}
|
|
4929
|
+
async initOnce(opts) {
|
|
4930
|
+
const p = this.prefix;
|
|
4767
4931
|
const state = await this.metadataRepo.getMeta(FTS5_STATE_KEY);
|
|
4768
|
-
|
|
4769
|
-
|
|
4932
|
+
const schema = await this.readSchema();
|
|
4933
|
+
const ftsExists = schema.tables.has(`${p}entries_fts`);
|
|
4934
|
+
const mapExists = schema.tables.has(`${p}fts_map`);
|
|
4935
|
+
const pendingExists = schema.tables.has(`${p}fts_pending`);
|
|
4936
|
+
const trig = schema.triggersPresent;
|
|
4937
|
+
const ftsDdl = schema.tables.get(`${p}entries_fts`) ?? null;
|
|
4938
|
+
const tokenizerOk = ftsDdl === null || /tokenize\s*=\s*['"]porter\s+unicode61['"]/i.test(ftsDdl);
|
|
4939
|
+
if (state === null && opts.entriesExisted === false) {
|
|
4940
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
4941
|
+
await tx.execAsync(fts5TablesDdl(p) + fts5PendingIndexesDdl(p) + fts5TriggersDdl(p));
|
|
4942
|
+
await this.metadataRepo.setMeta(FTS5_STATE_KEY, "live", tx);
|
|
4943
|
+
});
|
|
4944
|
+
return;
|
|
4945
|
+
}
|
|
4946
|
+
if (state === "detached") return this.resetAndRearm();
|
|
4947
|
+
if (!tokenizerOk) return this.resetAndRearm();
|
|
4948
|
+
const intact = trig && ftsExists && mapExists && pendingExists;
|
|
4949
|
+
if (intact && state === "rebuilding") return;
|
|
4950
|
+
if (intact && state === "live") {
|
|
4951
|
+
await this.ftsOnly(() => this.db.getFirstAsync(`SELECT rowid FROM ${p}entries_fts ORDER BY rowid DESC LIMIT 1`));
|
|
4952
|
+
if (await this.liveRowUnindexed()) return this.resetAndRearm();
|
|
4953
|
+
return;
|
|
4954
|
+
}
|
|
4955
|
+
if (ftsExists && mapExists && await this.keepMapConsistent()) {
|
|
4956
|
+
await this.db.withTransactionAsync((tx) => this.rearmInTx(tx));
|
|
4957
|
+
return;
|
|
4958
|
+
}
|
|
4959
|
+
return this.resetAndRearm();
|
|
4960
|
+
}
|
|
4961
|
+
/**
|
|
4962
|
+
* The index was lost (rev 8 m3 / rev 9 m4): the oldest or the newest live entry is neither mapped nor
|
|
4963
|
+
* ledgered, which `live` never allows. The oldest one keeps detection armed after post-loss writes
|
|
4964
|
+
* repopulate the map and the ledger (#284 review). Every-open path: each side reads at most
|
|
4965
|
+
* FTS5_DRAIN_CHUNK entries rows (final review M1), so an all-soft-deleted store is never fully scanned;
|
|
4966
|
+
* a lost index whose oldest and newest rows are all deleted goes undetected.
|
|
4967
|
+
*/
|
|
4968
|
+
async liveRowUnindexed() {
|
|
4969
|
+
const p = this.prefix;
|
|
4970
|
+
const edge = (dir) => `SELECT id FROM (SELECT id FROM (SELECT id, deleted_at FROM ${p}entries ORDER BY rowid ${dir} LIMIT ${FTS5_DRAIN_CHUNK}) WHERE deleted_at IS NULL LIMIT 1)`;
|
|
4971
|
+
return await this.db.getFirstAsync(
|
|
4972
|
+
`SELECT 1 AS x FROM (${edge("ASC")} UNION ALL ${edge("DESC")}) e
|
|
4973
|
+
WHERE NOT EXISTS (SELECT 1 FROM ${p}fts_map m WHERE m.id = e.id)
|
|
4974
|
+
AND NOT EXISTS (SELECT 1 FROM ${p}fts_pending q WHERE q.id = e.id)
|
|
4975
|
+
LIMIT 1`
|
|
4976
|
+
) !== null;
|
|
4977
|
+
}
|
|
4978
|
+
/**
|
|
4979
|
+
* Read-only keep-map check (spec DDL note): every surviving entries_fts
|
|
4980
|
+
* rowid must have a map row. Max-rowid seek, then docsize vs map counts.
|
|
4981
|
+
*/
|
|
4982
|
+
async keepMapConsistent() {
|
|
4983
|
+
const p = this.prefix;
|
|
4984
|
+
const fts = await this.ftsOnly(() => this.db.getFirstAsync(`SELECT rowid FROM ${p}entries_fts ORDER BY rowid DESC LIMIT 1`));
|
|
4985
|
+
const map = await this.db.getFirstAsync(
|
|
4986
|
+
`SELECT coalesce(max(fts_rowid), 0) AS hi, count(*) AS n FROM ${p}fts_map`
|
|
4987
|
+
);
|
|
4988
|
+
if ((fts?.rowid ?? 0) > (map?.hi ?? 0)) return false;
|
|
4989
|
+
const doc = await this.ftsOnly(() => this.db.getFirstAsync(`SELECT count(*) AS n FROM ${p}entries_fts_docsize`));
|
|
4990
|
+
return (doc?.n ?? 0) === (map?.n ?? 0);
|
|
4991
|
+
}
|
|
4992
|
+
/** Keep-map re-arm: triggers, empty ledger, rebuild from 0, state rebuilding. One transaction. */
|
|
4993
|
+
async rearmInTx(tx) {
|
|
4994
|
+
const p = this.prefix;
|
|
4995
|
+
await tx.execAsync(fts5TablesDdl(p));
|
|
4996
|
+
await tx.execAsync(`DELETE FROM ${p}fts_pending;`);
|
|
4997
|
+
await tx.execAsync(fts5PendingIndexesDdl(p) + fts5TriggersDdl(p));
|
|
4998
|
+
await writeRebuildStart(tx, p, this.metadataRepo);
|
|
4999
|
+
await this.metadataRepo.setMeta(FTS5_STATE_KEY, "rebuilding", tx);
|
|
5000
|
+
}
|
|
5001
|
+
/** Reset path: retire entries_fts, recreate it empty, clear map + ledger, then the re-arm. One transaction. */
|
|
5002
|
+
async resetAndRearm() {
|
|
5003
|
+
const p = this.prefix;
|
|
5004
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
5005
|
+
await this.retireFtsTable(tx);
|
|
5006
|
+
await tx.execAsync(fts5TablesDdl(p));
|
|
5007
|
+
await tx.execAsync(`DELETE FROM ${p}fts_map; DELETE FROM ${p}fts_pending;`);
|
|
5008
|
+
await this.rearmInTx(tx);
|
|
5009
|
+
});
|
|
5010
|
+
}
|
|
5011
|
+
/** Rename-retire entries_fts (spec #280 rev 11); DROP only when the RENAME throws. */
|
|
5012
|
+
async retireFtsTable(tx) {
|
|
5013
|
+
const p = this.prefix;
|
|
5014
|
+
if (!await this.metadataRepo.tableExists(`${p}entries_fts`, tx)) return;
|
|
5015
|
+
const n = Number(await this.metadataRepo.getMeta(FTS5_RETIRED_COUNTER_KEY, tx) ?? "0") + 1;
|
|
5016
|
+
const name = retiredTableName(p, n);
|
|
5017
|
+
try {
|
|
5018
|
+
await tx.execAsync(`ALTER TABLE ${p}entries_fts RENAME TO ${name}`);
|
|
5019
|
+
} catch (err) {
|
|
5020
|
+
if (isTransientOrTxFatal(err)) throw err;
|
|
5021
|
+
await tx.execAsync(`DROP TABLE ${p}entries_fts`);
|
|
4770
5022
|
return;
|
|
4771
5023
|
}
|
|
5024
|
+
await this.metadataRepo.setMeta(FTS5_RETIRED_TABLES_KEY, JSON.stringify([...await this.retiredList(tx), name]), tx);
|
|
5025
|
+
await this.metadataRepo.setMeta(FTS5_RETIRED_COUNTER_KEY, String(n), tx);
|
|
5026
|
+
}
|
|
5027
|
+
async retiredList(tx = this.db) {
|
|
5028
|
+
const raw = parseJsonArray(await this.metadataRepo.getMeta(FTS5_RETIRED_TABLES_KEY, tx), []);
|
|
5029
|
+
return raw.filter((n) => isRetiredTableName(this.prefix, n));
|
|
5030
|
+
}
|
|
5031
|
+
async drain(opts = {}) {
|
|
5032
|
+
try {
|
|
5033
|
+
await this.drainOnce(opts);
|
|
5034
|
+
} catch (err) {
|
|
5035
|
+
if (ftsCorruptionClass(err) !== "vtab") throw err;
|
|
5036
|
+
await this.resetAndRearm();
|
|
5037
|
+
if (opts.ids && opts.ids.length > 0) await this.drainIds(opts.ids, idReindexChunkSql(this.prefix));
|
|
5038
|
+
}
|
|
5039
|
+
}
|
|
5040
|
+
async drainOnce(opts) {
|
|
5041
|
+
const ids = opts.ids ?? [];
|
|
5042
|
+
if (ids.length > 0) await this.drainIds(ids);
|
|
5043
|
+
if (opts.mode === "ledger") {
|
|
5044
|
+
if (ids.length === 0 && opts.entityId !== void 0) await this.drainEntityChunk(opts.entityId);
|
|
5045
|
+
else await this.drainLedgerChunk();
|
|
5046
|
+
return;
|
|
5047
|
+
}
|
|
5048
|
+
await this.fullQuantum(opts.deadline ?? monotonicNow() + this.budgetMs);
|
|
5049
|
+
}
|
|
5050
|
+
/**
|
|
5051
|
+
* One full quantum (spec #280 Change 3): ledger first (at most half the
|
|
5052
|
+
* budget while `rebuilding`); then, while `rebuilding`, ≥1 window unless
|
|
5053
|
+
* the ledger drain overran the deadline, one sweep tx, and the flip check;
|
|
5054
|
+
* then reclamation. A `live` store reads only the state.
|
|
5055
|
+
*/
|
|
5056
|
+
async fullQuantum(deadline) {
|
|
5057
|
+
const rebuilding = await this.metadataRepo.getMeta(FTS5_STATE_KEY) === "rebuilding";
|
|
5058
|
+
const start = monotonicNow();
|
|
5059
|
+
const ledgerRan = await this.drainLedgerUntil(rebuilding ? start + (deadline - start) / 2 : deadline);
|
|
5060
|
+
const ledgerOverran = ledgerRan && monotonicNow() >= deadline;
|
|
5061
|
+
if (rebuilding) {
|
|
5062
|
+
const progress = await readRebuildProgress(this.metadataRepo);
|
|
5063
|
+
if (!progress || !await this.fingerprintValid(progress.wm)) {
|
|
5064
|
+
await this.restartRebuild();
|
|
5065
|
+
} else {
|
|
5066
|
+
if (!ledgerOverran) {
|
|
5067
|
+
let step;
|
|
5068
|
+
do {
|
|
5069
|
+
step = await this.runWindow(progress);
|
|
5070
|
+
} while (step === "more" && monotonicNow() < deadline);
|
|
5071
|
+
if (step !== "stale") await this.runSweepTx(progress);
|
|
5072
|
+
}
|
|
5073
|
+
await this.tryFlip();
|
|
5074
|
+
}
|
|
5075
|
+
}
|
|
5076
|
+
try {
|
|
5077
|
+
await this.reclaimRetired(deadline);
|
|
5078
|
+
} catch (err) {
|
|
5079
|
+
if (!isSqliteError(err)) throw err;
|
|
5080
|
+
}
|
|
5081
|
+
}
|
|
5082
|
+
/** Bounded reclamation steps, each its own tx, until the deadline or nothing is retired. */
|
|
5083
|
+
async reclaimRetired(deadline) {
|
|
5084
|
+
while (monotonicNow() < deadline) {
|
|
5085
|
+
const [name] = await this.retiredList();
|
|
5086
|
+
if (name === void 0) return;
|
|
5087
|
+
try {
|
|
5088
|
+
if (!await this.metadataRepo.tableExists(name)) {
|
|
5089
|
+
await this.delistRetired(name, false);
|
|
5090
|
+
continue;
|
|
5091
|
+
}
|
|
5092
|
+
while (!await this.reclaimStep(name) && monotonicNow() < deadline) ;
|
|
5093
|
+
} catch (err) {
|
|
5094
|
+
if (ftsCorruptionClass(err) === null) throw err;
|
|
5095
|
+
try {
|
|
5096
|
+
await this.delistRetired(name, true);
|
|
5097
|
+
} catch (dropErr) {
|
|
5098
|
+
if (!isSqliteError(dropErr)) throw dropErr;
|
|
5099
|
+
await this.delistRetired(name, false);
|
|
5100
|
+
}
|
|
5101
|
+
}
|
|
5102
|
+
}
|
|
5103
|
+
}
|
|
5104
|
+
/** (1) delete <=500 rows, else (2) one bounded merge, else (3) DROP + delist. True once delisted. */
|
|
5105
|
+
async reclaimStep(name) {
|
|
5106
|
+
if (await this.db.getFirstAsync(`SELECT rowid FROM ${name} LIMIT 1`)) {
|
|
5107
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
5108
|
+
await tx.runAsync(`DELETE FROM ${name} WHERE rowid IN (SELECT rowid FROM ${name} ORDER BY rowid LIMIT ${FTS5_DRAIN_CHUNK})`);
|
|
5109
|
+
});
|
|
5110
|
+
return false;
|
|
5111
|
+
}
|
|
5112
|
+
let merged = false;
|
|
5113
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
5114
|
+
const before = await tx.getFirstAsync(`SELECT total_changes() AS n`);
|
|
5115
|
+
await tx.runAsync(`INSERT INTO ${name}(${name}, rank) VALUES('merge', ${FTS5_RECLAIM_MERGE_PAGES})`);
|
|
5116
|
+
const after = await tx.getFirstAsync(`SELECT total_changes() AS n`);
|
|
5117
|
+
merged = (after?.n ?? 0) - (before?.n ?? 0) >= 2;
|
|
5118
|
+
});
|
|
5119
|
+
if (!merged) await this.delistRetired(name, true);
|
|
5120
|
+
return !merged;
|
|
5121
|
+
}
|
|
5122
|
+
async delistRetired(name, drop) {
|
|
5123
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
5124
|
+
if (drop) await tx.execAsync(`DROP TABLE IF EXISTS ${name}`);
|
|
5125
|
+
const rest = (await this.retiredList(tx)).filter((n) => n !== name);
|
|
5126
|
+
await this.metadataRepo.setMeta(FTS5_RETIRED_TABLES_KEY, JSON.stringify(rest), tx);
|
|
5127
|
+
});
|
|
5128
|
+
}
|
|
5129
|
+
/** The id at the watermark's rowid still matches. A null id (start, or windows complete) has nothing to check. */
|
|
5130
|
+
async fingerprintValid(wm, tx = this.db) {
|
|
5131
|
+
if (wm.id === null) return true;
|
|
5132
|
+
const row = await tx.getFirstAsync(`SELECT id FROM ${this.prefix}entries WHERE rowid = ?`, [wm.rowid]);
|
|
5133
|
+
return row?.id === wm.id;
|
|
5134
|
+
}
|
|
5135
|
+
/** External change under the watermark: start over (same re-arm transaction as a trigger-loss restart). */
|
|
5136
|
+
async restartRebuild() {
|
|
5137
|
+
await this.db.withTransactionAsync((tx) => this.rearmInTx(tx));
|
|
5138
|
+
}
|
|
5139
|
+
/**
|
|
5140
|
+
* Progress as stored, re-read inside a window/sweep tx: false when it no
|
|
5141
|
+
* longer matches what this quantum assumed (final review I1). Writing a
|
|
5142
|
+
* watermark or cursor computed from a stale `pr` would overwrite a
|
|
5143
|
+
* concurrent restart (runPrune VACUUM is not on the sync chain).
|
|
5144
|
+
*/
|
|
5145
|
+
async progressUnchanged(pr, tx) {
|
|
5146
|
+
if (await this.metadataRepo.getMeta(FTS5_STATE_KEY, tx) !== "rebuilding") return false;
|
|
5147
|
+
const cur = await readRebuildProgress(this.metadataRepo, tx);
|
|
5148
|
+
return cur !== null && cur.wm.rowid === pr.wm.rowid && cur.wm.id === pr.wm.id && cur.cursor === pr.cursor && cur.snapshot.entries === pr.snapshot.entries && cur.snapshot.map === pr.snapshot.map;
|
|
5149
|
+
}
|
|
5150
|
+
/** One window tx. 'done' once no rows remain in (wm, snapshot] (the watermark is then pinned at the snapshot, F9); 'stale' when progress moved under the quantum. */
|
|
5151
|
+
async runWindow(pr) {
|
|
5152
|
+
if (pr.wm.rowid >= pr.snapshot.entries) return "done";
|
|
5153
|
+
const p = this.prefix;
|
|
5154
|
+
let next = pr.wm;
|
|
5155
|
+
let more = false;
|
|
5156
|
+
let stale = false;
|
|
4772
5157
|
await this.db.withTransactionAsync(async (tx) => {
|
|
4773
|
-
await
|
|
4774
|
-
|
|
4775
|
-
|
|
4776
|
-
|
|
5158
|
+
if (!await this.progressUnchanged(pr, tx)) {
|
|
5159
|
+
stale = true;
|
|
5160
|
+
return;
|
|
5161
|
+
}
|
|
5162
|
+
const row = await tx.getFirstAsync(windowHiSql(p), [pr.wm.rowid, pr.snapshot.entries]);
|
|
5163
|
+
if (row?.hi == null) {
|
|
5164
|
+
next = { rowid: pr.snapshot.entries, id: null };
|
|
5165
|
+
} else {
|
|
5166
|
+
for (const sql of windowChunkSql(p)) await tx.runAsync(sql, [pr.wm.rowid, row.hi]);
|
|
5167
|
+
const fp = await tx.getFirstAsync(`SELECT id FROM ${p}entries WHERE rowid = ?`, [row.hi]);
|
|
5168
|
+
next = { rowid: row.hi, id: fp.id };
|
|
5169
|
+
more = true;
|
|
5170
|
+
}
|
|
5171
|
+
await this.metadataRepo.setMeta(FTS5_REBUILD_WM_KEY, JSON.stringify(next), tx);
|
|
5172
|
+
});
|
|
5173
|
+
if (stale) return "stale";
|
|
5174
|
+
pr.wm = next;
|
|
5175
|
+
return more ? "more" : "done";
|
|
5176
|
+
}
|
|
5177
|
+
/** At most one bounded sweep tx; the cursor advances in the same tx. */
|
|
5178
|
+
async runSweepTx(pr) {
|
|
5179
|
+
if (pr.cursor >= pr.snapshot.map) return;
|
|
5180
|
+
const p = this.prefix;
|
|
5181
|
+
let next = pr.cursor;
|
|
5182
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
5183
|
+
if (!await this.progressUnchanged(pr, tx)) return;
|
|
5184
|
+
const row = await tx.getFirstAsync(sweepHiSql(p), [pr.cursor, pr.snapshot.map]);
|
|
5185
|
+
const hi = row?.hi ?? pr.snapshot.map;
|
|
5186
|
+
for (const sql of sweepChunkSql(p)) await tx.runAsync(sql, [pr.cursor, hi]);
|
|
5187
|
+
await this.metadataRepo.setMeta(FTS5_SWEEP_CURSOR_KEY, String(hi), tx);
|
|
5188
|
+
next = hi;
|
|
5189
|
+
});
|
|
5190
|
+
pr.cursor = next;
|
|
5191
|
+
}
|
|
5192
|
+
/** Completion, re-checked inside the flipping tx; the residual ledger (≤ one chunk) drains in it. */
|
|
5193
|
+
async tryFlip() {
|
|
5194
|
+
const p = this.prefix;
|
|
5195
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
5196
|
+
if (await this.metadataRepo.getMeta(FTS5_STATE_KEY, tx) !== "rebuilding") return;
|
|
5197
|
+
const pr = await readRebuildProgress(this.metadataRepo, tx);
|
|
5198
|
+
if (!pr || pr.wm.rowid < pr.snapshot.entries || pr.cursor < pr.snapshot.map) return;
|
|
5199
|
+
if (!await this.fingerprintValid(pr.wm, tx)) return;
|
|
5200
|
+
const residual = await tx.getFirstAsync(
|
|
5201
|
+
`SELECT count(*) AS n FROM (SELECT 1 FROM ${p}fts_pending LIMIT ${FTS5_DRAIN_CHUNK + 1})`
|
|
5202
|
+
);
|
|
5203
|
+
if ((residual?.n ?? 0) > FTS5_DRAIN_CHUNK) return;
|
|
5204
|
+
const hi = await tx.getFirstAsync(drainHiSql(p));
|
|
5205
|
+
if (hi?.hi != null) for (const sql of drainChunkSql(p)) await tx.runAsync(sql, [hi.hi]);
|
|
5206
|
+
await this.metadataRepo.deleteMeta(FTS5_REBUILD_WM_KEY, tx);
|
|
5207
|
+
await this.metadataRepo.deleteMeta(FTS5_REBUILD_SNAPSHOT_KEY, tx);
|
|
5208
|
+
await this.metadataRepo.deleteMeta(FTS5_SWEEP_CURSOR_KEY, tx);
|
|
4777
5209
|
await this.metadataRepo.setMeta(FTS5_STATE_KEY, "live", tx);
|
|
4778
5210
|
});
|
|
4779
5211
|
}
|
|
4780
|
-
|
|
4781
|
-
|
|
5212
|
+
/** Seq-ordered ledger chunks until empty or `stopAt`. True when a chunk ran. */
|
|
5213
|
+
async drainLedgerUntil(stopAt) {
|
|
5214
|
+
let ran = false;
|
|
5215
|
+
while (await this.drainLedgerChunk()) {
|
|
5216
|
+
ran = true;
|
|
5217
|
+
if (monotonicNow() >= stopAt) break;
|
|
5218
|
+
}
|
|
5219
|
+
return ran;
|
|
4782
5220
|
}
|
|
4783
|
-
|
|
4784
|
-
|
|
4785
|
-
const
|
|
4786
|
-
for (; ; ) {
|
|
4787
|
-
const
|
|
4788
|
-
|
|
4789
|
-
|
|
4790
|
-
for (const sql of chunk) await tx.runAsync(sql, [row.hi]);
|
|
4791
|
-
return false;
|
|
5221
|
+
/** The caller's own ids, to completion, ≤ FTS5_DRAIN_CHUNK per transaction. */
|
|
5222
|
+
async drainIds(ids, chunk = idDrainChunkSql(this.prefix)) {
|
|
5223
|
+
const unique = [...new Set(ids)];
|
|
5224
|
+
for (let i = 0; i < unique.length; i += FTS5_DRAIN_CHUNK) {
|
|
5225
|
+
const json = JSON.stringify(unique.slice(i, i + FTS5_DRAIN_CHUNK));
|
|
5226
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
5227
|
+
for (const sql of chunk) await tx.runAsync(sql, [json]);
|
|
4792
5228
|
});
|
|
4793
|
-
if (done) return;
|
|
4794
5229
|
}
|
|
4795
5230
|
}
|
|
5231
|
+
/** One seq-ordered chunk. False when the ledger was empty. */
|
|
5232
|
+
async drainLedgerChunk() {
|
|
5233
|
+
const hiSql = drainHiSql(this.prefix);
|
|
5234
|
+
const chunk = drainChunkSql(this.prefix);
|
|
5235
|
+
let ran = false;
|
|
5236
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
5237
|
+
const row = await tx.getFirstAsync(hiSql);
|
|
5238
|
+
if (row?.hi == null) return;
|
|
5239
|
+
for (const sql of chunk) await tx.runAsync(sql, [row.hi]);
|
|
5240
|
+
ran = true;
|
|
5241
|
+
});
|
|
5242
|
+
return ran;
|
|
5243
|
+
}
|
|
5244
|
+
/** One entity-targeted chunk (ORDER BY seq, ≤ FTS5_DRAIN_CHUNK). */
|
|
5245
|
+
async drainEntityChunk(entityId) {
|
|
5246
|
+
const hiSql = entityDrainHiSql(this.prefix);
|
|
5247
|
+
const chunk = entityDrainChunkSql(this.prefix);
|
|
5248
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
5249
|
+
const row = await tx.getFirstAsync(hiSql, [entityId]);
|
|
5250
|
+
if (row?.hi == null) return;
|
|
5251
|
+
for (const sql of chunk) await tx.runAsync(sql, [entityId, row.hi]);
|
|
5252
|
+
});
|
|
5253
|
+
}
|
|
4796
5254
|
async replace(entityId, ids, documents) {
|
|
4797
5255
|
assertEntity(entityId, documents);
|
|
4798
5256
|
const docIds = documents.map((d) => d.id);
|
|
@@ -4870,20 +5328,57 @@ var Fts5IndexStrategy = class {
|
|
|
4870
5328
|
);
|
|
4871
5329
|
}
|
|
4872
5330
|
}
|
|
4873
|
-
|
|
5331
|
+
/** The init probes' schema facts in one sqlite_master read (#284 review): FTS tables with their DDL, and the triggers. */
|
|
5332
|
+
async readSchema() {
|
|
4874
5333
|
const p = this.prefix;
|
|
4875
|
-
|
|
4876
|
-
|
|
4877
|
-
}
|
|
4878
|
-
async triggersPresent() {
|
|
4879
|
-
const names = fts5TriggerNames(this.prefix);
|
|
5334
|
+
const triggers = fts5TriggerNames(p);
|
|
5335
|
+
const names = [`${p}entries_fts`, `${p}fts_map`, `${p}fts_pending`, ...triggers];
|
|
4880
5336
|
const rows = await this.db.getAllAsync(
|
|
4881
|
-
`SELECT name FROM sqlite_master WHERE
|
|
4882
|
-
|
|
5337
|
+
`SELECT type, name, sql FROM sqlite_master WHERE name IN (${names.map(() => "?").join(", ")})`,
|
|
5338
|
+
names
|
|
4883
5339
|
);
|
|
4884
|
-
|
|
5340
|
+
const tables = new Map(rows.filter((r) => r.type === "table").map((r) => [r.name, r.sql]));
|
|
5341
|
+
const present = new Set(rows.filter((r) => r.type === "trigger").map((r) => r.name));
|
|
5342
|
+
return { tables, triggersPresent: triggers.every((n) => present.has(n)) };
|
|
4885
5343
|
}
|
|
4886
5344
|
};
|
|
5345
|
+
var FtsOnlyStatementError = class extends Error {
|
|
5346
|
+
constructor(cause) {
|
|
5347
|
+
super("FTS-only statement failed");
|
|
5348
|
+
this.ftsOnlyCause = cause;
|
|
5349
|
+
}
|
|
5350
|
+
};
|
|
5351
|
+
function unwrapFtsOnly(err) {
|
|
5352
|
+
try {
|
|
5353
|
+
if (err instanceof FtsOnlyStatementError) return { cause: err.ftsOnlyCause, ftsOnly: true };
|
|
5354
|
+
} catch {
|
|
5355
|
+
}
|
|
5356
|
+
return { cause: err, ftsOnly: false };
|
|
5357
|
+
}
|
|
5358
|
+
function safeSqliteCode(err) {
|
|
5359
|
+
try {
|
|
5360
|
+
const code = extractSqliteCode(err);
|
|
5361
|
+
if (code !== void 0) return code;
|
|
5362
|
+
const lifted = err?.sqliteErrorCode;
|
|
5363
|
+
return typeof lifted === "string" ? lifted : void 0;
|
|
5364
|
+
} catch {
|
|
5365
|
+
return void 0;
|
|
5366
|
+
}
|
|
5367
|
+
}
|
|
5368
|
+
var TRANSIENT_OR_TX_FATAL = ["SQLITE_BUSY", "SQLITE_LOCKED", "SQLITE_FULL", "SQLITE_IOERR", "SQLITE_NOMEM", "SQLITE_INTERRUPT", "SQLITE_ABORT", "SQLITE_READONLY"];
|
|
5369
|
+
function isTransientOrTxFatal(err) {
|
|
5370
|
+
const code = safeSqliteCode(err);
|
|
5371
|
+
return code !== void 0 && TRANSIENT_OR_TX_FATAL.some((c) => code === c || code.startsWith(`${c}_`));
|
|
5372
|
+
}
|
|
5373
|
+
function ftsCorruptionClass(err) {
|
|
5374
|
+
const code = safeSqliteCode(err);
|
|
5375
|
+
if (code === "SQLITE_CORRUPT_VTAB") return "vtab";
|
|
5376
|
+
if (code?.startsWith("SQLITE_CORRUPT")) return "corrupt";
|
|
5377
|
+
return null;
|
|
5378
|
+
}
|
|
5379
|
+
function isSqliteError(err) {
|
|
5380
|
+
return safeSqliteCode(err) !== void 0;
|
|
5381
|
+
}
|
|
4887
5382
|
function assertEntity(entityId, documents) {
|
|
4888
5383
|
for (const d of documents) {
|
|
4889
5384
|
if (d.entity_id !== entityId) {
|
|
@@ -4941,15 +5436,15 @@ async function detachFts5(db, prefix, metadataRepo) {
|
|
|
4941
5436
|
} catch {
|
|
4942
5437
|
}
|
|
4943
5438
|
}
|
|
4944
|
-
async function createIndexStrategy(db, prefix, metadataRepo, preferred) {
|
|
5439
|
+
async function createIndexStrategy(db, prefix, metadataRepo, preferred, opts = {}) {
|
|
4945
5440
|
if (preferred === "minisearch") {
|
|
4946
5441
|
await detachFts5(db, prefix, metadataRepo);
|
|
4947
5442
|
return new MiniSearchIndexStrategy();
|
|
4948
5443
|
}
|
|
4949
5444
|
const attempt = async () => {
|
|
4950
5445
|
if (!await probeFts5(db, prefix)) return "absent";
|
|
4951
|
-
const strategy = new Fts5IndexStrategy(db, prefix, metadataRepo);
|
|
4952
|
-
await strategy.init();
|
|
5446
|
+
const strategy = new Fts5IndexStrategy(db, prefix, metadataRepo, opts.setupBudgetMs);
|
|
5447
|
+
await strategy.init({ entriesExisted: opts.entriesExisted });
|
|
4953
5448
|
return strategy;
|
|
4954
5449
|
};
|
|
4955
5450
|
const absent = async () => {
|
|
@@ -6312,7 +6807,8 @@ var IngestionService = class {
|
|
|
6312
6807
|
*
|
|
6313
6808
|
* @returns Counts: nodesWritten (validated nodes persisted), edgesWritten
|
|
6314
6809
|
* (manifest-valid edges persisted), superseded (prior facts soft-deleted
|
|
6315
|
-
* plus prior source-ref edges hard-deleted)
|
|
6810
|
+
* plus prior source-ref edges hard-deleted), and supersededFactIds (the
|
|
6811
|
+
* ids of those soft-deleted facts, for the caller's post-commit index sync).
|
|
6316
6812
|
*
|
|
6317
6813
|
* Node-ownership pre-flight (spec §3, REQ-GRAPH-01): the first statement
|
|
6318
6814
|
* is `assertGraphNodeOwnership(entityId, params.nodes, tx)`, which runs
|
|
@@ -6440,7 +6936,8 @@ var IngestionService = class {
|
|
|
6440
6936
|
return {
|
|
6441
6937
|
nodesWritten: wikiFacts.length,
|
|
6442
6938
|
edgesWritten: edgesInsertedCount,
|
|
6443
|
-
superseded: deletedFactIds.length + deletedEdgeCount
|
|
6939
|
+
superseded: deletedFactIds.length + deletedEdgeCount,
|
|
6940
|
+
supersededFactIds: deletedFactIds
|
|
6444
6941
|
};
|
|
6445
6942
|
}
|
|
6446
6943
|
/**
|
|
@@ -7724,6 +8221,7 @@ var MaintenanceService = class {
|
|
|
7724
8221
|
if (vacuum) {
|
|
7725
8222
|
await resetNormalizeWatermarks(this.metadataRepo);
|
|
7726
8223
|
await this.metadataRepo.vacuum();
|
|
8224
|
+
await restartRebuildAfterVacuum(this.db, this.prefix, this.metadataRepo);
|
|
7727
8225
|
}
|
|
7728
8226
|
await this.searchService.syncEntries(entityId, syncedIds);
|
|
7729
8227
|
return { entries: deletedEntries, tasks: deletedTasks, events: deletedEvents };
|
|
@@ -9167,7 +9665,7 @@ var ImportExportService = class {
|
|
|
9167
9665
|
if (merge) {
|
|
9168
9666
|
await this.searchService.syncEntries(entityId, [...upsertedFactIds]);
|
|
9169
9667
|
} else {
|
|
9170
|
-
await this.searchService.sync(entityId);
|
|
9668
|
+
await this.searchService.sync(entityId, { ids: [.../* @__PURE__ */ new Set([...upsertedFactIds, ...softDeletedFactIds])] });
|
|
9171
9669
|
}
|
|
9172
9670
|
for (const fact of bundle.facts) {
|
|
9173
9671
|
if (!fact.deleted_at && upsertedFactIds.has(fact.id) && !factsWithPreservedBlob.has(fact.id)) {
|
|
@@ -10971,7 +11469,6 @@ var TABLE_PREFIX_PATTERN = /^[A-Za-z][A-Za-z0-9_]{0,30}_$/;
|
|
|
10971
11469
|
var SEARCH_REINIT_COOLDOWN_MS = 5e3;
|
|
10972
11470
|
var SEARCH_REINIT_COOLDOWN_MAX_MS = 5 * 6e4;
|
|
10973
11471
|
var SEARCH_REINIT_COST_FACTOR = 10;
|
|
10974
|
-
var monotonicNow = () => typeof performance !== "undefined" && typeof performance.now === "function" ? performance.now() : Date.now();
|
|
10975
11472
|
function decodeDraftCursor(cursor) {
|
|
10976
11473
|
const sep = typeof cursor === "string" ? cursor.indexOf(":") : -1;
|
|
10977
11474
|
const createdAt = sep > 0 ? Number(cursor.slice(0, sep)) : Number.NaN;
|
|
@@ -10986,6 +11483,16 @@ var WikiMemory = class {
|
|
|
10986
11483
|
constructor(db, options) {
|
|
10987
11484
|
/** Emits `__testAccess` console warning at most once per instance when NODE_ENV ≠ "test". */
|
|
10988
11485
|
__privateAdd(this, _testAccessNonTestEnvWarned, false);
|
|
11486
|
+
/**
|
|
11487
|
+
* upsertGraph node and superseded-fact ids per entity, awaiting syncSearchIndex(), each with the
|
|
11488
|
+
* generation of its latest stash. Ids leave the stash only once a drain
|
|
11489
|
+
* succeeded, so a failed drain retries them next call.
|
|
11490
|
+
*/
|
|
11491
|
+
this.upsertIdStash = /* @__PURE__ */ new Map();
|
|
11492
|
+
/** Total ids across upsertIdStash, kept in step so the cap check is O(1). */
|
|
11493
|
+
this.upsertIdStashSize = 0;
|
|
11494
|
+
/** Bumped per stashed id; a re-stash during an in-flight drain outlives that drain's drop. */
|
|
11495
|
+
this.upsertIdStashGen = 0;
|
|
10989
11496
|
this.reinitSearch = null;
|
|
10990
11497
|
/** Monotonic time before which a no-arg syncSearchIndex() skips the re-init (0 = no cooldown). */
|
|
10991
11498
|
this.reinitRetryAt = 0;
|
|
@@ -11122,6 +11629,49 @@ var WikiMemory = class {
|
|
|
11122
11629
|
jobManager: this.jobManager
|
|
11123
11630
|
};
|
|
11124
11631
|
}
|
|
11632
|
+
stashUpsertIds(entityId, ids) {
|
|
11633
|
+
if (!this.searchService.hasLedger()) return;
|
|
11634
|
+
const cap = this.constructor.UPSERT_ID_STASH_CAP;
|
|
11635
|
+
const stash = this.upsertIdStash.get(entityId) ?? /* @__PURE__ */ new Map();
|
|
11636
|
+
for (const id of ids) {
|
|
11637
|
+
if (!stash.has(id)) {
|
|
11638
|
+
if (this.upsertIdStashSize >= cap) continue;
|
|
11639
|
+
this.upsertIdStashSize++;
|
|
11640
|
+
}
|
|
11641
|
+
stash.set(id, ++this.upsertIdStashGen);
|
|
11642
|
+
}
|
|
11643
|
+
if (stash.size > 0) this.upsertIdStash.set(entityId, stash);
|
|
11644
|
+
}
|
|
11645
|
+
/** Snapshot of the stash (one entity, or all) with generations, without removing anything. */
|
|
11646
|
+
peekStashedIds(entityId) {
|
|
11647
|
+
const snapshot = /* @__PURE__ */ new Map();
|
|
11648
|
+
for (const [id, stash] of this.upsertIdStash) {
|
|
11649
|
+
if (entityId === void 0 || id === entityId) snapshot.set(id, new Map(stash));
|
|
11650
|
+
}
|
|
11651
|
+
return snapshot;
|
|
11652
|
+
}
|
|
11653
|
+
/**
|
|
11654
|
+
* Removes a drained snapshot. An id re-stashed after the peek has a newer
|
|
11655
|
+
* generation and stays: the drain may have run before that write committed.
|
|
11656
|
+
*/
|
|
11657
|
+
dropStashedIds(snapshot) {
|
|
11658
|
+
for (const [entityId, drained] of snapshot) {
|
|
11659
|
+
const stash = this.upsertIdStash.get(entityId);
|
|
11660
|
+
if (!stash) continue;
|
|
11661
|
+
for (const [id, gen] of drained) {
|
|
11662
|
+
if (stash.get(id) === gen && stash.delete(id)) this.upsertIdStashSize--;
|
|
11663
|
+
}
|
|
11664
|
+
if (stash.size === 0) this.upsertIdStash.delete(entityId);
|
|
11665
|
+
}
|
|
11666
|
+
}
|
|
11667
|
+
/** `config.setupBudgetMs` or the default, validated: a positive finite number. */
|
|
11668
|
+
setupBudgetMs() {
|
|
11669
|
+
const budget = this.options.config?.setupBudgetMs ?? DEFAULT_SETUP_BUDGET_MS;
|
|
11670
|
+
if (typeof budget !== "number" || !Number.isFinite(budget) || budget <= 0) {
|
|
11671
|
+
throw new TypeError(`setupBudgetMs must be a positive finite number; got ${String(budget)}`);
|
|
11672
|
+
}
|
|
11673
|
+
return budget;
|
|
11674
|
+
}
|
|
11125
11675
|
/**
|
|
11126
11676
|
* Opens the store: migration gate, DDL, schema migrations, then the stamped
|
|
11127
11677
|
* source_ref normalize. Await it before any other call on this instance — the
|
|
@@ -11132,8 +11682,15 @@ var WikiMemory = class {
|
|
|
11132
11682
|
* Pass `{ migrations: 'refuse' }` from inside a writer lease: pending one-time
|
|
11133
11683
|
* work then throws WikiMigrationRequiredError before any mutation, and the
|
|
11134
11684
|
* host runs a plain `setup()` outside the lease.
|
|
11685
|
+
*
|
|
11686
|
+
* Keyword-index catch-up work is bounded by `config.setupBudgetMs`; anything
|
|
11687
|
+
* left converges on later opens and no-arg `syncSearchIndex()` calls. Not
|
|
11688
|
+
* bounded: an index reset (tokenizer change, re-attach after detach, a lost
|
|
11689
|
+
* or inconsistent index), which runs as one unchunked transaction.
|
|
11135
11690
|
*/
|
|
11136
11691
|
async setup(opts) {
|
|
11692
|
+
const setupBudgetMs = this.setupBudgetMs();
|
|
11693
|
+
const openDeadline = monotonicNow() + setupBudgetMs;
|
|
11137
11694
|
const entriesExistedBeforeSetup = await this.metadataRepo.tableExists(`${this.prefix}entries`);
|
|
11138
11695
|
const migrationPolicy = opts?.migrations ?? this.options.config?.migrations ?? "allow";
|
|
11139
11696
|
await setupDatabaseWithGate(this.db, this.prefix, this.metadataRepo, migrationPolicy);
|
|
@@ -11168,10 +11725,11 @@ var WikiMemory = class {
|
|
|
11168
11725
|
this.db,
|
|
11169
11726
|
this.prefix,
|
|
11170
11727
|
this.metadataRepo,
|
|
11171
|
-
this.options.config?.indexStrategy ?? "auto"
|
|
11728
|
+
this.options.config?.indexStrategy ?? "auto",
|
|
11729
|
+
{ setupBudgetMs, entriesExisted: entriesExistedBeforeSetup }
|
|
11172
11730
|
)
|
|
11173
11731
|
);
|
|
11174
|
-
await this.searchService.sync();
|
|
11732
|
+
await this.searchService.sync(void 0, { deadline: openDeadline });
|
|
11175
11733
|
}
|
|
11176
11734
|
async hasChanged(entityId, sourceRefOrEntries, sourceHashArg) {
|
|
11177
11735
|
if (typeof sourceRefOrEntries === "string") {
|
|
@@ -11422,14 +11980,18 @@ var WikiMemory = class {
|
|
|
11422
11980
|
* commit, so without this the nodes stay out of keyword, hybrid and
|
|
11423
11981
|
* fallback retrieval until the entity's next core write.
|
|
11424
11982
|
*
|
|
11425
|
-
* With `entityId
|
|
11426
|
-
*
|
|
11427
|
-
*
|
|
11428
|
-
*
|
|
11429
|
-
*
|
|
11430
|
-
*
|
|
11431
|
-
*
|
|
11432
|
-
*
|
|
11983
|
+
* With `entityId`: drains that entity's pending `upsertGraph` ids (its new
|
|
11984
|
+
* nodes and the facts they superseded) first, to completion, then at most
|
|
11985
|
+
* one more ledger chunk — so its new nodes are searchable, and the
|
|
11986
|
+
* superseded facts gone, when the call resolves, under any backlog. Without an
|
|
11987
|
+
* argument: drains every stashed `upsertGraph` id across entities first,
|
|
11988
|
+
* then runs one keyword-index quantum bounded by `config.setupBudgetMs`
|
|
11989
|
+
* (backlog drain, then rebuild work while the index is rebuilding). Hosts
|
|
11990
|
+
* with a long-lived process should call the no-arg form on their own
|
|
11991
|
+
* schedule: that is what converges a rebuilding index. While keyword search
|
|
11992
|
+
* is unavailable it first retries the index init; after a failed attempt it
|
|
11993
|
+
* waits at least 5 s, doubling per consecutive failure up to 5 min (longer
|
|
11994
|
+
* when an attempt is itself slow), before retrying again.
|
|
11433
11995
|
*
|
|
11434
11996
|
* Call only after the host's transaction commits: called inside it, the
|
|
11435
11997
|
* rebuild may index rows that later roll back, or miss them and clear the
|
|
@@ -11455,10 +12017,11 @@ var WikiMemory = class {
|
|
|
11455
12017
|
this.db,
|
|
11456
12018
|
this.prefix,
|
|
11457
12019
|
this.metadataRepo,
|
|
11458
|
-
this.options.config?.indexStrategy ?? "auto"
|
|
12020
|
+
this.options.config?.indexStrategy ?? "auto",
|
|
12021
|
+
{ setupBudgetMs: this.setupBudgetMs() }
|
|
11459
12022
|
);
|
|
11460
12023
|
this.searchService.setIndexStrategy(strategy);
|
|
11461
|
-
if (this.searchService.isKeywordAvailable()) await this.searchService.sync();
|
|
12024
|
+
if (this.searchService.isKeywordAvailable() && !this.searchService.hasLedger()) await this.searchService.sync();
|
|
11462
12025
|
} finally {
|
|
11463
12026
|
if (this.searchService.isKeywordAvailable()) {
|
|
11464
12027
|
this.reinitFailures = 0;
|
|
@@ -11474,10 +12037,24 @@ var WikiMemory = class {
|
|
|
11474
12037
|
})());
|
|
11475
12038
|
await this.reinitSearch;
|
|
11476
12039
|
}
|
|
11477
|
-
|
|
12040
|
+
const stashed2 = this.peekStashedIds();
|
|
12041
|
+
const ids = [...stashed2.values()].flatMap((m) => [...m.keys()]);
|
|
12042
|
+
if (await this.syncDrained(this.searchService.syncStale({ ids }))) this.dropStashedIds(stashed2);
|
|
12043
|
+
return;
|
|
11478
12044
|
}
|
|
11479
12045
|
assertEntityId(entityId);
|
|
11480
|
-
|
|
12046
|
+
const stashed = this.peekStashedIds(entityId);
|
|
12047
|
+
if (await this.syncDrained(this.searchService.syncEntries(entityId, stashed.get(entityId)?.keys() ?? []))) {
|
|
12048
|
+
this.dropStashedIds(stashed);
|
|
12049
|
+
}
|
|
12050
|
+
}
|
|
12051
|
+
/**
|
|
12052
|
+
* True when a sync applied the stashed ids. While keyword search is
|
|
12053
|
+
* unavailable the stand-in strategy's drain is a no-op that reports success,
|
|
12054
|
+
* so the ids stay stashed for the first sync after a re-init (#284 review).
|
|
12055
|
+
*/
|
|
12056
|
+
async syncDrained(sync) {
|
|
12057
|
+
return await sync && this.searchService.isKeywordAvailable();
|
|
11481
12058
|
}
|
|
11482
12059
|
async exportDump(entityIds) {
|
|
11483
12060
|
return this.importExportService.exportDump(entityIds);
|
|
@@ -11528,7 +12105,10 @@ var WikiMemory = class {
|
|
|
11528
12105
|
* post-commit work (search sync, embedding, cache eviction). After the
|
|
11529
12106
|
* transaction commits, call `syncSearchIndex(entityId)` — or
|
|
11530
12107
|
* `syncSearchIndex()` after writing several entities in one transaction —
|
|
11531
|
-
* to make the new nodes keyword-searchable
|
|
12108
|
+
* to make the new nodes keyword-searchable: both forms drain this call's node ids
|
|
12109
|
+
* first, to completion, so the nodes are searchable under any backlog (up to
|
|
12110
|
+
* `WikiMemory.UPSERT_ID_STASH_CAP` pending ids; beyond that they converge with
|
|
12111
|
+
* the backlog). Embeddings still come from the maintenance
|
|
11532
12112
|
* sweep (`runLibrarian` / `runHeal` / `runOntologyBackfill` / `runPrune`,
|
|
11533
12113
|
* scoped via `listEntityIds`).
|
|
11534
12114
|
*
|
|
@@ -11578,12 +12158,13 @@ var WikiMemory = class {
|
|
|
11578
12158
|
});
|
|
11579
12159
|
}
|
|
11580
12160
|
const diagBuffer = new DiagnosticBuffer();
|
|
11581
|
-
const result = await this.ingestionService.upsertGraphCore(
|
|
12161
|
+
const { supersededFactIds, ...result } = await this.ingestionService.upsertGraphCore(
|
|
11582
12162
|
entityId,
|
|
11583
12163
|
{ sourceRef, sourceHash, nodes: params.nodes, edges: params.edges },
|
|
11584
12164
|
adapter,
|
|
11585
12165
|
{ diag: { buffer: diagBuffer, operation: "upsertGraph" } }
|
|
11586
12166
|
);
|
|
12167
|
+
this.stashUpsertIds(entityId, [...supersededFactIds, ...params.nodes.map((n) => n.id)]);
|
|
11587
12168
|
this.searchService.markStale(entityId);
|
|
11588
12169
|
diagBuffer.flush(this.options);
|
|
11589
12170
|
return result;
|
|
@@ -11766,6 +12347,8 @@ var WikiMemory = class {
|
|
|
11766
12347
|
}
|
|
11767
12348
|
};
|
|
11768
12349
|
_testAccessNonTestEnvWarned = new WeakMap();
|
|
12350
|
+
/** Max upsertGraph node ids held for their post-commit sync (spec #280 id threading). */
|
|
12351
|
+
WikiMemory.UPSERT_ID_STASH_CAP = 1e4;
|
|
11769
12352
|
function assertEntityId(entityId) {
|
|
11770
12353
|
if (typeof entityId !== "string" || entityId.length === 0) {
|
|
11771
12354
|
throw new TypeError("Invalid entityId: must be a non-empty string.");
|
|
@@ -12695,6 +13278,7 @@ exports.DEFAULT_CHUNK_OVERLAP = DEFAULT_CHUNK_OVERLAP;
|
|
|
12695
13278
|
exports.DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT = DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT;
|
|
12696
13279
|
exports.DEFAULT_MAX_CHUNK_LENGTH = DEFAULT_MAX_CHUNK_LENGTH;
|
|
12697
13280
|
exports.DEFAULT_MAX_EMBED_CHARS = DEFAULT_MAX_EMBED_CHARS;
|
|
13281
|
+
exports.DEFAULT_SETUP_BUDGET_MS = DEFAULT_SETUP_BUDGET_MS;
|
|
12698
13282
|
exports.EMBED_CHARS_CEILING = EMBED_CHARS_CEILING;
|
|
12699
13283
|
exports.HEAL_ANCHORS_PER_CANDIDATE = HEAL_ANCHORS_PER_CANDIDATE;
|
|
12700
13284
|
exports.HEAL_BATCH_SIZE = HEAL_BATCH_SIZE;
|