@equationalapplications/core-llm-wiki 7.12.0 → 7.13.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -75,6 +75,8 @@ var SQLITE_RESULT_CODE_NAMES = {
75
75
  // prefix and we normalize it back to the symbolic name so callers can branch on
76
76
  // the same string the rest of the ecosystem uses.
77
77
  2067: "SQLITE_CONSTRAINT_UNIQUE",
78
+ // 267 = SQLITE_CORRUPT_VTAB — a corrupt FTS5 shadow table (ftsCorruptionClass).
79
+ 267: "SQLITE_CORRUPT_VTAB",
78
80
  100: "SQLITE_ROW",
79
81
  101: "SQLITE_DONE"
80
82
  };
@@ -935,6 +937,143 @@ async function createTemporalIndexesIfColumnsExist(db, prefix) {
935
937
  }
936
938
  }
937
939
 
940
+ // src/services/search/fts5Sql.ts
941
+ var FTS5_STATE_KEY = "fts5_index_state";
942
+ var FTS5_DRAIN_CHUNK = 500;
943
+ var FTS5_SWEEP_CHUNK = 500;
944
+ var FTS5_RETIRED_TABLES_KEY = "fts5_retired_tables";
945
+ var FTS5_RETIRED_COUNTER_KEY = "fts5_retired_counter";
946
+ var FTS5_RECLAIM_MERGE_PAGES = 500;
947
+ function retiredTableName(p, n) {
948
+ return `${p}entries_fts_retired_${n}`;
949
+ }
950
+ function isRetiredTableName(p, name) {
951
+ return typeof name === "string" && name.startsWith(`${p}entries_fts_retired_`) && /^\d+$/.test(name.slice(`${p}entries_fts_retired_`.length));
952
+ }
953
+ var FTS5_REBUILD_WM_KEY = "fts5_rebuild_wm";
954
+ var FTS5_REBUILD_SNAPSHOT_KEY = "fts5_rebuild_snapshot";
955
+ var FTS5_SWEEP_CURSOR_KEY = "fts5_orphan_sweep_cursor";
956
+ function fts5TriggerNames(prefix) {
957
+ return [`${prefix}entries_fts_ai`, `${prefix}entries_fts_au`, `${prefix}entries_fts_ad`];
958
+ }
959
+ function tagsExpr(col) {
960
+ return `CASE WHEN json_valid(${col}) THEN
961
+ CASE WHEN json_type(${col}) = 'array' THEN
962
+ (SELECT group_concat(CASE type WHEN 'true' THEN 'true' WHEN 'false' THEN 'false'
963
+ WHEN 'null' THEN '' ELSE value END, ' ')
964
+ FROM json_each(${col}))
965
+ ELSE ${col} END
966
+ ELSE ${col} END`;
967
+ }
968
+ function fts5TablesDdl(p) {
969
+ return `
970
+ CREATE VIRTUAL TABLE IF NOT EXISTS ${p}entries_fts USING fts5(
971
+ id UNINDEXED, entity_id UNINDEXED, title, body, tags,
972
+ tokenize = 'porter unicode61'
973
+ );
974
+ CREATE TABLE IF NOT EXISTS ${p}fts_map (
975
+ fts_rowid INTEGER PRIMARY KEY,
976
+ id TEXT NOT NULL UNIQUE,
977
+ entity_id TEXT NOT NULL
978
+ );
979
+ CREATE INDEX IF NOT EXISTS ${p}fts_map_entity ON ${p}fts_map(entity_id);
980
+ CREATE TABLE IF NOT EXISTS ${p}fts_pending (
981
+ seq INTEGER PRIMARY KEY,
982
+ entity_id TEXT NOT NULL,
983
+ id TEXT NOT NULL
984
+ );`;
985
+ }
986
+ function fts5TriggersDdl(p) {
987
+ const [ai, au, ad] = fts5TriggerNames(p);
988
+ return `
989
+ CREATE TRIGGER IF NOT EXISTS ${ai} AFTER INSERT ON ${p}entries BEGIN
990
+ INSERT INTO ${p}fts_pending (entity_id, id) VALUES (NEW.entity_id, NEW.id);
991
+ END;
992
+ CREATE TRIGGER IF NOT EXISTS ${au}
993
+ AFTER UPDATE OF id, entity_id, title, body, tags, deleted_at ON ${p}entries BEGIN
994
+ INSERT INTO ${p}fts_pending (entity_id, id) VALUES (OLD.entity_id, OLD.id);
995
+ INSERT INTO ${p}fts_pending (entity_id, id)
996
+ SELECT NEW.entity_id, NEW.id WHERE NEW.id <> OLD.id OR NEW.entity_id <> OLD.entity_id;
997
+ END;
998
+ CREATE TRIGGER IF NOT EXISTS ${ad} AFTER DELETE ON ${p}entries BEGIN
999
+ INSERT INTO ${p}fts_pending (entity_id, id) VALUES (OLD.entity_id, OLD.id);
1000
+ END;`;
1001
+ }
1002
+ function drainHiSql(p) {
1003
+ return `SELECT max(seq) AS hi FROM (SELECT seq FROM ${p}fts_pending ORDER BY seq LIMIT ${FTS5_DRAIN_CHUNK})`;
1004
+ }
1005
+ function ledgerChunkSql(p, chunkIds, ledgerWhere) {
1006
+ return [
1007
+ `DELETE FROM ${p}entries_fts WHERE rowid IN (SELECT m.fts_rowid FROM ${p}fts_map m WHERE m.id IN (${chunkIds}))`,
1008
+ `DELETE FROM ${p}fts_map WHERE id IN (${chunkIds})`,
1009
+ `INSERT INTO ${p}fts_map (id, entity_id)
1010
+ SELECT e.id, e.entity_id FROM ${p}entries e WHERE e.id IN (${chunkIds}) AND e.deleted_at IS NULL`,
1011
+ `INSERT INTO ${p}entries_fts (rowid, id, entity_id, title, body, tags)
1012
+ SELECT m.fts_rowid, e.id, e.entity_id, e.title, e.body, ${tagsExpr("e.tags")}
1013
+ FROM ${p}fts_map m JOIN ${p}entries e ON e.id = m.id
1014
+ WHERE m.id IN (${chunkIds})`,
1015
+ `DELETE FROM ${p}fts_pending WHERE ${ledgerWhere}`
1016
+ ];
1017
+ }
1018
+ function drainChunkSql(p) {
1019
+ return ledgerChunkSql(p, `SELECT id FROM ${p}fts_pending WHERE seq <= ?`, "seq <= ?");
1020
+ }
1021
+ function fts5PendingIndexesDdl(p) {
1022
+ return `
1023
+ CREATE INDEX IF NOT EXISTS ${p}fts_pending_id ON ${p}fts_pending(id);
1024
+ CREATE INDEX IF NOT EXISTS ${p}fts_pending_entity ON ${p}fts_pending(entity_id);`;
1025
+ }
1026
+ function idDrainChunkSql(p) {
1027
+ return ledgerChunkSql(
1028
+ p,
1029
+ `SELECT id FROM ${p}fts_pending WHERE id IN (SELECT value FROM json_each(?))`,
1030
+ "id IN (SELECT value FROM json_each(?))"
1031
+ );
1032
+ }
1033
+ function idReindexChunkSql(p) {
1034
+ return ledgerChunkSql(p, "SELECT value FROM json_each(?)", "id IN (SELECT value FROM json_each(?))");
1035
+ }
1036
+ function entityDrainHiSql(p) {
1037
+ return `SELECT max(seq) AS hi FROM (SELECT seq FROM ${p}fts_pending WHERE entity_id = ? ORDER BY seq LIMIT ${FTS5_DRAIN_CHUNK})`;
1038
+ }
1039
+ function entityDrainChunkSql(p) {
1040
+ return ledgerChunkSql(p, `SELECT id FROM ${p}fts_pending WHERE entity_id = ? AND seq <= ?`, "entity_id = ? AND seq <= ?");
1041
+ }
1042
+ function fts5LedgerDropSql(p) {
1043
+ return [
1044
+ ...fts5TriggerNames(p).map((n) => `DROP TRIGGER IF EXISTS ${n};`),
1045
+ `DROP TABLE IF EXISTS ${p}fts_pending;`,
1046
+ `DROP TABLE IF EXISTS ${p}fts_map;`
1047
+ ].join("\n");
1048
+ }
1049
+ function windowHiSql(p) {
1050
+ return `SELECT max(rowid) AS hi FROM (SELECT rowid FROM ${p}entries WHERE rowid > ? AND rowid <= ? ORDER BY rowid LIMIT ${FTS5_DRAIN_CHUNK})`;
1051
+ }
1052
+ function windowChunkSql(p) {
1053
+ const chunkIds = `SELECT id FROM ${p}entries WHERE rowid > ? AND rowid <= ?`;
1054
+ return [
1055
+ `DELETE FROM ${p}entries_fts WHERE rowid IN (SELECT m.fts_rowid FROM ${p}fts_map m WHERE m.id IN (${chunkIds}))`,
1056
+ `DELETE FROM ${p}fts_map WHERE id IN (${chunkIds})`,
1057
+ `INSERT INTO ${p}fts_map (id, entity_id)
1058
+ SELECT e.id, e.entity_id FROM ${p}entries e WHERE e.rowid > ? AND e.rowid <= ? AND e.deleted_at IS NULL`,
1059
+ `INSERT INTO ${p}entries_fts (rowid, id, entity_id, title, body, tags)
1060
+ SELECT m.fts_rowid, e.id, e.entity_id, e.title, e.body, ${tagsExpr("e.tags")}
1061
+ FROM ${p}entries e JOIN ${p}fts_map m ON m.id = e.id
1062
+ WHERE e.rowid > ? AND e.rowid <= ?`
1063
+ ];
1064
+ }
1065
+ function sweepHiSql(p) {
1066
+ return `SELECT max(fts_rowid) AS hi FROM (SELECT fts_rowid FROM ${p}fts_map WHERE fts_rowid > ? AND fts_rowid <= ? ORDER BY fts_rowid LIMIT ${FTS5_SWEEP_CHUNK})`;
1067
+ }
1068
+ function sweepChunkSql(p) {
1069
+ const orphans = `SELECT m.fts_rowid FROM ${p}fts_map m
1070
+ WHERE m.fts_rowid > ? AND m.fts_rowid <= ? AND NOT EXISTS (SELECT 1 FROM ${p}entries e WHERE e.id = m.id)`;
1071
+ return [
1072
+ `DELETE FROM ${p}entries_fts WHERE rowid IN (${orphans})`,
1073
+ `DELETE FROM ${p}fts_map WHERE fts_rowid IN (${orphans})`
1074
+ ];
1075
+ }
1076
+
938
1077
  // src/db/migrations.ts
939
1078
  var MIGRATIONS = [
940
1079
  {
@@ -1215,6 +1354,17 @@ var MIGRATIONS = [
1215
1354
  run: async (db, prefix) => {
1216
1355
  await createLegacySourceTypeIndex(db, prefix);
1217
1356
  }
1357
+ },
1358
+ {
1359
+ version: 15,
1360
+ description: "fts_pending(id) and fts_pending(entity_id) indexes for the ids-first and entity-targeted drains",
1361
+ run: async (db, prefix) => {
1362
+ const row = await db.getFirstAsync(
1363
+ `SELECT count(*) AS n FROM sqlite_master WHERE type = 'table' AND name = ?`,
1364
+ [`${prefix}fts_pending`]
1365
+ );
1366
+ if (row?.n) await db.execAsync(fts5PendingIndexesDdl(prefix));
1367
+ }
1218
1368
  }
1219
1369
  ];
1220
1370
  async function createLegacySourceTypeIndex(db, prefix) {
@@ -1400,6 +1550,44 @@ var BaseRepository = class {
1400
1550
  }
1401
1551
  };
1402
1552
 
1553
+ // src/services/search/fts5Progress.ts
1554
+ async function writeRebuildStart(tx, prefix, meta) {
1555
+ const e = await tx.getFirstAsync(`SELECT coalesce(max(rowid), 0) AS hi FROM ${prefix}entries`);
1556
+ const m = await tx.getFirstAsync(`SELECT coalesce(max(fts_rowid), 0) AS hi FROM ${prefix}fts_map`);
1557
+ await meta.setMeta(FTS5_REBUILD_WM_KEY, JSON.stringify({ rowid: 0, id: null }), tx);
1558
+ await meta.setMeta(FTS5_REBUILD_SNAPSHOT_KEY, JSON.stringify({ entries: e?.hi ?? 0, map: m?.hi ?? 0 }), tx);
1559
+ await meta.setMeta(FTS5_SWEEP_CURSOR_KEY, "0", tx);
1560
+ }
1561
+ async function readRebuildProgress(meta, tx) {
1562
+ const wm = parseJsonObject(await meta.getMeta(FTS5_REBUILD_WM_KEY, tx));
1563
+ const snap = parseJsonObject(await meta.getMeta(FTS5_REBUILD_SNAPSHOT_KEY, tx));
1564
+ const rawCursor = await meta.getMeta(FTS5_SWEEP_CURSOR_KEY, tx);
1565
+ const cursor = rawCursor === null ? Number.NaN : Number(rawCursor);
1566
+ if (typeof wm?.rowid !== "number" || !(typeof wm.id === "string" || wm.id === null)) return null;
1567
+ if (typeof snap?.entries !== "number" || typeof snap.map !== "number" || !Number.isFinite(cursor)) return null;
1568
+ return { wm: { rowid: wm.rowid, id: wm.id }, snapshot: { entries: snap.entries, map: snap.map }, cursor };
1569
+ }
1570
+ async function reanchorRebuildFingerprint(tx, prefix, deletedIds) {
1571
+ if (deletedIds.length === 0) return;
1572
+ const metaValue = async (key) => (await tx.getFirstAsync(`SELECT value FROM ${prefix}meta WHERE key = ?`, [key]))?.value ?? null;
1573
+ if (await metaValue(FTS5_STATE_KEY) !== "rebuilding") return;
1574
+ const wm = parseJsonObject(await metaValue(FTS5_REBUILD_WM_KEY));
1575
+ if (typeof wm?.rowid !== "number" || typeof wm.id !== "string" || !deletedIds.includes(wm.id)) return;
1576
+ if ((await tx.getFirstAsync(`SELECT id FROM ${prefix}entries WHERE rowid = ?`, [wm.rowid]))?.id === wm.id) return;
1577
+ const below = await tx.getFirstAsync(
1578
+ `SELECT rowid, id FROM ${prefix}entries WHERE rowid < ? ORDER BY rowid DESC LIMIT 1`,
1579
+ [wm.rowid]
1580
+ );
1581
+ const next = below ? { rowid: below.rowid, id: below.id } : { rowid: 0, id: null };
1582
+ await tx.runAsync(`UPDATE ${prefix}meta SET value = ? WHERE key = ?`, [JSON.stringify(next), FTS5_REBUILD_WM_KEY]);
1583
+ }
1584
+ async function restartRebuildAfterVacuum(db, prefix, meta) {
1585
+ await db.withTransactionAsync(async (tx) => {
1586
+ if (await meta.getMeta(FTS5_STATE_KEY, tx) !== "rebuilding") return;
1587
+ await writeRebuildStart(tx, prefix, meta);
1588
+ });
1589
+ }
1590
+
1403
1591
  // src/utils/temporal.ts
1404
1592
  function liveAtSql(mode, alias = "") {
1405
1593
  const from = mode === "asOf" ? `COALESCE(${alias}valid_from, ${alias}created_at) <= ?` : `(${alias}valid_from IS NULL OR ${alias}valid_from <= ?)`;
@@ -2047,6 +2235,7 @@ var _EntryRepository = class _EntryRepository extends BaseRepository {
2047
2235
  }
2048
2236
  }
2049
2237
  }
2238
+ if (totalChanges > 0) await reanchorRebuildFingerprint(executor, this.prefix, ids);
2050
2239
  return totalChanges;
2051
2240
  }
2052
2241
  /**
@@ -4153,7 +4342,7 @@ var UnavailableIndexStrategy = class {
4153
4342
  hasIndexedEntity(_entityId) {
4154
4343
  return false;
4155
4344
  }
4156
- async drain() {
4345
+ async drain(_opts) {
4157
4346
  }
4158
4347
  };
4159
4348
 
@@ -4199,24 +4388,31 @@ var _SearchService = class _SearchService {
4199
4388
  isKeywordAvailable() {
4200
4389
  return !(this.indexStrategy instanceof UnavailableIndexStrategy);
4201
4390
  }
4391
+ /** True when the installed strategy drains a durable ledger (FTS5, or the degraded stand-in). */
4392
+ hasLedger() {
4393
+ return this.indexStrategy.drain !== void 0;
4394
+ }
4202
4395
  /**
4203
4396
  * One serialized drain turn for strategies with a durable ledger. Like
4204
4397
  * sync(), it never rejects: the keyword index is rebuildable, and a failed
4205
- * drain leaves the ledger rows in place for the next turn.
4398
+ * drain leaves the ledger rows in place for the next turn. Resolves false
4399
+ * when the drain failed.
4206
4400
  */
4207
- drainTurn(entityId) {
4401
+ drainTurn(entityId, opts) {
4208
4402
  const work = this.syncChain.then(async () => {
4209
4403
  try {
4210
4404
  try {
4211
- await this.indexStrategy.drain();
4405
+ await this.indexStrategy.drain(opts);
4212
4406
  } finally {
4213
4407
  this.evictCache(entityId);
4214
4408
  }
4409
+ return true;
4215
4410
  } catch (err) {
4216
4411
  console.warn(`[WikiMemory] search index drain failed for ${entityId ?? "*"}:`, err);
4412
+ return false;
4217
4413
  }
4218
4414
  });
4219
- this.syncChain = work;
4415
+ this.syncChain = work.then(() => void 0);
4220
4416
  return work;
4221
4417
  }
4222
4418
  /**
@@ -4226,9 +4422,18 @@ var _SearchService = class _SearchService {
4226
4422
  * Rebuilds are serialized per instance and never reject: the keyword index
4227
4423
  * is a rebuildable cache over SQLite, so degraded keyword search is the
4228
4424
  * correct failure mode and killing the host process is not.
4229
- */
4230
- async sync(entityId) {
4231
- if (this.indexStrategy.drain) return this.drainTurn(entityId);
4425
+ *
4426
+ * With a ledger strategy, `sync()` runs one full quantum bounded by
4427
+ * `opts.deadline`; `sync(entityId)` is a write turn (ledger only).
4428
+ * `opts.ids` are the caller's touched ids: a ledger strategy drains them
4429
+ * first, to completion; MiniSearch ignores them and rebuilds the entity in
4430
+ * one scan.
4431
+ */
4432
+ async sync(entityId, opts) {
4433
+ if (this.indexStrategy.drain) {
4434
+ await this.drainTurn(entityId, entityId === void 0 ? { mode: "full", deadline: opts?.deadline, ids: opts?.ids } : { mode: "ledger", entityId, ids: opts?.ids });
4435
+ return;
4436
+ }
4232
4437
  const work = this.syncChain.then(async () => {
4233
4438
  try {
4234
4439
  try {
@@ -4269,13 +4474,17 @@ var _SearchService = class _SearchService {
4269
4474
  * — including on an empty `ids` list, which is a no-op only for an entity
4270
4475
  * the index already tracks, and even then one that waits for rebuilds
4271
4476
  * already queued on the chain.
4477
+ *
4478
+ * Resolves false only when a ledger drain failed: the caller's ids were not
4479
+ * applied (their ledger rows remain), so a caller holding them retries.
4272
4480
  */
4273
4481
  async syncEntries(entityId, ids) {
4274
- if (this.indexStrategy.drain) return this.drainTurn(entityId);
4482
+ if (this.indexStrategy.drain) return this.drainTurn(entityId, { mode: "ledger", entityId, ids: [...new Set(ids)] });
4275
4483
  const uniqueIds = [...new Set(ids)];
4276
4484
  const indexed = this.indexStrategy.hasIndexedEntity?.(entityId);
4277
4485
  if (uniqueIds.length === 0 && !this.staleEntities.has(entityId) && indexed === true) {
4278
- return this.syncChain;
4486
+ await this.syncChain;
4487
+ return true;
4279
4488
  }
4280
4489
  const work = this.syncChain.then(async () => {
4281
4490
  try {
@@ -4304,7 +4513,8 @@ var _SearchService = class _SearchService {
4304
4513
  }
4305
4514
  });
4306
4515
  this.syncChain = work;
4307
- return work;
4516
+ await work;
4517
+ return true;
4308
4518
  }
4309
4519
  /**
4310
4520
  * Forces the entity's next syncEntries to rebuild it in full. For writes core
@@ -4320,11 +4530,13 @@ var _SearchService = class _SearchService {
4320
4530
  * so only those entities are rebuilt. Each rebuild still takes its own turn
4321
4531
  * on the sync chain, and like syncEntries this never rejects. With nothing
4322
4532
  * stale it resolves at once, without waiting on work already on the chain.
4533
+ * Resolves false only when a ledger drain failed (see syncEntries).
4323
4534
  */
4324
- async syncStale() {
4325
- if (this.indexStrategy.drain) return this.drainTurn();
4535
+ async syncStale(opts) {
4536
+ if (this.indexStrategy.drain) return this.drainTurn(void 0, { mode: "full", ids: opts?.ids });
4326
4537
  const ids = [...this.staleEntities];
4327
4538
  await Promise.all(ids.map((id) => this.syncEntries(id, [])));
4539
+ return true;
4328
4540
  }
4329
4541
  /**
4330
4542
  * Clears the parsed vector cache. Useful for mid-loop flush guarantees
@@ -4665,87 +4877,11 @@ function assertOwnedBy(entityId, documents) {
4665
4877
  }
4666
4878
  }
4667
4879
 
4668
- // src/services/search/fts5Sql.ts
4669
- var FTS5_STATE_KEY = "fts5_index_state";
4670
- var FTS5_DRAIN_CHUNK = 500;
4671
- function fts5TriggerNames(prefix) {
4672
- return [`${prefix}entries_fts_ai`, `${prefix}entries_fts_au`, `${prefix}entries_fts_ad`];
4673
- }
4674
- function tagsExpr(col) {
4675
- return `CASE WHEN json_valid(${col}) THEN
4676
- CASE WHEN json_type(${col}) = 'array' THEN
4677
- (SELECT group_concat(CASE type WHEN 'true' THEN 'true' WHEN 'false' THEN 'false'
4678
- WHEN 'null' THEN '' ELSE value END, ' ')
4679
- FROM json_each(${col}))
4680
- ELSE ${col} END
4681
- ELSE ${col} END`;
4682
- }
4683
- function fts5TablesDdl(p) {
4684
- return `
4685
- CREATE VIRTUAL TABLE IF NOT EXISTS ${p}entries_fts USING fts5(
4686
- id UNINDEXED, entity_id UNINDEXED, title, body, tags,
4687
- tokenize = 'porter unicode61'
4688
- );
4689
- CREATE TABLE IF NOT EXISTS ${p}fts_map (
4690
- fts_rowid INTEGER PRIMARY KEY,
4691
- id TEXT NOT NULL UNIQUE,
4692
- entity_id TEXT NOT NULL
4693
- );
4694
- CREATE INDEX IF NOT EXISTS ${p}fts_map_entity ON ${p}fts_map(entity_id);
4695
- CREATE TABLE IF NOT EXISTS ${p}fts_pending (
4696
- seq INTEGER PRIMARY KEY,
4697
- entity_id TEXT NOT NULL,
4698
- id TEXT NOT NULL
4699
- );`;
4700
- }
4701
- function fts5TriggersDdl(p) {
4702
- const [ai, au, ad] = fts5TriggerNames(p);
4703
- return `
4704
- CREATE TRIGGER IF NOT EXISTS ${ai} AFTER INSERT ON ${p}entries BEGIN
4705
- INSERT INTO ${p}fts_pending (entity_id, id) VALUES (NEW.entity_id, NEW.id);
4706
- END;
4707
- CREATE TRIGGER IF NOT EXISTS ${au}
4708
- AFTER UPDATE OF id, entity_id, title, body, tags, deleted_at ON ${p}entries BEGIN
4709
- INSERT INTO ${p}fts_pending (entity_id, id) VALUES (OLD.entity_id, OLD.id);
4710
- INSERT INTO ${p}fts_pending (entity_id, id)
4711
- SELECT NEW.entity_id, NEW.id WHERE NEW.id <> OLD.id OR NEW.entity_id <> OLD.entity_id;
4712
- END;
4713
- CREATE TRIGGER IF NOT EXISTS ${ad} AFTER DELETE ON ${p}entries BEGIN
4714
- INSERT INTO ${p}fts_pending (entity_id, id) VALUES (OLD.entity_id, OLD.id);
4715
- END;`;
4716
- }
4717
- function drainHiSql(p) {
4718
- return `SELECT max(seq) AS hi FROM (SELECT seq FROM ${p}fts_pending ORDER BY seq LIMIT ${FTS5_DRAIN_CHUNK})`;
4719
- }
4720
- function drainChunkSql(p) {
4721
- const chunkIds = `SELECT id FROM ${p}fts_pending WHERE seq <= ?`;
4722
- return [
4723
- `DELETE FROM ${p}entries_fts WHERE rowid IN (SELECT m.fts_rowid FROM ${p}fts_map m WHERE m.id IN (${chunkIds}))`,
4724
- `DELETE FROM ${p}fts_map WHERE id IN (${chunkIds})`,
4725
- `INSERT INTO ${p}fts_map (id, entity_id)
4726
- SELECT e.id, e.entity_id FROM ${p}entries e WHERE e.id IN (${chunkIds}) AND e.deleted_at IS NULL`,
4727
- `INSERT INTO ${p}entries_fts (rowid, id, entity_id, title, body, tags)
4728
- SELECT m.fts_rowid, e.id, e.entity_id, e.title, e.body, ${tagsExpr("e.tags")}
4729
- FROM ${p}fts_map m JOIN ${p}entries e ON e.id = m.id
4730
- WHERE m.id IN (${chunkIds})`,
4731
- `DELETE FROM ${p}fts_pending WHERE seq <= ?`
4732
- ];
4733
- }
4734
- function rebuildSql(p) {
4735
- return [
4736
- `INSERT INTO ${p}fts_map (id, entity_id) SELECT id, entity_id FROM ${p}entries WHERE deleted_at IS NULL`,
4737
- `INSERT INTO ${p}entries_fts (rowid, id, entity_id, title, body, tags)
4738
- SELECT m.fts_rowid, e.id, e.entity_id, e.title, e.body, ${tagsExpr("e.tags")}
4739
- FROM ${p}fts_map m JOIN ${p}entries e ON e.id = m.id`
4740
- ];
4741
- }
4742
- function fts5LedgerDropSql(p) {
4743
- return [
4744
- ...fts5TriggerNames(p).map((n) => `DROP TRIGGER IF EXISTS ${n};`),
4745
- `DROP TABLE IF EXISTS ${p}fts_pending;`,
4746
- `DROP TABLE IF EXISTS ${p}fts_map;`
4747
- ].join("\n");
4748
- }
4880
+ // src/utils/monotonicNow.ts
4881
+ var monotonicNow = () => typeof performance !== "undefined" && typeof performance.now === "function" ? performance.now() : Date.now();
4882
+
4883
+ // src/utils/setupDefaults.ts
4884
+ var DEFAULT_SETUP_BUDGET_MS = 1500;
4749
4885
 
4750
4886
  // src/services/search/ftsQuery.ts
4751
4887
  var TOKEN = /[\p{L}\p{N}]+/gu;
@@ -4758,41 +4894,363 @@ function buildFtsMatchQuery(query) {
4758
4894
 
4759
4895
  // src/services/search/Fts5IndexStrategy.ts
4760
4896
  var Fts5IndexStrategy = class {
4761
- constructor(db, prefix, metadataRepo) {
4897
+ constructor(db, prefix, metadataRepo, budgetMs = DEFAULT_SETUP_BUDGET_MS) {
4762
4898
  this.db = db;
4763
4899
  this.prefix = prefix;
4764
4900
  this.metadataRepo = metadataRepo;
4901
+ this.budgetMs = budgetMs;
4765
4902
  }
4766
- async init() {
4903
+ /**
4904
+ * Resolves the index state (spec #280 Change 3 transition table). Every
4905
+ * read-only probe runs before the one transition transaction, so a probe
4906
+ * failure leaves zero mutation (Change 4 ordering).
4907
+ */
4908
+ async init(opts = {}) {
4909
+ try {
4910
+ await this.initOnce(opts);
4911
+ } catch (err) {
4912
+ const wrapped = unwrapFtsOnly(err);
4913
+ const cls = ftsCorruptionClass(wrapped.cause);
4914
+ if (cls === "vtab" || cls === "corrupt" && wrapped.ftsOnly) {
4915
+ await this.resetAndRearm();
4916
+ return;
4917
+ }
4918
+ throw wrapped.cause;
4919
+ }
4920
+ }
4921
+ /** Marks errors from statements that touch only FTS tables (entries_fts + shadows). */
4922
+ async ftsOnly(fn) {
4923
+ try {
4924
+ return await fn();
4925
+ } catch (err) {
4926
+ throw new FtsOnlyStatementError(err);
4927
+ }
4928
+ }
4929
+ async initOnce(opts) {
4930
+ const p = this.prefix;
4767
4931
  const state = await this.metadataRepo.getMeta(FTS5_STATE_KEY);
4768
- if (state === "live" && await this.triggersPresent()) {
4769
- await this.db.execAsync(fts5TablesDdl(this.prefix));
4932
+ const schema = await this.readSchema();
4933
+ const ftsExists = schema.tables.has(`${p}entries_fts`);
4934
+ const mapExists = schema.tables.has(`${p}fts_map`);
4935
+ const pendingExists = schema.tables.has(`${p}fts_pending`);
4936
+ const trig = schema.triggersPresent;
4937
+ const ftsDdl = schema.tables.get(`${p}entries_fts`) ?? null;
4938
+ const tokenizerOk = ftsDdl === null || /tokenize\s*=\s*['"]porter\s+unicode61['"]/i.test(ftsDdl);
4939
+ if (state === null && opts.entriesExisted === false) {
4940
+ await this.db.withTransactionAsync(async (tx) => {
4941
+ await tx.execAsync(fts5TablesDdl(p) + fts5PendingIndexesDdl(p) + fts5TriggersDdl(p));
4942
+ await this.metadataRepo.setMeta(FTS5_STATE_KEY, "live", tx);
4943
+ });
4944
+ return;
4945
+ }
4946
+ if (state === "detached") return this.resetAndRearm();
4947
+ if (!tokenizerOk) return this.resetAndRearm();
4948
+ const intact = trig && ftsExists && mapExists && pendingExists;
4949
+ if (intact && state === "rebuilding") return;
4950
+ if (intact && state === "live") {
4951
+ await this.ftsOnly(() => this.db.getFirstAsync(`SELECT rowid FROM ${p}entries_fts ORDER BY rowid DESC LIMIT 1`));
4952
+ if (await this.liveRowUnindexed()) return this.resetAndRearm();
4953
+ return;
4954
+ }
4955
+ if (ftsExists && mapExists && await this.keepMapConsistent()) {
4956
+ await this.db.withTransactionAsync((tx) => this.rearmInTx(tx));
4957
+ return;
4958
+ }
4959
+ return this.resetAndRearm();
4960
+ }
4961
+ /**
4962
+ * The index was lost (rev 8 m3 / rev 9 m4): the oldest or the newest live entry is neither mapped nor
4963
+ * ledgered, which `live` never allows. The oldest one keeps detection armed after post-loss writes
4964
+ * repopulate the map and the ledger (#284 review). Every-open path: each side reads at most
4965
+ * FTS5_DRAIN_CHUNK entries rows (final review M1), so an all-soft-deleted store is never fully scanned;
4966
+ * a lost index whose oldest and newest rows are all deleted goes undetected.
4967
+ */
4968
+ async liveRowUnindexed() {
4969
+ const p = this.prefix;
4970
+ const edge = (dir) => `SELECT id FROM (SELECT id FROM (SELECT id, deleted_at FROM ${p}entries ORDER BY rowid ${dir} LIMIT ${FTS5_DRAIN_CHUNK}) WHERE deleted_at IS NULL LIMIT 1)`;
4971
+ return await this.db.getFirstAsync(
4972
+ `SELECT 1 AS x FROM (${edge("ASC")} UNION ALL ${edge("DESC")}) e
4973
+ WHERE NOT EXISTS (SELECT 1 FROM ${p}fts_map m WHERE m.id = e.id)
4974
+ AND NOT EXISTS (SELECT 1 FROM ${p}fts_pending q WHERE q.id = e.id)
4975
+ LIMIT 1`
4976
+ ) !== null;
4977
+ }
4978
+ /**
4979
+ * Read-only keep-map check (spec DDL note): every surviving entries_fts
4980
+ * rowid must have a map row. Max-rowid seek, then docsize vs map counts.
4981
+ */
4982
+ async keepMapConsistent() {
4983
+ const p = this.prefix;
4984
+ const fts = await this.ftsOnly(() => this.db.getFirstAsync(`SELECT rowid FROM ${p}entries_fts ORDER BY rowid DESC LIMIT 1`));
4985
+ const map = await this.db.getFirstAsync(
4986
+ `SELECT coalesce(max(fts_rowid), 0) AS hi, count(*) AS n FROM ${p}fts_map`
4987
+ );
4988
+ if ((fts?.rowid ?? 0) > (map?.hi ?? 0)) return false;
4989
+ const doc = await this.ftsOnly(() => this.db.getFirstAsync(`SELECT count(*) AS n FROM ${p}entries_fts_docsize`));
4990
+ return (doc?.n ?? 0) === (map?.n ?? 0);
4991
+ }
4992
+ /** Keep-map re-arm: triggers, empty ledger, rebuild from 0, state rebuilding. One transaction. */
4993
+ async rearmInTx(tx) {
4994
+ const p = this.prefix;
4995
+ await tx.execAsync(fts5TablesDdl(p));
4996
+ await tx.execAsync(`DELETE FROM ${p}fts_pending;`);
4997
+ await tx.execAsync(fts5PendingIndexesDdl(p) + fts5TriggersDdl(p));
4998
+ await writeRebuildStart(tx, p, this.metadataRepo);
4999
+ await this.metadataRepo.setMeta(FTS5_STATE_KEY, "rebuilding", tx);
5000
+ }
5001
+ /** Reset path: retire entries_fts, recreate it empty, clear map + ledger, then the re-arm. One transaction. */
5002
+ async resetAndRearm() {
5003
+ const p = this.prefix;
5004
+ await this.db.withTransactionAsync(async (tx) => {
5005
+ await this.retireFtsTable(tx);
5006
+ await tx.execAsync(fts5TablesDdl(p));
5007
+ await tx.execAsync(`DELETE FROM ${p}fts_map; DELETE FROM ${p}fts_pending;`);
5008
+ await this.rearmInTx(tx);
5009
+ });
5010
+ }
5011
+ /** Rename-retire entries_fts (spec #280 rev 11); DROP only when the RENAME throws. */
5012
+ async retireFtsTable(tx) {
5013
+ const p = this.prefix;
5014
+ if (!await this.metadataRepo.tableExists(`${p}entries_fts`, tx)) return;
5015
+ const n = Number(await this.metadataRepo.getMeta(FTS5_RETIRED_COUNTER_KEY, tx) ?? "0") + 1;
5016
+ const name = retiredTableName(p, n);
5017
+ try {
5018
+ await tx.execAsync(`ALTER TABLE ${p}entries_fts RENAME TO ${name}`);
5019
+ } catch (err) {
5020
+ if (isTransientOrTxFatal(err)) throw err;
5021
+ await tx.execAsync(`DROP TABLE ${p}entries_fts`);
4770
5022
  return;
4771
5023
  }
5024
+ await this.metadataRepo.setMeta(FTS5_RETIRED_TABLES_KEY, JSON.stringify([...await this.retiredList(tx), name]), tx);
5025
+ await this.metadataRepo.setMeta(FTS5_RETIRED_COUNTER_KEY, String(n), tx);
5026
+ }
5027
+ async retiredList(tx = this.db) {
5028
+ const raw = parseJsonArray(await this.metadataRepo.getMeta(FTS5_RETIRED_TABLES_KEY, tx), []);
5029
+ return raw.filter((n) => isRetiredTableName(this.prefix, n));
5030
+ }
5031
+ async drain(opts = {}) {
5032
+ try {
5033
+ await this.drainOnce(opts);
5034
+ } catch (err) {
5035
+ if (ftsCorruptionClass(err) !== "vtab") throw err;
5036
+ await this.resetAndRearm();
5037
+ if (opts.ids && opts.ids.length > 0) await this.drainIds(opts.ids, idReindexChunkSql(this.prefix));
5038
+ }
5039
+ }
5040
+ async drainOnce(opts) {
5041
+ const ids = opts.ids ?? [];
5042
+ if (ids.length > 0) await this.drainIds(ids);
5043
+ if (opts.mode === "ledger") {
5044
+ if (ids.length === 0 && opts.entityId !== void 0) await this.drainEntityChunk(opts.entityId);
5045
+ else await this.drainLedgerChunk();
5046
+ return;
5047
+ }
5048
+ await this.fullQuantum(opts.deadline ?? monotonicNow() + this.budgetMs);
5049
+ }
5050
+ /**
5051
+ * One full quantum (spec #280 Change 3): ledger first (at most half the
5052
+ * budget while `rebuilding`); then, while `rebuilding`, ≥1 window unless
5053
+ * the ledger drain overran the deadline, one sweep tx, and the flip check;
5054
+ * then reclamation. A `live` store reads only the state.
5055
+ */
5056
+ async fullQuantum(deadline) {
5057
+ const rebuilding = await this.metadataRepo.getMeta(FTS5_STATE_KEY) === "rebuilding";
5058
+ const start = monotonicNow();
5059
+ const ledgerRan = await this.drainLedgerUntil(rebuilding ? start + (deadline - start) / 2 : deadline);
5060
+ const ledgerOverran = ledgerRan && monotonicNow() >= deadline;
5061
+ if (rebuilding) {
5062
+ const progress = await readRebuildProgress(this.metadataRepo);
5063
+ if (!progress || !await this.fingerprintValid(progress.wm)) {
5064
+ await this.restartRebuild();
5065
+ } else {
5066
+ if (!ledgerOverran) {
5067
+ let step;
5068
+ do {
5069
+ step = await this.runWindow(progress);
5070
+ } while (step === "more" && monotonicNow() < deadline);
5071
+ if (step !== "stale") await this.runSweepTx(progress);
5072
+ }
5073
+ await this.tryFlip();
5074
+ }
5075
+ }
5076
+ try {
5077
+ await this.reclaimRetired(deadline);
5078
+ } catch (err) {
5079
+ if (!isSqliteError(err)) throw err;
5080
+ }
5081
+ }
5082
+ /** Bounded reclamation steps, each its own tx, until the deadline or nothing is retired. */
5083
+ async reclaimRetired(deadline) {
5084
+ while (monotonicNow() < deadline) {
5085
+ const [name] = await this.retiredList();
5086
+ if (name === void 0) return;
5087
+ try {
5088
+ if (!await this.metadataRepo.tableExists(name)) {
5089
+ await this.delistRetired(name, false);
5090
+ continue;
5091
+ }
5092
+ while (!await this.reclaimStep(name) && monotonicNow() < deadline) ;
5093
+ } catch (err) {
5094
+ if (ftsCorruptionClass(err) === null) throw err;
5095
+ try {
5096
+ await this.delistRetired(name, true);
5097
+ } catch (dropErr) {
5098
+ if (!isSqliteError(dropErr)) throw dropErr;
5099
+ await this.delistRetired(name, false);
5100
+ }
5101
+ }
5102
+ }
5103
+ }
5104
+ /** (1) delete <=500 rows, else (2) one bounded merge, else (3) DROP + delist. True once delisted. */
5105
+ async reclaimStep(name) {
5106
+ if (await this.db.getFirstAsync(`SELECT rowid FROM ${name} LIMIT 1`)) {
5107
+ await this.db.withTransactionAsync(async (tx) => {
5108
+ await tx.runAsync(`DELETE FROM ${name} WHERE rowid IN (SELECT rowid FROM ${name} ORDER BY rowid LIMIT ${FTS5_DRAIN_CHUNK})`);
5109
+ });
5110
+ return false;
5111
+ }
5112
+ let merged = false;
5113
+ await this.db.withTransactionAsync(async (tx) => {
5114
+ const before = await tx.getFirstAsync(`SELECT total_changes() AS n`);
5115
+ await tx.runAsync(`INSERT INTO ${name}(${name}, rank) VALUES('merge', ${FTS5_RECLAIM_MERGE_PAGES})`);
5116
+ const after = await tx.getFirstAsync(`SELECT total_changes() AS n`);
5117
+ merged = (after?.n ?? 0) - (before?.n ?? 0) >= 2;
5118
+ });
5119
+ if (!merged) await this.delistRetired(name, true);
5120
+ return !merged;
5121
+ }
5122
+ async delistRetired(name, drop) {
5123
+ await this.db.withTransactionAsync(async (tx) => {
5124
+ if (drop) await tx.execAsync(`DROP TABLE IF EXISTS ${name}`);
5125
+ const rest = (await this.retiredList(tx)).filter((n) => n !== name);
5126
+ await this.metadataRepo.setMeta(FTS5_RETIRED_TABLES_KEY, JSON.stringify(rest), tx);
5127
+ });
5128
+ }
5129
+ /** The id at the watermark's rowid still matches. A null id (start, or windows complete) has nothing to check. */
5130
+ async fingerprintValid(wm, tx = this.db) {
5131
+ if (wm.id === null) return true;
5132
+ const row = await tx.getFirstAsync(`SELECT id FROM ${this.prefix}entries WHERE rowid = ?`, [wm.rowid]);
5133
+ return row?.id === wm.id;
5134
+ }
5135
+ /** External change under the watermark: start over (same re-arm transaction as a trigger-loss restart). */
5136
+ async restartRebuild() {
5137
+ await this.db.withTransactionAsync((tx) => this.rearmInTx(tx));
5138
+ }
5139
+ /**
5140
+ * Progress as stored, re-read inside a window/sweep tx: false when it no
5141
+ * longer matches what this quantum assumed (final review I1). Writing a
5142
+ * watermark or cursor computed from a stale `pr` would overwrite a
5143
+ * concurrent restart (runPrune VACUUM is not on the sync chain).
5144
+ */
5145
+ async progressUnchanged(pr, tx) {
5146
+ if (await this.metadataRepo.getMeta(FTS5_STATE_KEY, tx) !== "rebuilding") return false;
5147
+ const cur = await readRebuildProgress(this.metadataRepo, tx);
5148
+ return cur !== null && cur.wm.rowid === pr.wm.rowid && cur.wm.id === pr.wm.id && cur.cursor === pr.cursor && cur.snapshot.entries === pr.snapshot.entries && cur.snapshot.map === pr.snapshot.map;
5149
+ }
5150
+ /** One window tx. 'done' once no rows remain in (wm, snapshot] (the watermark is then pinned at the snapshot, F9); 'stale' when progress moved under the quantum. */
5151
+ async runWindow(pr) {
5152
+ if (pr.wm.rowid >= pr.snapshot.entries) return "done";
5153
+ const p = this.prefix;
5154
+ let next = pr.wm;
5155
+ let more = false;
5156
+ let stale = false;
4772
5157
  await this.db.withTransactionAsync(async (tx) => {
4773
- await tx.execAsync(`DROP TABLE IF EXISTS ${this.prefix}entries_fts;`);
4774
- await tx.execAsync(fts5TablesDdl(this.prefix));
4775
- await this.rebuildInTx(tx);
4776
- await tx.execAsync(fts5TriggersDdl(this.prefix));
5158
+ if (!await this.progressUnchanged(pr, tx)) {
5159
+ stale = true;
5160
+ return;
5161
+ }
5162
+ const row = await tx.getFirstAsync(windowHiSql(p), [pr.wm.rowid, pr.snapshot.entries]);
5163
+ if (row?.hi == null) {
5164
+ next = { rowid: pr.snapshot.entries, id: null };
5165
+ } else {
5166
+ for (const sql of windowChunkSql(p)) await tx.runAsync(sql, [pr.wm.rowid, row.hi]);
5167
+ const fp = await tx.getFirstAsync(`SELECT id FROM ${p}entries WHERE rowid = ?`, [row.hi]);
5168
+ next = { rowid: row.hi, id: fp.id };
5169
+ more = true;
5170
+ }
5171
+ await this.metadataRepo.setMeta(FTS5_REBUILD_WM_KEY, JSON.stringify(next), tx);
5172
+ });
5173
+ if (stale) return "stale";
5174
+ pr.wm = next;
5175
+ return more ? "more" : "done";
5176
+ }
5177
+ /** At most one bounded sweep tx; the cursor advances in the same tx. */
5178
+ async runSweepTx(pr) {
5179
+ if (pr.cursor >= pr.snapshot.map) return;
5180
+ const p = this.prefix;
5181
+ let next = pr.cursor;
5182
+ await this.db.withTransactionAsync(async (tx) => {
5183
+ if (!await this.progressUnchanged(pr, tx)) return;
5184
+ const row = await tx.getFirstAsync(sweepHiSql(p), [pr.cursor, pr.snapshot.map]);
5185
+ const hi = row?.hi ?? pr.snapshot.map;
5186
+ for (const sql of sweepChunkSql(p)) await tx.runAsync(sql, [pr.cursor, hi]);
5187
+ await this.metadataRepo.setMeta(FTS5_SWEEP_CURSOR_KEY, String(hi), tx);
5188
+ next = hi;
5189
+ });
5190
+ pr.cursor = next;
5191
+ }
5192
+ /** Completion, re-checked inside the flipping tx; the residual ledger (≤ one chunk) drains in it. */
5193
+ async tryFlip() {
5194
+ const p = this.prefix;
5195
+ await this.db.withTransactionAsync(async (tx) => {
5196
+ if (await this.metadataRepo.getMeta(FTS5_STATE_KEY, tx) !== "rebuilding") return;
5197
+ const pr = await readRebuildProgress(this.metadataRepo, tx);
5198
+ if (!pr || pr.wm.rowid < pr.snapshot.entries || pr.cursor < pr.snapshot.map) return;
5199
+ if (!await this.fingerprintValid(pr.wm, tx)) return;
5200
+ const residual = await tx.getFirstAsync(
5201
+ `SELECT count(*) AS n FROM (SELECT 1 FROM ${p}fts_pending LIMIT ${FTS5_DRAIN_CHUNK + 1})`
5202
+ );
5203
+ if ((residual?.n ?? 0) > FTS5_DRAIN_CHUNK) return;
5204
+ const hi = await tx.getFirstAsync(drainHiSql(p));
5205
+ if (hi?.hi != null) for (const sql of drainChunkSql(p)) await tx.runAsync(sql, [hi.hi]);
5206
+ await this.metadataRepo.deleteMeta(FTS5_REBUILD_WM_KEY, tx);
5207
+ await this.metadataRepo.deleteMeta(FTS5_REBUILD_SNAPSHOT_KEY, tx);
5208
+ await this.metadataRepo.deleteMeta(FTS5_SWEEP_CURSOR_KEY, tx);
4777
5209
  await this.metadataRepo.setMeta(FTS5_STATE_KEY, "live", tx);
4778
5210
  });
4779
5211
  }
4780
- async rebuildFromSource() {
4781
- await this.db.withTransactionAsync((tx) => this.rebuildInTx(tx));
5212
+ /** Seq-ordered ledger chunks until empty or `stopAt`. True when a chunk ran. */
5213
+ async drainLedgerUntil(stopAt) {
5214
+ let ran = false;
5215
+ while (await this.drainLedgerChunk()) {
5216
+ ran = true;
5217
+ if (monotonicNow() >= stopAt) break;
5218
+ }
5219
+ return ran;
4782
5220
  }
4783
- async drain() {
4784
- const hiSql = drainHiSql(this.prefix);
4785
- const chunk = drainChunkSql(this.prefix);
4786
- for (; ; ) {
4787
- const done = await this.db.withTransactionAsync(async (tx) => {
4788
- const row = await tx.getFirstAsync(hiSql);
4789
- if (row?.hi == null) return true;
4790
- for (const sql of chunk) await tx.runAsync(sql, [row.hi]);
4791
- return false;
5221
+ /** The caller's own ids, to completion, ≤ FTS5_DRAIN_CHUNK per transaction. */
5222
+ async drainIds(ids, chunk = idDrainChunkSql(this.prefix)) {
5223
+ const unique = [...new Set(ids)];
5224
+ for (let i = 0; i < unique.length; i += FTS5_DRAIN_CHUNK) {
5225
+ const json = JSON.stringify(unique.slice(i, i + FTS5_DRAIN_CHUNK));
5226
+ await this.db.withTransactionAsync(async (tx) => {
5227
+ for (const sql of chunk) await tx.runAsync(sql, [json]);
4792
5228
  });
4793
- if (done) return;
4794
5229
  }
4795
5230
  }
5231
+ /** One seq-ordered chunk. False when the ledger was empty. */
5232
+ async drainLedgerChunk() {
5233
+ const hiSql = drainHiSql(this.prefix);
5234
+ const chunk = drainChunkSql(this.prefix);
5235
+ let ran = false;
5236
+ await this.db.withTransactionAsync(async (tx) => {
5237
+ const row = await tx.getFirstAsync(hiSql);
5238
+ if (row?.hi == null) return;
5239
+ for (const sql of chunk) await tx.runAsync(sql, [row.hi]);
5240
+ ran = true;
5241
+ });
5242
+ return ran;
5243
+ }
5244
+ /** One entity-targeted chunk (ORDER BY seq, ≤ FTS5_DRAIN_CHUNK). */
5245
+ async drainEntityChunk(entityId) {
5246
+ const hiSql = entityDrainHiSql(this.prefix);
5247
+ const chunk = entityDrainChunkSql(this.prefix);
5248
+ await this.db.withTransactionAsync(async (tx) => {
5249
+ const row = await tx.getFirstAsync(hiSql, [entityId]);
5250
+ if (row?.hi == null) return;
5251
+ for (const sql of chunk) await tx.runAsync(sql, [entityId, row.hi]);
5252
+ });
5253
+ }
4796
5254
  async replace(entityId, ids, documents) {
4797
5255
  assertEntity(entityId, documents);
4798
5256
  const docIds = documents.map((d) => d.id);
@@ -4870,20 +5328,57 @@ var Fts5IndexStrategy = class {
4870
5328
  );
4871
5329
  }
4872
5330
  }
4873
- async rebuildInTx(tx) {
5331
+ /** The init probes' schema facts in one sqlite_master read (#284 review): FTS tables with their DDL, and the triggers. */
5332
+ async readSchema() {
4874
5333
  const p = this.prefix;
4875
- await tx.execAsync(`DELETE FROM ${p}entries_fts; DELETE FROM ${p}fts_map; DELETE FROM ${p}fts_pending;`);
4876
- for (const sql of rebuildSql(p)) await tx.runAsync(sql);
4877
- }
4878
- async triggersPresent() {
4879
- const names = fts5TriggerNames(this.prefix);
5334
+ const triggers = fts5TriggerNames(p);
5335
+ const names = [`${p}entries_fts`, `${p}fts_map`, `${p}fts_pending`, ...triggers];
4880
5336
  const rows = await this.db.getAllAsync(
4881
- `SELECT name FROM sqlite_master WHERE type = 'trigger' AND name IN (?, ?, ?)`,
4882
- [...names]
5337
+ `SELECT type, name, sql FROM sqlite_master WHERE name IN (${names.map(() => "?").join(", ")})`,
5338
+ names
4883
5339
  );
4884
- return rows.length === names.length;
5340
+ const tables = new Map(rows.filter((r) => r.type === "table").map((r) => [r.name, r.sql]));
5341
+ const present = new Set(rows.filter((r) => r.type === "trigger").map((r) => r.name));
5342
+ return { tables, triggersPresent: triggers.every((n) => present.has(n)) };
4885
5343
  }
4886
5344
  };
5345
+ var FtsOnlyStatementError = class extends Error {
5346
+ constructor(cause) {
5347
+ super("FTS-only statement failed");
5348
+ this.ftsOnlyCause = cause;
5349
+ }
5350
+ };
5351
+ function unwrapFtsOnly(err) {
5352
+ try {
5353
+ if (err instanceof FtsOnlyStatementError) return { cause: err.ftsOnlyCause, ftsOnly: true };
5354
+ } catch {
5355
+ }
5356
+ return { cause: err, ftsOnly: false };
5357
+ }
5358
+ function safeSqliteCode(err) {
5359
+ try {
5360
+ const code = extractSqliteCode(err);
5361
+ if (code !== void 0) return code;
5362
+ const lifted = err?.sqliteErrorCode;
5363
+ return typeof lifted === "string" ? lifted : void 0;
5364
+ } catch {
5365
+ return void 0;
5366
+ }
5367
+ }
5368
+ var TRANSIENT_OR_TX_FATAL = ["SQLITE_BUSY", "SQLITE_LOCKED", "SQLITE_FULL", "SQLITE_IOERR", "SQLITE_NOMEM", "SQLITE_INTERRUPT", "SQLITE_ABORT", "SQLITE_READONLY"];
5369
+ function isTransientOrTxFatal(err) {
5370
+ const code = safeSqliteCode(err);
5371
+ return code !== void 0 && TRANSIENT_OR_TX_FATAL.some((c) => code === c || code.startsWith(`${c}_`));
5372
+ }
5373
+ function ftsCorruptionClass(err) {
5374
+ const code = safeSqliteCode(err);
5375
+ if (code === "SQLITE_CORRUPT_VTAB") return "vtab";
5376
+ if (code?.startsWith("SQLITE_CORRUPT")) return "corrupt";
5377
+ return null;
5378
+ }
5379
+ function isSqliteError(err) {
5380
+ return safeSqliteCode(err) !== void 0;
5381
+ }
4887
5382
  function assertEntity(entityId, documents) {
4888
5383
  for (const d of documents) {
4889
5384
  if (d.entity_id !== entityId) {
@@ -4941,15 +5436,15 @@ async function detachFts5(db, prefix, metadataRepo) {
4941
5436
  } catch {
4942
5437
  }
4943
5438
  }
4944
- async function createIndexStrategy(db, prefix, metadataRepo, preferred) {
5439
+ async function createIndexStrategy(db, prefix, metadataRepo, preferred, opts = {}) {
4945
5440
  if (preferred === "minisearch") {
4946
5441
  await detachFts5(db, prefix, metadataRepo);
4947
5442
  return new MiniSearchIndexStrategy();
4948
5443
  }
4949
5444
  const attempt = async () => {
4950
5445
  if (!await probeFts5(db, prefix)) return "absent";
4951
- const strategy = new Fts5IndexStrategy(db, prefix, metadataRepo);
4952
- await strategy.init();
5446
+ const strategy = new Fts5IndexStrategy(db, prefix, metadataRepo, opts.setupBudgetMs);
5447
+ await strategy.init({ entriesExisted: opts.entriesExisted });
4953
5448
  return strategy;
4954
5449
  };
4955
5450
  const absent = async () => {
@@ -6312,7 +6807,8 @@ var IngestionService = class {
6312
6807
  *
6313
6808
  * @returns Counts: nodesWritten (validated nodes persisted), edgesWritten
6314
6809
  * (manifest-valid edges persisted), superseded (prior facts soft-deleted
6315
- * plus prior source-ref edges hard-deleted).
6810
+ * plus prior source-ref edges hard-deleted), and supersededFactIds (the
6811
+ * ids of those soft-deleted facts, for the caller's post-commit index sync).
6316
6812
  *
6317
6813
  * Node-ownership pre-flight (spec §3, REQ-GRAPH-01): the first statement
6318
6814
  * is `assertGraphNodeOwnership(entityId, params.nodes, tx)`, which runs
@@ -6440,7 +6936,8 @@ var IngestionService = class {
6440
6936
  return {
6441
6937
  nodesWritten: wikiFacts.length,
6442
6938
  edgesWritten: edgesInsertedCount,
6443
- superseded: deletedFactIds.length + deletedEdgeCount
6939
+ superseded: deletedFactIds.length + deletedEdgeCount,
6940
+ supersededFactIds: deletedFactIds
6444
6941
  };
6445
6942
  }
6446
6943
  /**
@@ -7724,6 +8221,7 @@ var MaintenanceService = class {
7724
8221
  if (vacuum) {
7725
8222
  await resetNormalizeWatermarks(this.metadataRepo);
7726
8223
  await this.metadataRepo.vacuum();
8224
+ await restartRebuildAfterVacuum(this.db, this.prefix, this.metadataRepo);
7727
8225
  }
7728
8226
  await this.searchService.syncEntries(entityId, syncedIds);
7729
8227
  return { entries: deletedEntries, tasks: deletedTasks, events: deletedEvents };
@@ -9167,7 +9665,7 @@ var ImportExportService = class {
9167
9665
  if (merge) {
9168
9666
  await this.searchService.syncEntries(entityId, [...upsertedFactIds]);
9169
9667
  } else {
9170
- await this.searchService.sync(entityId);
9668
+ await this.searchService.sync(entityId, { ids: [.../* @__PURE__ */ new Set([...upsertedFactIds, ...softDeletedFactIds])] });
9171
9669
  }
9172
9670
  for (const fact of bundle.facts) {
9173
9671
  if (!fact.deleted_at && upsertedFactIds.has(fact.id) && !factsWithPreservedBlob.has(fact.id)) {
@@ -10971,7 +11469,6 @@ var TABLE_PREFIX_PATTERN = /^[A-Za-z][A-Za-z0-9_]{0,30}_$/;
10971
11469
  var SEARCH_REINIT_COOLDOWN_MS = 5e3;
10972
11470
  var SEARCH_REINIT_COOLDOWN_MAX_MS = 5 * 6e4;
10973
11471
  var SEARCH_REINIT_COST_FACTOR = 10;
10974
- var monotonicNow = () => typeof performance !== "undefined" && typeof performance.now === "function" ? performance.now() : Date.now();
10975
11472
  function decodeDraftCursor(cursor) {
10976
11473
  const sep = typeof cursor === "string" ? cursor.indexOf(":") : -1;
10977
11474
  const createdAt = sep > 0 ? Number(cursor.slice(0, sep)) : Number.NaN;
@@ -10986,6 +11483,16 @@ var WikiMemory = class {
10986
11483
  constructor(db, options) {
10987
11484
  /** Emits `__testAccess` console warning at most once per instance when NODE_ENV ≠ "test". */
10988
11485
  __privateAdd(this, _testAccessNonTestEnvWarned, false);
11486
+ /**
11487
+ * upsertGraph node and superseded-fact ids per entity, awaiting syncSearchIndex(), each with the
11488
+ * generation of its latest stash. Ids leave the stash only once a drain
11489
+ * succeeded, so a failed drain retries them next call.
11490
+ */
11491
+ this.upsertIdStash = /* @__PURE__ */ new Map();
11492
+ /** Total ids across upsertIdStash, kept in step so the cap check is O(1). */
11493
+ this.upsertIdStashSize = 0;
11494
+ /** Bumped per stashed id; a re-stash during an in-flight drain outlives that drain's drop. */
11495
+ this.upsertIdStashGen = 0;
10989
11496
  this.reinitSearch = null;
10990
11497
  /** Monotonic time before which a no-arg syncSearchIndex() skips the re-init (0 = no cooldown). */
10991
11498
  this.reinitRetryAt = 0;
@@ -11122,6 +11629,49 @@ var WikiMemory = class {
11122
11629
  jobManager: this.jobManager
11123
11630
  };
11124
11631
  }
11632
+ stashUpsertIds(entityId, ids) {
11633
+ if (!this.searchService.hasLedger()) return;
11634
+ const cap = this.constructor.UPSERT_ID_STASH_CAP;
11635
+ const stash = this.upsertIdStash.get(entityId) ?? /* @__PURE__ */ new Map();
11636
+ for (const id of ids) {
11637
+ if (!stash.has(id)) {
11638
+ if (this.upsertIdStashSize >= cap) continue;
11639
+ this.upsertIdStashSize++;
11640
+ }
11641
+ stash.set(id, ++this.upsertIdStashGen);
11642
+ }
11643
+ if (stash.size > 0) this.upsertIdStash.set(entityId, stash);
11644
+ }
11645
+ /** Snapshot of the stash (one entity, or all) with generations, without removing anything. */
11646
+ peekStashedIds(entityId) {
11647
+ const snapshot = /* @__PURE__ */ new Map();
11648
+ for (const [id, stash] of this.upsertIdStash) {
11649
+ if (entityId === void 0 || id === entityId) snapshot.set(id, new Map(stash));
11650
+ }
11651
+ return snapshot;
11652
+ }
11653
+ /**
11654
+ * Removes a drained snapshot. An id re-stashed after the peek has a newer
11655
+ * generation and stays: the drain may have run before that write committed.
11656
+ */
11657
+ dropStashedIds(snapshot) {
11658
+ for (const [entityId, drained] of snapshot) {
11659
+ const stash = this.upsertIdStash.get(entityId);
11660
+ if (!stash) continue;
11661
+ for (const [id, gen] of drained) {
11662
+ if (stash.get(id) === gen && stash.delete(id)) this.upsertIdStashSize--;
11663
+ }
11664
+ if (stash.size === 0) this.upsertIdStash.delete(entityId);
11665
+ }
11666
+ }
11667
+ /** `config.setupBudgetMs` or the default, validated: a positive finite number. */
11668
+ setupBudgetMs() {
11669
+ const budget = this.options.config?.setupBudgetMs ?? DEFAULT_SETUP_BUDGET_MS;
11670
+ if (typeof budget !== "number" || !Number.isFinite(budget) || budget <= 0) {
11671
+ throw new TypeError(`setupBudgetMs must be a positive finite number; got ${String(budget)}`);
11672
+ }
11673
+ return budget;
11674
+ }
11125
11675
  /**
11126
11676
  * Opens the store: migration gate, DDL, schema migrations, then the stamped
11127
11677
  * source_ref normalize. Await it before any other call on this instance — the
@@ -11132,8 +11682,15 @@ var WikiMemory = class {
11132
11682
  * Pass `{ migrations: 'refuse' }` from inside a writer lease: pending one-time
11133
11683
  * work then throws WikiMigrationRequiredError before any mutation, and the
11134
11684
  * host runs a plain `setup()` outside the lease.
11685
+ *
11686
+ * Keyword-index catch-up work is bounded by `config.setupBudgetMs`; anything
11687
+ * left converges on later opens and no-arg `syncSearchIndex()` calls. Not
11688
+ * bounded: an index reset (tokenizer change, re-attach after detach, a lost
11689
+ * or inconsistent index), which runs as one unchunked transaction.
11135
11690
  */
11136
11691
  async setup(opts) {
11692
+ const setupBudgetMs = this.setupBudgetMs();
11693
+ const openDeadline = monotonicNow() + setupBudgetMs;
11137
11694
  const entriesExistedBeforeSetup = await this.metadataRepo.tableExists(`${this.prefix}entries`);
11138
11695
  const migrationPolicy = opts?.migrations ?? this.options.config?.migrations ?? "allow";
11139
11696
  await setupDatabaseWithGate(this.db, this.prefix, this.metadataRepo, migrationPolicy);
@@ -11168,10 +11725,11 @@ var WikiMemory = class {
11168
11725
  this.db,
11169
11726
  this.prefix,
11170
11727
  this.metadataRepo,
11171
- this.options.config?.indexStrategy ?? "auto"
11728
+ this.options.config?.indexStrategy ?? "auto",
11729
+ { setupBudgetMs, entriesExisted: entriesExistedBeforeSetup }
11172
11730
  )
11173
11731
  );
11174
- await this.searchService.sync();
11732
+ await this.searchService.sync(void 0, { deadline: openDeadline });
11175
11733
  }
11176
11734
  async hasChanged(entityId, sourceRefOrEntries, sourceHashArg) {
11177
11735
  if (typeof sourceRefOrEntries === "string") {
@@ -11422,14 +11980,18 @@ var WikiMemory = class {
11422
11980
  * commit, so without this the nodes stay out of keyword, hybrid and
11423
11981
  * fallback retrieval until the entity's next core write.
11424
11982
  *
11425
- * With `entityId`, rebuilds that entity if it is stale or has never been
11426
- * indexed; otherwise it is a no-op that still waits for rebuilds already
11427
- * queued. Without an argument, rebuilds every entity currently marked
11428
- * stale; when none is, it resolves immediately and does not wait for rebuilds
11429
- * already queued. While keyword search is unavailable it first retries the
11430
- * index init; after a failed attempt it waits at least 5 s, doubling per
11431
- * consecutive failure up to 5 min (longer when an attempt is itself slow),
11432
- * before retrying again.
11983
+ * With `entityId`: drains that entity's pending `upsertGraph` ids (its new
11984
+ * nodes and the facts they superseded) first, to completion, then at most
11985
+ * one more ledger chunk — so its new nodes are searchable, and the
11986
+ * superseded facts gone, when the call resolves, under any backlog. Without an
11987
+ * argument: drains every stashed `upsertGraph` id across entities first,
11988
+ * then runs one keyword-index quantum bounded by `config.setupBudgetMs`
11989
+ * (backlog drain, then rebuild work while the index is rebuilding). Hosts
11990
+ * with a long-lived process should call the no-arg form on their own
11991
+ * schedule: that is what converges a rebuilding index. While keyword search
11992
+ * is unavailable it first retries the index init; after a failed attempt it
11993
+ * waits at least 5 s, doubling per consecutive failure up to 5 min (longer
11994
+ * when an attempt is itself slow), before retrying again.
11433
11995
  *
11434
11996
  * Call only after the host's transaction commits: called inside it, the
11435
11997
  * rebuild may index rows that later roll back, or miss them and clear the
@@ -11455,10 +12017,11 @@ var WikiMemory = class {
11455
12017
  this.db,
11456
12018
  this.prefix,
11457
12019
  this.metadataRepo,
11458
- this.options.config?.indexStrategy ?? "auto"
12020
+ this.options.config?.indexStrategy ?? "auto",
12021
+ { setupBudgetMs: this.setupBudgetMs() }
11459
12022
  );
11460
12023
  this.searchService.setIndexStrategy(strategy);
11461
- if (this.searchService.isKeywordAvailable()) await this.searchService.sync();
12024
+ if (this.searchService.isKeywordAvailable() && !this.searchService.hasLedger()) await this.searchService.sync();
11462
12025
  } finally {
11463
12026
  if (this.searchService.isKeywordAvailable()) {
11464
12027
  this.reinitFailures = 0;
@@ -11474,10 +12037,24 @@ var WikiMemory = class {
11474
12037
  })());
11475
12038
  await this.reinitSearch;
11476
12039
  }
11477
- return this.searchService.syncStale();
12040
+ const stashed2 = this.peekStashedIds();
12041
+ const ids = [...stashed2.values()].flatMap((m) => [...m.keys()]);
12042
+ if (await this.syncDrained(this.searchService.syncStale({ ids }))) this.dropStashedIds(stashed2);
12043
+ return;
11478
12044
  }
11479
12045
  assertEntityId(entityId);
11480
- return this.searchService.syncEntries(entityId, []);
12046
+ const stashed = this.peekStashedIds(entityId);
12047
+ if (await this.syncDrained(this.searchService.syncEntries(entityId, stashed.get(entityId)?.keys() ?? []))) {
12048
+ this.dropStashedIds(stashed);
12049
+ }
12050
+ }
12051
+ /**
12052
+ * True when a sync applied the stashed ids. While keyword search is
12053
+ * unavailable the stand-in strategy's drain is a no-op that reports success,
12054
+ * so the ids stay stashed for the first sync after a re-init (#284 review).
12055
+ */
12056
+ async syncDrained(sync) {
12057
+ return await sync && this.searchService.isKeywordAvailable();
11481
12058
  }
11482
12059
  async exportDump(entityIds) {
11483
12060
  return this.importExportService.exportDump(entityIds);
@@ -11528,7 +12105,10 @@ var WikiMemory = class {
11528
12105
  * post-commit work (search sync, embedding, cache eviction). After the
11529
12106
  * transaction commits, call `syncSearchIndex(entityId)` — or
11530
12107
  * `syncSearchIndex()` after writing several entities in one transaction —
11531
- * to make the new nodes keyword-searchable. Embeddings still come from the maintenance
12108
+ * to make the new nodes keyword-searchable: both forms drain this call's node ids
12109
+ * first, to completion, so the nodes are searchable under any backlog (up to
12110
+ * `WikiMemory.UPSERT_ID_STASH_CAP` pending ids; beyond that they converge with
12111
+ * the backlog). Embeddings still come from the maintenance
11532
12112
  * sweep (`runLibrarian` / `runHeal` / `runOntologyBackfill` / `runPrune`,
11533
12113
  * scoped via `listEntityIds`).
11534
12114
  *
@@ -11578,12 +12158,13 @@ var WikiMemory = class {
11578
12158
  });
11579
12159
  }
11580
12160
  const diagBuffer = new DiagnosticBuffer();
11581
- const result = await this.ingestionService.upsertGraphCore(
12161
+ const { supersededFactIds, ...result } = await this.ingestionService.upsertGraphCore(
11582
12162
  entityId,
11583
12163
  { sourceRef, sourceHash, nodes: params.nodes, edges: params.edges },
11584
12164
  adapter,
11585
12165
  { diag: { buffer: diagBuffer, operation: "upsertGraph" } }
11586
12166
  );
12167
+ this.stashUpsertIds(entityId, [...supersededFactIds, ...params.nodes.map((n) => n.id)]);
11587
12168
  this.searchService.markStale(entityId);
11588
12169
  diagBuffer.flush(this.options);
11589
12170
  return result;
@@ -11766,6 +12347,8 @@ var WikiMemory = class {
11766
12347
  }
11767
12348
  };
11768
12349
  _testAccessNonTestEnvWarned = new WeakMap();
12350
+ /** Max upsertGraph node ids held for their post-commit sync (spec #280 id threading). */
12351
+ WikiMemory.UPSERT_ID_STASH_CAP = 1e4;
11769
12352
  function assertEntityId(entityId) {
11770
12353
  if (typeof entityId !== "string" || entityId.length === 0) {
11771
12354
  throw new TypeError("Invalid entityId: must be a non-empty string.");
@@ -12695,6 +13278,7 @@ exports.DEFAULT_CHUNK_OVERLAP = DEFAULT_CHUNK_OVERLAP;
12695
13278
  exports.DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT = DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT;
12696
13279
  exports.DEFAULT_MAX_CHUNK_LENGTH = DEFAULT_MAX_CHUNK_LENGTH;
12697
13280
  exports.DEFAULT_MAX_EMBED_CHARS = DEFAULT_MAX_EMBED_CHARS;
13281
+ exports.DEFAULT_SETUP_BUDGET_MS = DEFAULT_SETUP_BUDGET_MS;
12698
13282
  exports.EMBED_CHARS_CEILING = EMBED_CHARS_CEILING;
12699
13283
  exports.HEAL_ANCHORS_PER_CANDIDATE = HEAL_ANCHORS_PER_CANDIDATE;
12700
13284
  exports.HEAL_BATCH_SIZE = HEAL_BATCH_SIZE;