@equationalapplications/core-llm-wiki 7.12.0 → 7.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-KN4XSLLB.mjs → chunk-PCRYIUV5.mjs} +232 -18
- package/dist/chunk-PCRYIUV5.mjs.map +1 -0
- package/dist/index.d.mts +10 -3
- package/dist/index.d.ts +10 -3
- package/dist/index.js +730 -146
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +502 -133
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-7d76Ptlq.d.mts → testing-CqsXv2_l.d.mts} +111 -18
- package/dist/{testing-7d76Ptlq.d.ts → testing-CqsXv2_l.d.ts} +111 -18
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +80 -15
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +3 -3
- package/dist/chunk-KN4XSLLB.mjs.map +0 -1
|
@@ -44,6 +44,8 @@ var SQLITE_RESULT_CODE_NAMES = {
|
|
|
44
44
|
// prefix and we normalize it back to the symbolic name so callers can branch on
|
|
45
45
|
// the same string the rest of the ecosystem uses.
|
|
46
46
|
2067: "SQLITE_CONSTRAINT_UNIQUE",
|
|
47
|
+
// 267 = SQLITE_CORRUPT_VTAB — a corrupt FTS5 shadow table (ftsCorruptionClass).
|
|
48
|
+
267: "SQLITE_CORRUPT_VTAB",
|
|
47
49
|
100: "SQLITE_ROW",
|
|
48
50
|
101: "SQLITE_DONE"
|
|
49
51
|
};
|
|
@@ -999,7 +1001,7 @@ var UnavailableIndexStrategy = class {
|
|
|
999
1001
|
hasIndexedEntity(_entityId) {
|
|
1000
1002
|
return false;
|
|
1001
1003
|
}
|
|
1002
|
-
async drain() {
|
|
1004
|
+
async drain(_opts) {
|
|
1003
1005
|
}
|
|
1004
1006
|
};
|
|
1005
1007
|
|
|
@@ -1045,24 +1047,31 @@ var _SearchService = class _SearchService {
|
|
|
1045
1047
|
isKeywordAvailable() {
|
|
1046
1048
|
return !(this.indexStrategy instanceof UnavailableIndexStrategy);
|
|
1047
1049
|
}
|
|
1050
|
+
/** True when the installed strategy drains a durable ledger (FTS5, or the degraded stand-in). */
|
|
1051
|
+
hasLedger() {
|
|
1052
|
+
return this.indexStrategy.drain !== void 0;
|
|
1053
|
+
}
|
|
1048
1054
|
/**
|
|
1049
1055
|
* One serialized drain turn for strategies with a durable ledger. Like
|
|
1050
1056
|
* sync(), it never rejects: the keyword index is rebuildable, and a failed
|
|
1051
|
-
* drain leaves the ledger rows in place for the next turn.
|
|
1057
|
+
* drain leaves the ledger rows in place for the next turn. Resolves false
|
|
1058
|
+
* when the drain failed.
|
|
1052
1059
|
*/
|
|
1053
|
-
drainTurn(entityId) {
|
|
1060
|
+
drainTurn(entityId, opts) {
|
|
1054
1061
|
const work = this.syncChain.then(async () => {
|
|
1055
1062
|
try {
|
|
1056
1063
|
try {
|
|
1057
|
-
await this.indexStrategy.drain();
|
|
1064
|
+
await this.indexStrategy.drain(opts);
|
|
1058
1065
|
} finally {
|
|
1059
1066
|
this.evictCache(entityId);
|
|
1060
1067
|
}
|
|
1068
|
+
return true;
|
|
1061
1069
|
} catch (err) {
|
|
1062
1070
|
console.warn(`[WikiMemory] search index drain failed for ${entityId ?? "*"}:`, err);
|
|
1071
|
+
return false;
|
|
1063
1072
|
}
|
|
1064
1073
|
});
|
|
1065
|
-
this.syncChain = work;
|
|
1074
|
+
this.syncChain = work.then(() => void 0);
|
|
1066
1075
|
return work;
|
|
1067
1076
|
}
|
|
1068
1077
|
/**
|
|
@@ -1072,9 +1081,18 @@ var _SearchService = class _SearchService {
|
|
|
1072
1081
|
* Rebuilds are serialized per instance and never reject: the keyword index
|
|
1073
1082
|
* is a rebuildable cache over SQLite, so degraded keyword search is the
|
|
1074
1083
|
* correct failure mode and killing the host process is not.
|
|
1084
|
+
*
|
|
1085
|
+
* With a ledger strategy, `sync()` runs one full quantum bounded by
|
|
1086
|
+
* `opts.deadline`; `sync(entityId)` is a write turn (ledger only).
|
|
1087
|
+
* `opts.ids` are the caller's touched ids: a ledger strategy drains them
|
|
1088
|
+
* first, to completion; MiniSearch ignores them and rebuilds the entity in
|
|
1089
|
+
* one scan.
|
|
1075
1090
|
*/
|
|
1076
|
-
async sync(entityId) {
|
|
1077
|
-
if (this.indexStrategy.drain)
|
|
1091
|
+
async sync(entityId, opts) {
|
|
1092
|
+
if (this.indexStrategy.drain) {
|
|
1093
|
+
await this.drainTurn(entityId, entityId === void 0 ? { mode: "full", deadline: opts?.deadline, ids: opts?.ids } : { mode: "ledger", entityId, ids: opts?.ids });
|
|
1094
|
+
return;
|
|
1095
|
+
}
|
|
1078
1096
|
const work = this.syncChain.then(async () => {
|
|
1079
1097
|
try {
|
|
1080
1098
|
try {
|
|
@@ -1115,13 +1133,17 @@ var _SearchService = class _SearchService {
|
|
|
1115
1133
|
* — including on an empty `ids` list, which is a no-op only for an entity
|
|
1116
1134
|
* the index already tracks, and even then one that waits for rebuilds
|
|
1117
1135
|
* already queued on the chain.
|
|
1136
|
+
*
|
|
1137
|
+
* Resolves false only when a ledger drain failed: the caller's ids were not
|
|
1138
|
+
* applied (their ledger rows remain), so a caller holding them retries.
|
|
1118
1139
|
*/
|
|
1119
1140
|
async syncEntries(entityId, ids) {
|
|
1120
|
-
if (this.indexStrategy.drain) return this.drainTurn(entityId);
|
|
1141
|
+
if (this.indexStrategy.drain) return this.drainTurn(entityId, { mode: "ledger", entityId, ids: [...new Set(ids)] });
|
|
1121
1142
|
const uniqueIds = [...new Set(ids)];
|
|
1122
1143
|
const indexed = this.indexStrategy.hasIndexedEntity?.(entityId);
|
|
1123
1144
|
if (uniqueIds.length === 0 && !this.staleEntities.has(entityId) && indexed === true) {
|
|
1124
|
-
|
|
1145
|
+
await this.syncChain;
|
|
1146
|
+
return true;
|
|
1125
1147
|
}
|
|
1126
1148
|
const work = this.syncChain.then(async () => {
|
|
1127
1149
|
try {
|
|
@@ -1150,7 +1172,8 @@ var _SearchService = class _SearchService {
|
|
|
1150
1172
|
}
|
|
1151
1173
|
});
|
|
1152
1174
|
this.syncChain = work;
|
|
1153
|
-
|
|
1175
|
+
await work;
|
|
1176
|
+
return true;
|
|
1154
1177
|
}
|
|
1155
1178
|
/**
|
|
1156
1179
|
* Forces the entity's next syncEntries to rebuild it in full. For writes core
|
|
@@ -1166,11 +1189,13 @@ var _SearchService = class _SearchService {
|
|
|
1166
1189
|
* so only those entities are rebuilt. Each rebuild still takes its own turn
|
|
1167
1190
|
* on the sync chain, and like syncEntries this never rejects. With nothing
|
|
1168
1191
|
* stale it resolves at once, without waiting on work already on the chain.
|
|
1192
|
+
* Resolves false only when a ledger drain failed (see syncEntries).
|
|
1169
1193
|
*/
|
|
1170
|
-
async syncStale() {
|
|
1171
|
-
if (this.indexStrategy.drain) return this.drainTurn();
|
|
1194
|
+
async syncStale(opts) {
|
|
1195
|
+
if (this.indexStrategy.drain) return this.drainTurn(void 0, { mode: "full", ids: opts?.ids });
|
|
1172
1196
|
const ids = [...this.staleEntities];
|
|
1173
1197
|
await Promise.all(ids.map((id) => this.syncEntries(id, [])));
|
|
1198
|
+
return true;
|
|
1174
1199
|
}
|
|
1175
1200
|
/**
|
|
1176
1201
|
* Clears the parsed vector cache. Useful for mid-loop flush guarantees
|
|
@@ -2660,7 +2685,8 @@ var IngestionService = class {
|
|
|
2660
2685
|
*
|
|
2661
2686
|
* @returns Counts: nodesWritten (validated nodes persisted), edgesWritten
|
|
2662
2687
|
* (manifest-valid edges persisted), superseded (prior facts soft-deleted
|
|
2663
|
-
* plus prior source-ref edges hard-deleted)
|
|
2688
|
+
* plus prior source-ref edges hard-deleted), and supersededFactIds (the
|
|
2689
|
+
* ids of those soft-deleted facts, for the caller's post-commit index sync).
|
|
2664
2690
|
*
|
|
2665
2691
|
* Node-ownership pre-flight (spec §3, REQ-GRAPH-01): the first statement
|
|
2666
2692
|
* is `assertGraphNodeOwnership(entityId, params.nodes, tx)`, which runs
|
|
@@ -2788,7 +2814,8 @@ var IngestionService = class {
|
|
|
2788
2814
|
return {
|
|
2789
2815
|
nodesWritten: wikiFacts.length,
|
|
2790
2816
|
edgesWritten: edgesInsertedCount,
|
|
2791
|
-
superseded: deletedFactIds.length + deletedEdgeCount
|
|
2817
|
+
superseded: deletedFactIds.length + deletedEdgeCount,
|
|
2818
|
+
supersededFactIds: deletedFactIds
|
|
2792
2819
|
};
|
|
2793
2820
|
}
|
|
2794
2821
|
/**
|
|
@@ -3580,6 +3607,143 @@ async function createTemporalIndexesIfColumnsExist(db, prefix) {
|
|
|
3580
3607
|
}
|
|
3581
3608
|
}
|
|
3582
3609
|
|
|
3610
|
+
// src/services/search/fts5Sql.ts
|
|
3611
|
+
var FTS5_STATE_KEY = "fts5_index_state";
|
|
3612
|
+
var FTS5_DRAIN_CHUNK = 500;
|
|
3613
|
+
var FTS5_SWEEP_CHUNK = 500;
|
|
3614
|
+
var FTS5_RETIRED_TABLES_KEY = "fts5_retired_tables";
|
|
3615
|
+
var FTS5_RETIRED_COUNTER_KEY = "fts5_retired_counter";
|
|
3616
|
+
var FTS5_RECLAIM_MERGE_PAGES = 500;
|
|
3617
|
+
function retiredTableName(p, n) {
|
|
3618
|
+
return `${p}entries_fts_retired_${n}`;
|
|
3619
|
+
}
|
|
3620
|
+
function isRetiredTableName(p, name) {
|
|
3621
|
+
return typeof name === "string" && name.startsWith(`${p}entries_fts_retired_`) && /^\d+$/.test(name.slice(`${p}entries_fts_retired_`.length));
|
|
3622
|
+
}
|
|
3623
|
+
var FTS5_REBUILD_WM_KEY = "fts5_rebuild_wm";
|
|
3624
|
+
var FTS5_REBUILD_SNAPSHOT_KEY = "fts5_rebuild_snapshot";
|
|
3625
|
+
var FTS5_SWEEP_CURSOR_KEY = "fts5_orphan_sweep_cursor";
|
|
3626
|
+
function fts5TriggerNames(prefix) {
|
|
3627
|
+
return [`${prefix}entries_fts_ai`, `${prefix}entries_fts_au`, `${prefix}entries_fts_ad`];
|
|
3628
|
+
}
|
|
3629
|
+
function tagsExpr(col) {
|
|
3630
|
+
return `CASE WHEN json_valid(${col}) THEN
|
|
3631
|
+
CASE WHEN json_type(${col}) = 'array' THEN
|
|
3632
|
+
(SELECT group_concat(CASE type WHEN 'true' THEN 'true' WHEN 'false' THEN 'false'
|
|
3633
|
+
WHEN 'null' THEN '' ELSE value END, ' ')
|
|
3634
|
+
FROM json_each(${col}))
|
|
3635
|
+
ELSE ${col} END
|
|
3636
|
+
ELSE ${col} END`;
|
|
3637
|
+
}
|
|
3638
|
+
function fts5TablesDdl(p) {
|
|
3639
|
+
return `
|
|
3640
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS ${p}entries_fts USING fts5(
|
|
3641
|
+
id UNINDEXED, entity_id UNINDEXED, title, body, tags,
|
|
3642
|
+
tokenize = 'porter unicode61'
|
|
3643
|
+
);
|
|
3644
|
+
CREATE TABLE IF NOT EXISTS ${p}fts_map (
|
|
3645
|
+
fts_rowid INTEGER PRIMARY KEY,
|
|
3646
|
+
id TEXT NOT NULL UNIQUE,
|
|
3647
|
+
entity_id TEXT NOT NULL
|
|
3648
|
+
);
|
|
3649
|
+
CREATE INDEX IF NOT EXISTS ${p}fts_map_entity ON ${p}fts_map(entity_id);
|
|
3650
|
+
CREATE TABLE IF NOT EXISTS ${p}fts_pending (
|
|
3651
|
+
seq INTEGER PRIMARY KEY,
|
|
3652
|
+
entity_id TEXT NOT NULL,
|
|
3653
|
+
id TEXT NOT NULL
|
|
3654
|
+
);`;
|
|
3655
|
+
}
|
|
3656
|
+
function fts5TriggersDdl(p) {
|
|
3657
|
+
const [ai, au, ad] = fts5TriggerNames(p);
|
|
3658
|
+
return `
|
|
3659
|
+
CREATE TRIGGER IF NOT EXISTS ${ai} AFTER INSERT ON ${p}entries BEGIN
|
|
3660
|
+
INSERT INTO ${p}fts_pending (entity_id, id) VALUES (NEW.entity_id, NEW.id);
|
|
3661
|
+
END;
|
|
3662
|
+
CREATE TRIGGER IF NOT EXISTS ${au}
|
|
3663
|
+
AFTER UPDATE OF id, entity_id, title, body, tags, deleted_at ON ${p}entries BEGIN
|
|
3664
|
+
INSERT INTO ${p}fts_pending (entity_id, id) VALUES (OLD.entity_id, OLD.id);
|
|
3665
|
+
INSERT INTO ${p}fts_pending (entity_id, id)
|
|
3666
|
+
SELECT NEW.entity_id, NEW.id WHERE NEW.id <> OLD.id OR NEW.entity_id <> OLD.entity_id;
|
|
3667
|
+
END;
|
|
3668
|
+
CREATE TRIGGER IF NOT EXISTS ${ad} AFTER DELETE ON ${p}entries BEGIN
|
|
3669
|
+
INSERT INTO ${p}fts_pending (entity_id, id) VALUES (OLD.entity_id, OLD.id);
|
|
3670
|
+
END;`;
|
|
3671
|
+
}
|
|
3672
|
+
function drainHiSql(p) {
|
|
3673
|
+
return `SELECT max(seq) AS hi FROM (SELECT seq FROM ${p}fts_pending ORDER BY seq LIMIT ${FTS5_DRAIN_CHUNK})`;
|
|
3674
|
+
}
|
|
3675
|
+
function ledgerChunkSql(p, chunkIds, ledgerWhere) {
|
|
3676
|
+
return [
|
|
3677
|
+
`DELETE FROM ${p}entries_fts WHERE rowid IN (SELECT m.fts_rowid FROM ${p}fts_map m WHERE m.id IN (${chunkIds}))`,
|
|
3678
|
+
`DELETE FROM ${p}fts_map WHERE id IN (${chunkIds})`,
|
|
3679
|
+
`INSERT INTO ${p}fts_map (id, entity_id)
|
|
3680
|
+
SELECT e.id, e.entity_id FROM ${p}entries e WHERE e.id IN (${chunkIds}) AND e.deleted_at IS NULL`,
|
|
3681
|
+
`INSERT INTO ${p}entries_fts (rowid, id, entity_id, title, body, tags)
|
|
3682
|
+
SELECT m.fts_rowid, e.id, e.entity_id, e.title, e.body, ${tagsExpr("e.tags")}
|
|
3683
|
+
FROM ${p}fts_map m JOIN ${p}entries e ON e.id = m.id
|
|
3684
|
+
WHERE m.id IN (${chunkIds})`,
|
|
3685
|
+
`DELETE FROM ${p}fts_pending WHERE ${ledgerWhere}`
|
|
3686
|
+
];
|
|
3687
|
+
}
|
|
3688
|
+
function drainChunkSql(p) {
|
|
3689
|
+
return ledgerChunkSql(p, `SELECT id FROM ${p}fts_pending WHERE seq <= ?`, "seq <= ?");
|
|
3690
|
+
}
|
|
3691
|
+
function fts5PendingIndexesDdl(p) {
|
|
3692
|
+
return `
|
|
3693
|
+
CREATE INDEX IF NOT EXISTS ${p}fts_pending_id ON ${p}fts_pending(id);
|
|
3694
|
+
CREATE INDEX IF NOT EXISTS ${p}fts_pending_entity ON ${p}fts_pending(entity_id);`;
|
|
3695
|
+
}
|
|
3696
|
+
function idDrainChunkSql(p) {
|
|
3697
|
+
return ledgerChunkSql(
|
|
3698
|
+
p,
|
|
3699
|
+
`SELECT id FROM ${p}fts_pending WHERE id IN (SELECT value FROM json_each(?))`,
|
|
3700
|
+
"id IN (SELECT value FROM json_each(?))"
|
|
3701
|
+
);
|
|
3702
|
+
}
|
|
3703
|
+
function idReindexChunkSql(p) {
|
|
3704
|
+
return ledgerChunkSql(p, "SELECT value FROM json_each(?)", "id IN (SELECT value FROM json_each(?))");
|
|
3705
|
+
}
|
|
3706
|
+
function entityDrainHiSql(p) {
|
|
3707
|
+
return `SELECT max(seq) AS hi FROM (SELECT seq FROM ${p}fts_pending WHERE entity_id = ? ORDER BY seq LIMIT ${FTS5_DRAIN_CHUNK})`;
|
|
3708
|
+
}
|
|
3709
|
+
function entityDrainChunkSql(p) {
|
|
3710
|
+
return ledgerChunkSql(p, `SELECT id FROM ${p}fts_pending WHERE entity_id = ? AND seq <= ?`, "entity_id = ? AND seq <= ?");
|
|
3711
|
+
}
|
|
3712
|
+
function fts5LedgerDropSql(p) {
|
|
3713
|
+
return [
|
|
3714
|
+
...fts5TriggerNames(p).map((n) => `DROP TRIGGER IF EXISTS ${n};`),
|
|
3715
|
+
`DROP TABLE IF EXISTS ${p}fts_pending;`,
|
|
3716
|
+
`DROP TABLE IF EXISTS ${p}fts_map;`
|
|
3717
|
+
].join("\n");
|
|
3718
|
+
}
|
|
3719
|
+
function windowHiSql(p) {
|
|
3720
|
+
return `SELECT max(rowid) AS hi FROM (SELECT rowid FROM ${p}entries WHERE rowid > ? AND rowid <= ? ORDER BY rowid LIMIT ${FTS5_DRAIN_CHUNK})`;
|
|
3721
|
+
}
|
|
3722
|
+
function windowChunkSql(p) {
|
|
3723
|
+
const chunkIds = `SELECT id FROM ${p}entries WHERE rowid > ? AND rowid <= ?`;
|
|
3724
|
+
return [
|
|
3725
|
+
`DELETE FROM ${p}entries_fts WHERE rowid IN (SELECT m.fts_rowid FROM ${p}fts_map m WHERE m.id IN (${chunkIds}))`,
|
|
3726
|
+
`DELETE FROM ${p}fts_map WHERE id IN (${chunkIds})`,
|
|
3727
|
+
`INSERT INTO ${p}fts_map (id, entity_id)
|
|
3728
|
+
SELECT e.id, e.entity_id FROM ${p}entries e WHERE e.rowid > ? AND e.rowid <= ? AND e.deleted_at IS NULL`,
|
|
3729
|
+
`INSERT INTO ${p}entries_fts (rowid, id, entity_id, title, body, tags)
|
|
3730
|
+
SELECT m.fts_rowid, e.id, e.entity_id, e.title, e.body, ${tagsExpr("e.tags")}
|
|
3731
|
+
FROM ${p}entries e JOIN ${p}fts_map m ON m.id = e.id
|
|
3732
|
+
WHERE e.rowid > ? AND e.rowid <= ?`
|
|
3733
|
+
];
|
|
3734
|
+
}
|
|
3735
|
+
function sweepHiSql(p) {
|
|
3736
|
+
return `SELECT max(fts_rowid) AS hi FROM (SELECT fts_rowid FROM ${p}fts_map WHERE fts_rowid > ? AND fts_rowid <= ? ORDER BY fts_rowid LIMIT ${FTS5_SWEEP_CHUNK})`;
|
|
3737
|
+
}
|
|
3738
|
+
function sweepChunkSql(p) {
|
|
3739
|
+
const orphans = `SELECT m.fts_rowid FROM ${p}fts_map m
|
|
3740
|
+
WHERE m.fts_rowid > ? AND m.fts_rowid <= ? AND NOT EXISTS (SELECT 1 FROM ${p}entries e WHERE e.id = m.id)`;
|
|
3741
|
+
return [
|
|
3742
|
+
`DELETE FROM ${p}entries_fts WHERE rowid IN (${orphans})`,
|
|
3743
|
+
`DELETE FROM ${p}fts_map WHERE fts_rowid IN (${orphans})`
|
|
3744
|
+
];
|
|
3745
|
+
}
|
|
3746
|
+
|
|
3583
3747
|
// src/db/migrations.ts
|
|
3584
3748
|
var MIGRATIONS = [
|
|
3585
3749
|
{
|
|
@@ -3860,6 +4024,17 @@ var MIGRATIONS = [
|
|
|
3860
4024
|
run: async (db, prefix) => {
|
|
3861
4025
|
await createLegacySourceTypeIndex(db, prefix);
|
|
3862
4026
|
}
|
|
4027
|
+
},
|
|
4028
|
+
{
|
|
4029
|
+
version: 15,
|
|
4030
|
+
description: "fts_pending(id) and fts_pending(entity_id) indexes for the ids-first and entity-targeted drains",
|
|
4031
|
+
run: async (db, prefix) => {
|
|
4032
|
+
const row = await db.getFirstAsync(
|
|
4033
|
+
`SELECT count(*) AS n FROM sqlite_master WHERE type = 'table' AND name = ?`,
|
|
4034
|
+
[`${prefix}fts_pending`]
|
|
4035
|
+
);
|
|
4036
|
+
if (row?.n) await db.execAsync(fts5PendingIndexesDdl(prefix));
|
|
4037
|
+
}
|
|
3863
4038
|
}
|
|
3864
4039
|
];
|
|
3865
4040
|
async function createLegacySourceTypeIndex(db, prefix) {
|
|
@@ -3967,6 +4142,44 @@ async function runSourceRefNormalize(db, prefix, meta) {
|
|
|
3967
4142
|
await runPass(db, prefix, meta, "source_ref_index", SRI_WM_KEY, true);
|
|
3968
4143
|
}
|
|
3969
4144
|
|
|
4145
|
+
// src/services/search/fts5Progress.ts
|
|
4146
|
+
async function writeRebuildStart(tx, prefix, meta) {
|
|
4147
|
+
const e = await tx.getFirstAsync(`SELECT coalesce(max(rowid), 0) AS hi FROM ${prefix}entries`);
|
|
4148
|
+
const m = await tx.getFirstAsync(`SELECT coalesce(max(fts_rowid), 0) AS hi FROM ${prefix}fts_map`);
|
|
4149
|
+
await meta.setMeta(FTS5_REBUILD_WM_KEY, JSON.stringify({ rowid: 0, id: null }), tx);
|
|
4150
|
+
await meta.setMeta(FTS5_REBUILD_SNAPSHOT_KEY, JSON.stringify({ entries: e?.hi ?? 0, map: m?.hi ?? 0 }), tx);
|
|
4151
|
+
await meta.setMeta(FTS5_SWEEP_CURSOR_KEY, "0", tx);
|
|
4152
|
+
}
|
|
4153
|
+
async function readRebuildProgress(meta, tx) {
|
|
4154
|
+
const wm = parseJsonObject(await meta.getMeta(FTS5_REBUILD_WM_KEY, tx));
|
|
4155
|
+
const snap = parseJsonObject(await meta.getMeta(FTS5_REBUILD_SNAPSHOT_KEY, tx));
|
|
4156
|
+
const rawCursor = await meta.getMeta(FTS5_SWEEP_CURSOR_KEY, tx);
|
|
4157
|
+
const cursor = rawCursor === null ? Number.NaN : Number(rawCursor);
|
|
4158
|
+
if (typeof wm?.rowid !== "number" || !(typeof wm.id === "string" || wm.id === null)) return null;
|
|
4159
|
+
if (typeof snap?.entries !== "number" || typeof snap.map !== "number" || !Number.isFinite(cursor)) return null;
|
|
4160
|
+
return { wm: { rowid: wm.rowid, id: wm.id }, snapshot: { entries: snap.entries, map: snap.map }, cursor };
|
|
4161
|
+
}
|
|
4162
|
+
async function reanchorRebuildFingerprint(tx, prefix, deletedIds) {
|
|
4163
|
+
if (deletedIds.length === 0) return;
|
|
4164
|
+
const metaValue = async (key) => (await tx.getFirstAsync(`SELECT value FROM ${prefix}meta WHERE key = ?`, [key]))?.value ?? null;
|
|
4165
|
+
if (await metaValue(FTS5_STATE_KEY) !== "rebuilding") return;
|
|
4166
|
+
const wm = parseJsonObject(await metaValue(FTS5_REBUILD_WM_KEY));
|
|
4167
|
+
if (typeof wm?.rowid !== "number" || typeof wm.id !== "string" || !deletedIds.includes(wm.id)) return;
|
|
4168
|
+
if ((await tx.getFirstAsync(`SELECT id FROM ${prefix}entries WHERE rowid = ?`, [wm.rowid]))?.id === wm.id) return;
|
|
4169
|
+
const below = await tx.getFirstAsync(
|
|
4170
|
+
`SELECT rowid, id FROM ${prefix}entries WHERE rowid < ? ORDER BY rowid DESC LIMIT 1`,
|
|
4171
|
+
[wm.rowid]
|
|
4172
|
+
);
|
|
4173
|
+
const next = below ? { rowid: below.rowid, id: below.id } : { rowid: 0, id: null };
|
|
4174
|
+
await tx.runAsync(`UPDATE ${prefix}meta SET value = ? WHERE key = ?`, [JSON.stringify(next), FTS5_REBUILD_WM_KEY]);
|
|
4175
|
+
}
|
|
4176
|
+
async function restartRebuildAfterVacuum(db, prefix, meta) {
|
|
4177
|
+
await db.withTransactionAsync(async (tx) => {
|
|
4178
|
+
if (await meta.getMeta(FTS5_STATE_KEY, tx) !== "rebuilding") return;
|
|
4179
|
+
await writeRebuildStart(tx, prefix, meta);
|
|
4180
|
+
});
|
|
4181
|
+
}
|
|
4182
|
+
|
|
3970
4183
|
// src/services/librarian/constants.ts
|
|
3971
4184
|
var FUZZY_THRESHOLD = 0.5;
|
|
3972
4185
|
var MIN_TOKENS_TO_QUALIFY = 3;
|
|
@@ -4899,6 +5112,7 @@ var MaintenanceService = class {
|
|
|
4899
5112
|
if (vacuum) {
|
|
4900
5113
|
await resetNormalizeWatermarks(this.metadataRepo);
|
|
4901
5114
|
await this.metadataRepo.vacuum();
|
|
5115
|
+
await restartRebuildAfterVacuum(this.db, this.prefix, this.metadataRepo);
|
|
4902
5116
|
}
|
|
4903
5117
|
await this.searchService.syncEntries(entityId, syncedIds);
|
|
4904
5118
|
return { entries: deletedEntries, tasks: deletedTasks, events: deletedEvents };
|
|
@@ -6146,7 +6360,7 @@ var ImportExportService = class {
|
|
|
6146
6360
|
if (merge) {
|
|
6147
6361
|
await this.searchService.syncEntries(entityId, [...upsertedFactIds]);
|
|
6148
6362
|
} else {
|
|
6149
|
-
await this.searchService.sync(entityId);
|
|
6363
|
+
await this.searchService.sync(entityId, { ids: [.../* @__PURE__ */ new Set([...upsertedFactIds, ...softDeletedFactIds])] });
|
|
6150
6364
|
}
|
|
6151
6365
|
for (const fact of bundle.facts) {
|
|
6152
6366
|
if (!fact.deleted_at && upsertedFactIds.has(fact.id) && !factsWithPreservedBlob.has(fact.id)) {
|
|
@@ -7459,6 +7673,6 @@ var WriteService = class {
|
|
|
7459
7673
|
}
|
|
7460
7674
|
};
|
|
7461
7675
|
|
|
7462
|
-
export { BaseRepository, CURRENT_SCHEMA_VERSION, DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, DEFAULT_MAX_EMBED_CHARS, DiagnosticBuffer, EMBED_CHARS_CEILING, EmbeddingService, HEAL_ANCHORS_PER_CANDIDATE, HEAL_BATCH_SIZE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MAX_EMBED_ATTEMPTS, MIGRATIONS, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, TemporalService, UnavailableIndexStrategy, UsageMeter, WikiBusyError, WikiDraftNotFound, WikiDuplicateHashError, WikiGraphNodeOwnershipConflict, WikiIngestEmptyError, WikiInvalidReadOptions, WikiMigrationRequiredError, WikiParseError, WikiSearchUnavailableError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiSupersedeError, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, chunkText, configureRandomSource, createLegacySourceTypeIndex, detectSchemaVersion, edgeTripleAllowed, emptyManifest, entitySummaryMetaKey, extractSqliteCode, factTokenCost, generateId, isLiveAt, isWikiSearchUnavailableError, liveAtSql, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, packFactsByBudget, parseEmbedding, parseJsonArray, parseJsonObject, probePendingWork, resolveEdgeDefinitions, resolveGrounding, resolveNodeType, runSourceRefNormalize, safeSlice, setupDatabase, typeSatisfies, validateInlineEdges, validateManifest, writeNormalizeStamp };
|
|
7463
|
-
//# sourceMappingURL=chunk-
|
|
7464
|
-
//# sourceMappingURL=chunk-
|
|
7676
|
+
export { BaseRepository, CURRENT_SCHEMA_VERSION, DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, DEFAULT_MAX_EMBED_CHARS, DiagnosticBuffer, EMBED_CHARS_CEILING, EmbeddingService, FTS5_DRAIN_CHUNK, FTS5_REBUILD_SNAPSHOT_KEY, FTS5_REBUILD_WM_KEY, FTS5_RECLAIM_MERGE_PAGES, FTS5_RETIRED_COUNTER_KEY, FTS5_RETIRED_TABLES_KEY, FTS5_STATE_KEY, FTS5_SWEEP_CURSOR_KEY, HEAL_ANCHORS_PER_CANDIDATE, HEAL_BATCH_SIZE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MAX_EMBED_ATTEMPTS, MIGRATIONS, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, TemporalService, UnavailableIndexStrategy, UsageMeter, WikiBusyError, WikiDraftNotFound, WikiDuplicateHashError, WikiGraphNodeOwnershipConflict, WikiIngestEmptyError, WikiInvalidReadOptions, WikiMigrationRequiredError, WikiParseError, WikiSearchUnavailableError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiSupersedeError, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, chunkText, configureRandomSource, createLegacySourceTypeIndex, detectSchemaVersion, drainChunkSql, drainHiSql, edgeTripleAllowed, emptyManifest, entityDrainChunkSql, entityDrainHiSql, entitySummaryMetaKey, extractSqliteCode, factTokenCost, fts5LedgerDropSql, fts5PendingIndexesDdl, fts5TablesDdl, fts5TriggerNames, fts5TriggersDdl, generateId, idDrainChunkSql, idReindexChunkSql, isLiveAt, isRetiredTableName, isWikiSearchUnavailableError, liveAtSql, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, packFactsByBudget, parseEmbedding, parseJsonArray, parseJsonObject, probePendingWork, readRebuildProgress, reanchorRebuildFingerprint, resolveEdgeDefinitions, resolveGrounding, resolveNodeType, retiredTableName, runSourceRefNormalize, safeSlice, setupDatabase, sweepChunkSql, sweepHiSql, typeSatisfies, validateInlineEdges, validateManifest, windowChunkSql, windowHiSql, writeNormalizeStamp, writeRebuildStart };
|
|
7677
|
+
//# sourceMappingURL=chunk-PCRYIUV5.mjs.map
|
|
7678
|
+
//# sourceMappingURL=chunk-PCRYIUV5.mjs.map
|