@equationalapplications/core-llm-wiki 7.11.2 → 7.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -0
- package/dist/{chunk-522SCN2E.mjs → chunk-PCRYIUV5.mjs} +2299 -1408
- package/dist/chunk-PCRYIUV5.mjs.map +1 -0
- package/dist/index.d.mts +10 -3
- package/dist/index.d.ts +10 -3
- package/dist/index.js +5336 -4465
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +645 -669
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-B0UtFCq1.d.mts → testing-CqsXv2_l.d.mts} +195 -23
- package/dist/{testing-B0UtFCq1.d.ts → testing-CqsXv2_l.d.ts} +195 -23
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +494 -35
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +3 -3
- package/dist/chunk-522SCN2E.mjs.map +0 -1
package/dist/index.mjs
CHANGED
|
@@ -1,187 +1,25 @@
|
|
|
1
|
-
import { BaseRepository, WikiGraphNodeOwnershipConflict, liveAtSql, __privateAdd, MetadataRepository, EmbeddingService, SearchService, TemporalService, JobManager, PromptService, resolveGrounding, IngestionService, MaintenanceService, ImportExportService, RetrievalService, WriteService, __privateGet, __privateSet, normalizeSourceRef, normalizeSourceHash, entitySummaryMetaKey, WikiSourceRefHashCollision, DiagnosticBuffer, validateManifest, WikiDraftNotFound, generateId, emptyManifest, resolveNodeType, WikiStrictOntologyViolation, validateInlineEdges, resolveEdgeDefinitions, typeSatisfies, normalizeTitleKey, edgeTripleAllowed, MAX_EMBED_ATTEMPTS, UsageMeter, WikiBusyError, WikiInvalidReadOptions, isLiveAt, packFactsByBudget, factTokenCost,
|
|
2
|
-
export { DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, DEFAULT_MAX_EMBED_CHARS, EMBED_CHARS_CEILING, HEAL_ANCHORS_PER_CANDIDATE, HEAL_BATCH_SIZE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, WikiBusyError, WikiDraftNotFound, WikiDuplicateHashError, WikiGraphNodeOwnershipConflict, WikiIngestEmptyError, WikiInvalidReadOptions, WikiParseError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiSupersedeError, WikiTransactionError, chunkText, configureRandomSource, parseEmbedding, safeSlice, validateManifest } from './chunk-
|
|
1
|
+
import { BaseRepository, WikiGraphNodeOwnershipConflict, parseJsonObject, parseJsonArray, liveAtSql, reanchorRebuildFingerprint, __privateAdd, MetadataRepository, EmbeddingService, SearchService, TemporalService, JobManager, PromptService, resolveGrounding, IngestionService, MaintenanceService, ImportExportService, RetrievalService, WriteService, __privateGet, __privateSet, createLegacySourceTypeIndex, writeNormalizeStamp, CURRENT_SCHEMA_VERSION, detectSchemaVersion, MIGRATIONS, runSourceRefNormalize, normalizeSourceRef, normalizeSourceHash, entitySummaryMetaKey, WikiSourceRefHashCollision, DiagnosticBuffer, validateManifest, WikiDraftNotFound, generateId, emptyManifest, resolveNodeType, WikiStrictOntologyViolation, validateInlineEdges, resolveEdgeDefinitions, typeSatisfies, normalizeTitleKey, edgeTripleAllowed, MAX_EMBED_ATTEMPTS, UsageMeter, isWikiSearchUnavailableError, WikiBusyError, WikiInvalidReadOptions, isLiveAt, packFactsByBudget, factTokenCost, probePendingWork, WikiMigrationRequiredError, setupDatabase, UnavailableIndexStrategy, FTS5_STATE_KEY, fts5LedgerDropSql, fts5TablesDdl, fts5PendingIndexesDdl, fts5TriggersDdl, writeRebuildStart, FTS5_RETIRED_COUNTER_KEY, retiredTableName, FTS5_RETIRED_TABLES_KEY, isRetiredTableName, idReindexChunkSql, readRebuildProgress, FTS5_DRAIN_CHUNK, FTS5_RECLAIM_MERGE_PAGES, windowHiSql, windowChunkSql, FTS5_REBUILD_WM_KEY, sweepHiSql, sweepChunkSql, FTS5_SWEEP_CURSOR_KEY, drainHiSql, drainChunkSql, FTS5_REBUILD_SNAPSHOT_KEY, idDrainChunkSql, entityDrainHiSql, entityDrainChunkSql, fts5TriggerNames, extractSqliteCode, WikiTransactionError } from './chunk-PCRYIUV5.mjs';
|
|
2
|
+
export { DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, DEFAULT_MAX_EMBED_CHARS, EMBED_CHARS_CEILING, HEAL_ANCHORS_PER_CANDIDATE, HEAL_BATCH_SIZE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, WikiBusyError, WikiDraftNotFound, WikiDuplicateHashError, WikiGraphNodeOwnershipConflict, WikiIngestEmptyError, WikiInvalidReadOptions, WikiMigrationRequiredError, WikiParseError, WikiSearchUnavailableError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiSupersedeError, WikiTransactionError, chunkText, configureRandomSource, isWikiSearchUnavailableError, parseEmbedding, safeSlice, validateManifest } from './chunk-PCRYIUV5.mjs';
|
|
3
3
|
import { deriveTrustTier, isStaleAfter, appendRelatedSection, buildConceptDocument, buildLogMd, buildEntityIndexMd, buildRootIndexMd, isAllowedOkfPath, parseRootIndexMd, parseEntityIndexMd, parseConcept, splitRelatedSection, extractMarkdownLinks, parseLogMd, parseEventIdComment, latestVerified, formatSourcesJson, appendEventIdComment, parseVerifiedFlexible, parseCitationsList } from '@equationalapplications/core-okf';
|
|
4
4
|
import MiniSearch from 'minisearch';
|
|
5
5
|
|
|
6
|
-
// src/db/
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
source_type TEXT NOT NULL DEFAULT 'librarian_inferred',
|
|
17
|
-
source_hash TEXT,
|
|
18
|
-
source_ref TEXT,
|
|
19
|
-
created_at INTEGER NOT NULL,
|
|
20
|
-
updated_at INTEGER NOT NULL,
|
|
21
|
-
last_accessed_at INTEGER,
|
|
22
|
-
access_count INTEGER NOT NULL DEFAULT 0,
|
|
23
|
-
deleted_at INTEGER,
|
|
24
|
-
embedding TEXT,
|
|
25
|
-
embedding_blob BLOB,
|
|
26
|
-
okf_type TEXT,
|
|
27
|
-
ontology_checked_at INTEGER,
|
|
28
|
-
heal_checked_at INTEGER,
|
|
29
|
-
lifecycle_status TEXT NOT NULL DEFAULT 'stable',
|
|
30
|
-
stale_after INTEGER,
|
|
31
|
-
generated_by TEXT,
|
|
32
|
-
last_verified_at INTEGER,
|
|
33
|
-
last_verified_by TEXT,
|
|
34
|
-
okf_sources TEXT,
|
|
35
|
-
okf_verified TEXT,
|
|
36
|
-
okf_usage_window TEXT,
|
|
37
|
-
embedding_failed_at INTEGER,
|
|
38
|
-
embedding_failure_kind TEXT,
|
|
39
|
-
embedding_attempts INTEGER NOT NULL DEFAULT 0,
|
|
40
|
-
valid_from INTEGER,
|
|
41
|
-
valid_to INTEGER,
|
|
42
|
-
superseded_by TEXT,
|
|
43
|
-
superseded_at INTEGER
|
|
44
|
-
);
|
|
45
|
-
|
|
46
|
-
CREATE INDEX IF NOT EXISTS ${prefix}entries_entity_idx ON ${prefix}entries(entity_id);
|
|
47
|
-
CREATE INDEX IF NOT EXISTS ${prefix}entries_source_ref_idx ON ${prefix}entries(entity_id, source_ref);
|
|
48
|
-
CREATE INDEX IF NOT EXISTS ${prefix}entries_source_hash_idx ON ${prefix}entries(entity_id, source_hash) WHERE source_hash IS NOT NULL;
|
|
49
|
-
CREATE INDEX IF NOT EXISTS ${prefix}entries_updated_idx ON ${prefix}entries(updated_at DESC);
|
|
50
|
-
|
|
51
|
-
-- source_ref_index: per-(entity, source_hash) record of the canonical sourceRef
|
|
52
|
-
-- currently holding that hash. The partial UNIQUE index on (entity_id, source_hash)
|
|
53
|
-
-- WHERE deleted_at IS NULL enforces the sourceRef-level TOCTOU-race invariant;
|
|
54
|
-
-- entries-level uniqueness cannot express it because a single ingestDocument call
|
|
55
|
-
-- writes N facts that all share (entity_id, source_ref, source_hash). See
|
|
56
|
-
-- docs/superpowers/specs/2026-08-07-dependabot-concurrency-release-hygiene-design.md \xA7B1.
|
|
57
|
-
CREATE TABLE IF NOT EXISTS ${prefix}source_ref_index (
|
|
58
|
-
id TEXT PRIMARY KEY,
|
|
59
|
-
entity_id TEXT NOT NULL,
|
|
60
|
-
source_hash TEXT NOT NULL,
|
|
61
|
-
source_ref TEXT NOT NULL,
|
|
62
|
-
created_at INTEGER NOT NULL,
|
|
63
|
-
deleted_at INTEGER
|
|
64
|
-
);
|
|
65
|
-
CREATE UNIQUE INDEX IF NOT EXISTS ${prefix}idx_source_ref_hash
|
|
66
|
-
ON ${prefix}source_ref_index (entity_id, source_hash)
|
|
67
|
-
WHERE deleted_at IS NULL;
|
|
68
|
-
|
|
69
|
-
CREATE TABLE IF NOT EXISTS ${prefix}tasks (
|
|
70
|
-
id TEXT PRIMARY KEY,
|
|
71
|
-
entity_id TEXT NOT NULL,
|
|
72
|
-
description TEXT NOT NULL,
|
|
73
|
-
status TEXT NOT NULL DEFAULT 'pending',
|
|
74
|
-
priority INTEGER NOT NULL DEFAULT 0,
|
|
75
|
-
created_at INTEGER NOT NULL,
|
|
76
|
-
updated_at INTEGER NOT NULL,
|
|
77
|
-
resolved_at INTEGER,
|
|
78
|
-
deleted_at INTEGER,
|
|
79
|
-
okf_type TEXT,
|
|
80
|
-
lifecycle_status TEXT NOT NULL DEFAULT 'stable',
|
|
81
|
-
stale_after INTEGER,
|
|
82
|
-
generated_by TEXT,
|
|
83
|
-
last_verified_at INTEGER,
|
|
84
|
-
last_verified_by TEXT,
|
|
85
|
-
okf_sources TEXT,
|
|
86
|
-
okf_verified TEXT,
|
|
87
|
-
okf_usage_window TEXT
|
|
88
|
-
);
|
|
89
|
-
|
|
90
|
-
CREATE INDEX IF NOT EXISTS ${prefix}tasks_entity_idx ON ${prefix}tasks(entity_id, status);
|
|
91
|
-
|
|
92
|
-
CREATE TABLE IF NOT EXISTS ${prefix}edges (
|
|
93
|
-
id TEXT PRIMARY KEY,
|
|
94
|
-
entity_id TEXT NOT NULL,
|
|
95
|
-
source_id TEXT NOT NULL,
|
|
96
|
-
target_id TEXT NOT NULL,
|
|
97
|
-
edge_type TEXT NOT NULL,
|
|
98
|
-
created_at INTEGER NOT NULL,
|
|
99
|
-
UNIQUE(entity_id, source_id, target_id, edge_type)
|
|
100
|
-
);
|
|
101
|
-
|
|
102
|
-
CREATE INDEX IF NOT EXISTS ${prefix}edges_entity_id_idx ON ${prefix}edges(entity_id, id);
|
|
103
|
-
|
|
104
|
-
CREATE TABLE IF NOT EXISTS ${prefix}events (
|
|
105
|
-
id TEXT PRIMARY KEY,
|
|
106
|
-
entity_id TEXT NOT NULL,
|
|
107
|
-
event_type TEXT NOT NULL,
|
|
108
|
-
summary TEXT NOT NULL,
|
|
109
|
-
related_entry_id TEXT,
|
|
110
|
-
created_at INTEGER NOT NULL,
|
|
111
|
-
occurred_at INTEGER
|
|
112
|
-
);
|
|
113
|
-
|
|
114
|
-
CREATE INDEX IF NOT EXISTS ${prefix}events_entity_idx ON ${prefix}events(entity_id, created_at DESC);
|
|
115
|
-
|
|
116
|
-
CREATE TABLE IF NOT EXISTS ${prefix}checkpoints (
|
|
117
|
-
entity_id TEXT PRIMARY KEY,
|
|
118
|
-
heal_checkpoint INTEGER NOT NULL DEFAULT 0,
|
|
119
|
-
memory_checkpoint INTEGER NOT NULL DEFAULT 0,
|
|
120
|
-
librarian_watermark_at INTEGER,
|
|
121
|
-
librarian_watermark_id TEXT
|
|
122
|
-
);
|
|
123
|
-
|
|
124
|
-
CREATE TABLE IF NOT EXISTS ${prefix}entity_manifests (
|
|
125
|
-
entity_id TEXT PRIMARY KEY,
|
|
126
|
-
mode TEXT NOT NULL DEFAULT 'off',
|
|
127
|
-
manifest_json TEXT NOT NULL DEFAULT '{"node_types":[],"edge_types":[]}',
|
|
128
|
-
updated_at INTEGER NOT NULL
|
|
129
|
-
);
|
|
130
|
-
|
|
131
|
-
CREATE TABLE IF NOT EXISTS ${prefix}meta (
|
|
132
|
-
key TEXT PRIMARY KEY,
|
|
133
|
-
value TEXT NOT NULL
|
|
134
|
-
);
|
|
135
|
-
|
|
136
|
-
CREATE TABLE IF NOT EXISTS ${prefix}outbox (
|
|
137
|
-
id TEXT PRIMARY KEY,
|
|
138
|
-
entity_id TEXT NOT NULL,
|
|
139
|
-
table_name TEXT NOT NULL,
|
|
140
|
-
record_id TEXT NOT NULL,
|
|
141
|
-
operation TEXT NOT NULL,
|
|
142
|
-
payload TEXT NOT NULL,
|
|
143
|
-
created_at INTEGER NOT NULL
|
|
144
|
-
);
|
|
145
|
-
|
|
146
|
-
CREATE INDEX IF NOT EXISTS ${prefix}outbox_entity_id_created_at
|
|
147
|
-
ON ${prefix}outbox (entity_id, created_at);
|
|
148
|
-
|
|
149
|
-
CREATE INDEX IF NOT EXISTS ${prefix}outbox_created_at
|
|
150
|
-
ON ${prefix}outbox (created_at);
|
|
151
|
-
`);
|
|
152
|
-
await createOkfV02IndexesIfColumnsExist(db, prefix);
|
|
153
|
-
await createTemporalIndexesIfColumnsExist(db, prefix);
|
|
154
|
-
}
|
|
155
|
-
async function createOkfV02IndexesIfColumnsExist(db, prefix) {
|
|
156
|
-
for (const table of ["entries", "tasks"]) {
|
|
157
|
-
const cols = await db.getAllAsync(
|
|
158
|
-
`PRAGMA table_info(${prefix}${table})`
|
|
159
|
-
);
|
|
160
|
-
const colNames = new Set(cols.map((c) => c.name));
|
|
161
|
-
if (colNames.has("lifecycle_status")) {
|
|
162
|
-
await db.execAsync(`CREATE INDEX IF NOT EXISTS ${prefix}${table}_lifecycle_status_idx ON ${prefix}${table}(lifecycle_status);`);
|
|
163
|
-
}
|
|
164
|
-
if (colNames.has("stale_after")) {
|
|
165
|
-
await db.execAsync(`CREATE INDEX IF NOT EXISTS ${prefix}${table}_stale_after_idx ON ${prefix}${table}(stale_after);`);
|
|
166
|
-
}
|
|
167
|
-
if (colNames.has("last_verified_at")) {
|
|
168
|
-
await db.execAsync(`CREATE INDEX IF NOT EXISTS ${prefix}${table}_last_verified_at_idx ON ${prefix}${table}(last_verified_at);`);
|
|
6
|
+
// src/db/migrationGate.ts
|
|
7
|
+
var MIGRATION_POLICIES = ["allow", "refuse", "auto"];
|
|
8
|
+
async function setupDatabaseWithGate(db, prefix, meta, migrations = "allow") {
|
|
9
|
+
if (!MIGRATION_POLICIES.includes(migrations)) {
|
|
10
|
+
throw new TypeError(`Invalid migrations policy ${JSON.stringify(migrations)}; expected 'allow', 'refuse' or 'auto'.`);
|
|
11
|
+
}
|
|
12
|
+
if (migrations !== "allow") {
|
|
13
|
+
const pending = await probePendingWork(db, meta, prefix);
|
|
14
|
+
if (pending && (migrations === "refuse" || pending.type === "schema_migration")) {
|
|
15
|
+
throw new WikiMigrationRequiredError(describePending(pending, migrations), pending);
|
|
169
16
|
}
|
|
170
17
|
}
|
|
18
|
+
return setupDatabase(db, prefix);
|
|
171
19
|
}
|
|
172
|
-
|
|
173
|
-
const
|
|
174
|
-
|
|
175
|
-
if (names.has("superseded_by")) {
|
|
176
|
-
await db.execAsync(
|
|
177
|
-
`CREATE INDEX IF NOT EXISTS ${prefix}entries_superseded_idx ON ${prefix}entries(entity_id, superseded_by) WHERE superseded_by IS NOT NULL;`
|
|
178
|
-
);
|
|
179
|
-
}
|
|
180
|
-
if (names.has("valid_from") && names.has("valid_to")) {
|
|
181
|
-
await db.execAsync(
|
|
182
|
-
`CREATE INDEX IF NOT EXISTS ${prefix}entries_temporal_idx ON ${prefix}entries(entity_id) WHERE valid_from IS NOT NULL OR valid_to IS NOT NULL;`
|
|
183
|
-
);
|
|
184
|
-
}
|
|
20
|
+
function describePending(pending, policy) {
|
|
21
|
+
const what = pending.type === "schema_migration" ? `schema migration v${pending.fromVersion} -> v${pending.toVersion} is pending` : pending.entriesWatermark === 0 && pending.sourceRefIndexWatermark === 0 ? "source_ref normalize has not run on this store (normalize stamp missing or outdated; may complete with zero rewrites)" : "source_ref normalize was interrupted and will resume from its watermark";
|
|
22
|
+
return `Migration refused (migrations: '${policy}'): ${what}. Open once with setup({ migrations: 'allow' }) outside any time-critical path to complete it.`;
|
|
185
23
|
}
|
|
186
24
|
|
|
187
25
|
// src/db/serializedAdapter.ts
|
|
@@ -227,314 +65,6 @@ function withSerializedTransactions(db) {
|
|
|
227
65
|
}
|
|
228
66
|
});
|
|
229
67
|
}
|
|
230
|
-
|
|
231
|
-
// src/db/migrations.ts
|
|
232
|
-
var MIGRATIONS = [
|
|
233
|
-
{
|
|
234
|
-
version: 1,
|
|
235
|
-
description: "Rebuild FTS5 with porter unicode61 tokenizer (superseded by v2)",
|
|
236
|
-
run: async (_db, _prefix) => {
|
|
237
|
-
}
|
|
238
|
-
},
|
|
239
|
-
{
|
|
240
|
-
version: 2,
|
|
241
|
-
description: "Remove FTS5; add embedding column for semantic retrieval",
|
|
242
|
-
run: async (db, prefix) => {
|
|
243
|
-
await db.withTransactionAsync(async (tx) => {
|
|
244
|
-
await tx.execAsync(`
|
|
245
|
-
DROP TRIGGER IF EXISTS ${prefix}entries_ai;
|
|
246
|
-
DROP TRIGGER IF EXISTS ${prefix}entries_ad;
|
|
247
|
-
DROP TRIGGER IF EXISTS ${prefix}entries_au;
|
|
248
|
-
DROP TABLE IF EXISTS ${prefix}entries_fts;
|
|
249
|
-
`);
|
|
250
|
-
});
|
|
251
|
-
const cols = await db.getAllAsync(
|
|
252
|
-
`PRAGMA table_info(${prefix}entries)`
|
|
253
|
-
);
|
|
254
|
-
if (!cols.some((c) => c.name === "embedding")) {
|
|
255
|
-
await db.execAsync(`ALTER TABLE ${prefix}entries ADD COLUMN embedding TEXT`);
|
|
256
|
-
}
|
|
257
|
-
}
|
|
258
|
-
},
|
|
259
|
-
{
|
|
260
|
-
version: 3,
|
|
261
|
-
description: "Add embedding_blob BLOB column for Float32Array vector storage",
|
|
262
|
-
run: async (db, prefix) => {
|
|
263
|
-
const cols = await db.getAllAsync(
|
|
264
|
-
`PRAGMA table_info(${prefix}entries)`
|
|
265
|
-
);
|
|
266
|
-
if (!cols.some((c) => c.name === "embedding_blob")) {
|
|
267
|
-
await db.execAsync(
|
|
268
|
-
`ALTER TABLE ${prefix}entries ADD COLUMN embedding_blob BLOB`
|
|
269
|
-
);
|
|
270
|
-
}
|
|
271
|
-
}
|
|
272
|
-
},
|
|
273
|
-
{
|
|
274
|
-
version: 4,
|
|
275
|
-
description: "Create outbox table for change data capture",
|
|
276
|
-
run: async (db, prefix) => {
|
|
277
|
-
await db.execAsync(`
|
|
278
|
-
CREATE TABLE IF NOT EXISTS ${prefix}outbox (
|
|
279
|
-
id TEXT PRIMARY KEY,
|
|
280
|
-
entity_id TEXT NOT NULL,
|
|
281
|
-
table_name TEXT NOT NULL,
|
|
282
|
-
record_id TEXT NOT NULL,
|
|
283
|
-
operation TEXT NOT NULL,
|
|
284
|
-
payload TEXT NOT NULL,
|
|
285
|
-
created_at INTEGER NOT NULL
|
|
286
|
-
);
|
|
287
|
-
CREATE INDEX IF NOT EXISTS ${prefix}outbox_entity_id_created_at
|
|
288
|
-
ON ${prefix}outbox (entity_id, created_at);
|
|
289
|
-
`);
|
|
290
|
-
}
|
|
291
|
-
},
|
|
292
|
-
{
|
|
293
|
-
version: 5,
|
|
294
|
-
description: "Add okf_type to entries/tasks for OKF type fidelity; create edges table for OKF graph import",
|
|
295
|
-
run: async (db, prefix) => {
|
|
296
|
-
for (const table of ["entries", "tasks"]) {
|
|
297
|
-
const cols = await db.getAllAsync(
|
|
298
|
-
`PRAGMA table_info(${prefix}${table})`
|
|
299
|
-
);
|
|
300
|
-
if (!cols.some((c) => c.name === "okf_type")) {
|
|
301
|
-
await db.execAsync(`ALTER TABLE ${prefix}${table} ADD COLUMN okf_type TEXT`);
|
|
302
|
-
}
|
|
303
|
-
}
|
|
304
|
-
await db.execAsync(`
|
|
305
|
-
CREATE TABLE IF NOT EXISTS ${prefix}edges (
|
|
306
|
-
id TEXT PRIMARY KEY,
|
|
307
|
-
entity_id TEXT NOT NULL,
|
|
308
|
-
source_id TEXT NOT NULL,
|
|
309
|
-
target_id TEXT NOT NULL,
|
|
310
|
-
edge_type TEXT NOT NULL,
|
|
311
|
-
created_at INTEGER NOT NULL,
|
|
312
|
-
UNIQUE(entity_id, source_id, target_id, edge_type)
|
|
313
|
-
);
|
|
314
|
-
CREATE INDEX IF NOT EXISTS ${prefix}edges_entity_idx ON ${prefix}edges (entity_id);
|
|
315
|
-
`);
|
|
316
|
-
}
|
|
317
|
-
},
|
|
318
|
-
{
|
|
319
|
-
version: 6,
|
|
320
|
-
description: "Add entity_manifests table for per-entity ontology state",
|
|
321
|
-
run: async (db, prefix) => {
|
|
322
|
-
await db.execAsync(`
|
|
323
|
-
CREATE TABLE IF NOT EXISTS ${prefix}entity_manifests (
|
|
324
|
-
entity_id TEXT PRIMARY KEY,
|
|
325
|
-
mode TEXT NOT NULL DEFAULT 'off',
|
|
326
|
-
manifest_json TEXT NOT NULL DEFAULT '{"node_types":[],"edge_types":[]}',
|
|
327
|
-
updated_at INTEGER NOT NULL
|
|
328
|
-
);
|
|
329
|
-
`);
|
|
330
|
-
}
|
|
331
|
-
},
|
|
332
|
-
{
|
|
333
|
-
version: 7,
|
|
334
|
-
description: "Add ontology_checked_at to entries for ontology backfill recheck cooldown",
|
|
335
|
-
run: async (db, prefix) => {
|
|
336
|
-
const cols = await db.getAllAsync(
|
|
337
|
-
`PRAGMA table_info(${prefix}entries)`
|
|
338
|
-
);
|
|
339
|
-
if (!cols.some((c) => c.name === "ontology_checked_at")) {
|
|
340
|
-
await db.execAsync(
|
|
341
|
-
`ALTER TABLE ${prefix}entries ADD COLUMN ontology_checked_at INTEGER`
|
|
342
|
-
);
|
|
343
|
-
}
|
|
344
|
-
}
|
|
345
|
-
},
|
|
346
|
-
{
|
|
347
|
-
version: 8,
|
|
348
|
-
description: "Add heal_checked_at to entries for heal recheck cooldown",
|
|
349
|
-
run: async (db, prefix) => {
|
|
350
|
-
const cols = await db.getAllAsync(
|
|
351
|
-
`PRAGMA table_info(${prefix}entries)`
|
|
352
|
-
);
|
|
353
|
-
if (!cols.some((c) => c.name === "heal_checked_at")) {
|
|
354
|
-
await db.execAsync(
|
|
355
|
-
`ALTER TABLE ${prefix}entries ADD COLUMN heal_checked_at INTEGER`
|
|
356
|
-
);
|
|
357
|
-
}
|
|
358
|
-
}
|
|
359
|
-
},
|
|
360
|
-
{
|
|
361
|
-
version: 9,
|
|
362
|
-
description: "add_source_ref_index",
|
|
363
|
-
run: async (db, prefix) => {
|
|
364
|
-
const duplicates = await db.getAllAsync(
|
|
365
|
-
`SELECT entity_id, source_hash, COUNT(DISTINCT source_ref) AS n_refs
|
|
366
|
-
FROM ${prefix}entries
|
|
367
|
-
WHERE deleted_at IS NULL AND source_hash IS NOT NULL
|
|
368
|
-
GROUP BY entity_id, source_hash
|
|
369
|
-
HAVING COUNT(DISTINCT source_ref) > 1`
|
|
370
|
-
);
|
|
371
|
-
if (duplicates.length > 0) {
|
|
372
|
-
const sample = duplicates.slice(0, 5).map((d) => `(entity_id=${d.entity_id}, source_hash=${d.source_hash.slice(0, 12)}\u2026, n_refs=${d.n_refs})`).join(", ");
|
|
373
|
-
throw new Error(
|
|
374
|
-
`Migration v9 (add_source_ref_index) failed: existing live rows have multiple sourceRefs sharing a hash. Found ${duplicates.length} duplicate (entity_id, source_hash) groups. First ${Math.min(5, duplicates.length)}: ${sample}. Resolve each by calling forget({ sourceRef: <loser> }) for the offending sourceRef, then re-run setup.`
|
|
375
|
-
);
|
|
376
|
-
}
|
|
377
|
-
await db.execAsync(
|
|
378
|
-
`CREATE TABLE IF NOT EXISTS ${prefix}source_ref_index (
|
|
379
|
-
id TEXT PRIMARY KEY,
|
|
380
|
-
entity_id TEXT NOT NULL,
|
|
381
|
-
source_hash TEXT NOT NULL,
|
|
382
|
-
source_ref TEXT NOT NULL,
|
|
383
|
-
created_at INTEGER NOT NULL,
|
|
384
|
-
deleted_at INTEGER
|
|
385
|
-
);
|
|
386
|
-
CREATE UNIQUE INDEX IF NOT EXISTS ${prefix}idx_source_ref_hash
|
|
387
|
-
ON ${prefix}source_ref_index (entity_id, source_hash)
|
|
388
|
-
WHERE deleted_at IS NULL;`
|
|
389
|
-
);
|
|
390
|
-
await db.execAsync(
|
|
391
|
-
`INSERT OR IGNORE INTO ${prefix}source_ref_index (id, entity_id, source_hash, source_ref, created_at, deleted_at)
|
|
392
|
-
SELECT
|
|
393
|
-
'sri:' || entity_id || ':' || source_hash,
|
|
394
|
-
entity_id,
|
|
395
|
-
source_hash,
|
|
396
|
-
source_ref,
|
|
397
|
-
updated_at,
|
|
398
|
-
NULL
|
|
399
|
-
FROM (
|
|
400
|
-
SELECT
|
|
401
|
-
entity_id, source_hash, source_ref, updated_at,
|
|
402
|
-
ROW_NUMBER() OVER (
|
|
403
|
-
PARTITION BY entity_id, source_hash
|
|
404
|
-
ORDER BY updated_at ASC, id ASC
|
|
405
|
-
) AS rn
|
|
406
|
-
FROM ${prefix}entries
|
|
407
|
-
WHERE deleted_at IS NULL
|
|
408
|
-
AND source_hash IS NOT NULL
|
|
409
|
-
AND source_ref IS NOT NULL
|
|
410
|
-
) ranked
|
|
411
|
-
WHERE rn = 1;`
|
|
412
|
-
);
|
|
413
|
-
}
|
|
414
|
-
},
|
|
415
|
-
{
|
|
416
|
-
version: 10,
|
|
417
|
-
description: "OKF v0.2: add lifecycle_status, stale_after, generated_by, last_verified_*, okf_sources, okf_verified, okf_usage_window to entries and tasks",
|
|
418
|
-
run: async (db, prefix) => {
|
|
419
|
-
for (const table of ["entries", "tasks"]) {
|
|
420
|
-
const cols = await db.getAllAsync(
|
|
421
|
-
`PRAGMA table_info(${prefix}${table})`
|
|
422
|
-
);
|
|
423
|
-
const existing = new Set(cols.map((c) => c.name));
|
|
424
|
-
const adds = [
|
|
425
|
-
["lifecycle_status", `ALTER TABLE ${prefix}${table} ADD COLUMN lifecycle_status TEXT NOT NULL DEFAULT 'stable'`],
|
|
426
|
-
["stale_after", `ALTER TABLE ${prefix}${table} ADD COLUMN stale_after INTEGER`],
|
|
427
|
-
["generated_by", `ALTER TABLE ${prefix}${table} ADD COLUMN generated_by TEXT`],
|
|
428
|
-
["last_verified_at", `ALTER TABLE ${prefix}${table} ADD COLUMN last_verified_at INTEGER`],
|
|
429
|
-
["last_verified_by", `ALTER TABLE ${prefix}${table} ADD COLUMN last_verified_by TEXT`],
|
|
430
|
-
["okf_sources", `ALTER TABLE ${prefix}${table} ADD COLUMN okf_sources TEXT`],
|
|
431
|
-
["okf_verified", `ALTER TABLE ${prefix}${table} ADD COLUMN okf_verified TEXT`],
|
|
432
|
-
["okf_usage_window", `ALTER TABLE ${prefix}${table} ADD COLUMN okf_usage_window TEXT`]
|
|
433
|
-
];
|
|
434
|
-
for (const [name, sql] of adds) {
|
|
435
|
-
if (!existing.has(name)) await db.execAsync(sql);
|
|
436
|
-
}
|
|
437
|
-
}
|
|
438
|
-
await db.execAsync(`
|
|
439
|
-
CREATE INDEX IF NOT EXISTS ${prefix}entries_lifecycle_status_idx ON ${prefix}entries(lifecycle_status);
|
|
440
|
-
CREATE INDEX IF NOT EXISTS ${prefix}entries_stale_after_idx ON ${prefix}entries(stale_after);
|
|
441
|
-
CREATE INDEX IF NOT EXISTS ${prefix}entries_last_verified_at_idx ON ${prefix}entries(last_verified_at);
|
|
442
|
-
CREATE INDEX IF NOT EXISTS ${prefix}tasks_lifecycle_status_idx ON ${prefix}tasks(lifecycle_status);
|
|
443
|
-
CREATE INDEX IF NOT EXISTS ${prefix}tasks_stale_after_idx ON ${prefix}tasks(stale_after);
|
|
444
|
-
CREATE INDEX IF NOT EXISTS ${prefix}tasks_last_verified_at_idx ON ${prefix}tasks(last_verified_at);
|
|
445
|
-
`);
|
|
446
|
-
}
|
|
447
|
-
},
|
|
448
|
-
{
|
|
449
|
-
version: 11,
|
|
450
|
-
description: "Add embedding_failed_at, embedding_failure_kind, embedding_attempts to entries for embed retry state",
|
|
451
|
-
run: async (db, prefix) => {
|
|
452
|
-
const cols = await db.getAllAsync(
|
|
453
|
-
`PRAGMA table_info(${prefix}entries)`
|
|
454
|
-
);
|
|
455
|
-
const existing = new Set(cols.map((c) => c.name));
|
|
456
|
-
const adds = [
|
|
457
|
-
["embedding_failed_at", `ALTER TABLE ${prefix}entries ADD COLUMN embedding_failed_at INTEGER`],
|
|
458
|
-
["embedding_failure_kind", `ALTER TABLE ${prefix}entries ADD COLUMN embedding_failure_kind TEXT`],
|
|
459
|
-
["embedding_attempts", `ALTER TABLE ${prefix}entries ADD COLUMN embedding_attempts INTEGER NOT NULL DEFAULT 0`]
|
|
460
|
-
];
|
|
461
|
-
for (const [name, sql] of adds) {
|
|
462
|
-
if (!existing.has(name)) await db.execAsync(sql);
|
|
463
|
-
}
|
|
464
|
-
}
|
|
465
|
-
},
|
|
466
|
-
{
|
|
467
|
-
version: 12,
|
|
468
|
-
description: "Replace edges_entity_idx with composite edges(entity_id, id) for keyset paging",
|
|
469
|
-
run: async (db, prefix) => {
|
|
470
|
-
await db.execAsync(`
|
|
471
|
-
CREATE INDEX IF NOT EXISTS ${prefix}edges_entity_id_idx ON ${prefix}edges(entity_id, id);
|
|
472
|
-
DROP INDEX IF EXISTS ${prefix}edges_entity_idx;
|
|
473
|
-
`);
|
|
474
|
-
}
|
|
475
|
-
},
|
|
476
|
-
{
|
|
477
|
-
version: 13,
|
|
478
|
-
description: "Temporal facts: valid_from/valid_to/superseded_by/superseded_at on entries, occurred_at on events, librarian watermark on checkpoints",
|
|
479
|
-
run: async (db, prefix) => {
|
|
480
|
-
const adds = [
|
|
481
|
-
["entries", "valid_from", "INTEGER"],
|
|
482
|
-
["entries", "valid_to", "INTEGER"],
|
|
483
|
-
["entries", "superseded_by", "TEXT"],
|
|
484
|
-
["entries", "superseded_at", "INTEGER"],
|
|
485
|
-
["events", "occurred_at", "INTEGER"],
|
|
486
|
-
["checkpoints", "librarian_watermark_at", "INTEGER"],
|
|
487
|
-
["checkpoints", "librarian_watermark_id", "TEXT"]
|
|
488
|
-
];
|
|
489
|
-
const existingByTable = /* @__PURE__ */ new Map();
|
|
490
|
-
for (const [table, column, type] of adds) {
|
|
491
|
-
let existing = existingByTable.get(table);
|
|
492
|
-
if (!existing) {
|
|
493
|
-
const cols = await db.getAllAsync(`PRAGMA table_info(${prefix}${table})`);
|
|
494
|
-
existing = new Set(cols.map((c) => c.name));
|
|
495
|
-
existingByTable.set(table, existing);
|
|
496
|
-
}
|
|
497
|
-
if (!existing.has(column)) {
|
|
498
|
-
await db.execAsync(`ALTER TABLE ${prefix}${table} ADD COLUMN ${column} ${type}`);
|
|
499
|
-
existing.add(column);
|
|
500
|
-
}
|
|
501
|
-
}
|
|
502
|
-
await createTemporalIndexesIfColumnsExist(db, prefix);
|
|
503
|
-
}
|
|
504
|
-
}
|
|
505
|
-
];
|
|
506
|
-
for (let i = 1; i < MIGRATIONS.length; i++) {
|
|
507
|
-
if (MIGRATIONS[i].version <= MIGRATIONS[i - 1].version) {
|
|
508
|
-
throw new Error(
|
|
509
|
-
`migrations.ts: MIGRATIONS must be in strictly ascending version order. Found version ${MIGRATIONS[i].version} after ${MIGRATIONS[i - 1].version} at index ${i}.`
|
|
510
|
-
);
|
|
511
|
-
}
|
|
512
|
-
}
|
|
513
|
-
var CURRENT_SCHEMA_VERSION = MIGRATIONS.length > 0 ? MIGRATIONS[MIGRATIONS.length - 1].version : 0;
|
|
514
|
-
|
|
515
|
-
// src/repositories/rowMappers.ts
|
|
516
|
-
function parseJsonArray(s, fallback) {
|
|
517
|
-
if (Array.isArray(s)) return s;
|
|
518
|
-
if (typeof s === "string" && s.length > 0) {
|
|
519
|
-
try {
|
|
520
|
-
const p = JSON.parse(s);
|
|
521
|
-
if (Array.isArray(p)) return p;
|
|
522
|
-
} catch {
|
|
523
|
-
}
|
|
524
|
-
}
|
|
525
|
-
return fallback;
|
|
526
|
-
}
|
|
527
|
-
function parseJsonObject(s, fallback = null) {
|
|
528
|
-
if (s && typeof s === "object" && !Array.isArray(s)) return s;
|
|
529
|
-
if (typeof s === "string" && s.length > 0) {
|
|
530
|
-
try {
|
|
531
|
-
const p = JSON.parse(s);
|
|
532
|
-
if (p && typeof p === "object" && !Array.isArray(p)) return p;
|
|
533
|
-
} catch {
|
|
534
|
-
}
|
|
535
|
-
}
|
|
536
|
-
return fallback;
|
|
537
|
-
}
|
|
538
68
|
function mapRowToFact(row) {
|
|
539
69
|
const tags = (() => {
|
|
540
70
|
if (Array.isArray(row.tags)) return row.tags;
|
|
@@ -612,6 +142,15 @@ var _EntryRepository = class _EntryRepository extends BaseRepository {
|
|
|
612
142
|
this.outbox = outbox;
|
|
613
143
|
this.chunkSize = 500;
|
|
614
144
|
}
|
|
145
|
+
/** Whether the entity has any non-deleted row (librarian gate: keyword ranking over zero facts is trivially empty). */
|
|
146
|
+
async hasLiveEntry(entityId, tx) {
|
|
147
|
+
const executor = this.getExecutor(tx);
|
|
148
|
+
const row = await executor.getFirstAsync(
|
|
149
|
+
`SELECT 1 AS one FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL LIMIT 1`,
|
|
150
|
+
[entityId]
|
|
151
|
+
);
|
|
152
|
+
return row != null;
|
|
153
|
+
}
|
|
615
154
|
/**
|
|
616
155
|
* Fetch facts by IDs, optionally scoped to entity IDs.
|
|
617
156
|
* Returns facts in the order of the input IDs (first match wins).
|
|
@@ -1156,6 +695,7 @@ var _EntryRepository = class _EntryRepository extends BaseRepository {
|
|
|
1156
695
|
}
|
|
1157
696
|
}
|
|
1158
697
|
}
|
|
698
|
+
if (totalChanges > 0) await reanchorRebuildFingerprint(executor, this.prefix, ids);
|
|
1159
699
|
return totalChanges;
|
|
1160
700
|
}
|
|
1161
701
|
/**
|
|
@@ -1498,27 +1038,6 @@ var _EntryRepository = class _EntryRepository extends BaseRepository {
|
|
|
1498
1038
|
`SELECT * FROM ${this.prefix}entries WHERE deleted_at IS NULL`
|
|
1499
1039
|
);
|
|
1500
1040
|
}
|
|
1501
|
-
async findRowsForSourceRefMigration(tx) {
|
|
1502
|
-
const executor = this.getExecutor(tx);
|
|
1503
|
-
return executor.getAllAsync(
|
|
1504
|
-
`SELECT rowid, source_ref FROM ${this.prefix}entries
|
|
1505
|
-
WHERE source_ref IS NOT NULL
|
|
1506
|
-
AND (
|
|
1507
|
-
TRIM(source_ref) != source_ref
|
|
1508
|
-
OR INSTR(source_ref, '/') > 0
|
|
1509
|
-
OR INSTR(source_ref, '\\') > 0
|
|
1510
|
-
OR INSTR(source_ref, CHAR(0)) > 0
|
|
1511
|
-
OR source_ref GLOB '*[^-A-Za-z0-9._ ]*'
|
|
1512
|
-
)`
|
|
1513
|
-
);
|
|
1514
|
-
}
|
|
1515
|
-
async updateSourceRefByRowid(rowid, sourceRef, tx) {
|
|
1516
|
-
const executor = this.getExecutor(tx);
|
|
1517
|
-
await executor.runAsync(
|
|
1518
|
-
`UPDATE ${this.prefix}entries SET source_ref = ? WHERE rowid = ?`,
|
|
1519
|
-
[sourceRef, rowid]
|
|
1520
|
-
);
|
|
1521
|
-
}
|
|
1522
1041
|
async findLatestSourceHash(entityId, sourceRef, tx) {
|
|
1523
1042
|
const executor = this.getExecutor(tx);
|
|
1524
1043
|
const row = await executor.getFirstAsync(
|
|
@@ -2994,87 +2513,11 @@ function assertOwnedBy(entityId, documents) {
|
|
|
2994
2513
|
}
|
|
2995
2514
|
}
|
|
2996
2515
|
|
|
2997
|
-
// src/
|
|
2998
|
-
var
|
|
2999
|
-
|
|
3000
|
-
|
|
3001
|
-
|
|
3002
|
-
}
|
|
3003
|
-
function tagsExpr(col) {
|
|
3004
|
-
return `CASE WHEN json_valid(${col}) THEN
|
|
3005
|
-
CASE WHEN json_type(${col}) = 'array' THEN
|
|
3006
|
-
(SELECT group_concat(CASE type WHEN 'true' THEN 'true' WHEN 'false' THEN 'false'
|
|
3007
|
-
WHEN 'null' THEN '' ELSE value END, ' ')
|
|
3008
|
-
FROM json_each(${col}))
|
|
3009
|
-
ELSE ${col} END
|
|
3010
|
-
ELSE ${col} END`;
|
|
3011
|
-
}
|
|
3012
|
-
function fts5TablesDdl(p) {
|
|
3013
|
-
return `
|
|
3014
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS ${p}entries_fts USING fts5(
|
|
3015
|
-
id UNINDEXED, entity_id UNINDEXED, title, body, tags,
|
|
3016
|
-
tokenize = 'porter unicode61'
|
|
3017
|
-
);
|
|
3018
|
-
CREATE TABLE IF NOT EXISTS ${p}fts_map (
|
|
3019
|
-
fts_rowid INTEGER PRIMARY KEY,
|
|
3020
|
-
id TEXT NOT NULL UNIQUE,
|
|
3021
|
-
entity_id TEXT NOT NULL
|
|
3022
|
-
);
|
|
3023
|
-
CREATE INDEX IF NOT EXISTS ${p}fts_map_entity ON ${p}fts_map(entity_id);
|
|
3024
|
-
CREATE TABLE IF NOT EXISTS ${p}fts_pending (
|
|
3025
|
-
seq INTEGER PRIMARY KEY,
|
|
3026
|
-
entity_id TEXT NOT NULL,
|
|
3027
|
-
id TEXT NOT NULL
|
|
3028
|
-
);`;
|
|
3029
|
-
}
|
|
3030
|
-
function fts5TriggersDdl(p) {
|
|
3031
|
-
const [ai, au, ad] = fts5TriggerNames(p);
|
|
3032
|
-
return `
|
|
3033
|
-
CREATE TRIGGER IF NOT EXISTS ${ai} AFTER INSERT ON ${p}entries BEGIN
|
|
3034
|
-
INSERT INTO ${p}fts_pending (entity_id, id) VALUES (NEW.entity_id, NEW.id);
|
|
3035
|
-
END;
|
|
3036
|
-
CREATE TRIGGER IF NOT EXISTS ${au}
|
|
3037
|
-
AFTER UPDATE OF id, entity_id, title, body, tags, deleted_at ON ${p}entries BEGIN
|
|
3038
|
-
INSERT INTO ${p}fts_pending (entity_id, id) VALUES (OLD.entity_id, OLD.id);
|
|
3039
|
-
INSERT INTO ${p}fts_pending (entity_id, id)
|
|
3040
|
-
SELECT NEW.entity_id, NEW.id WHERE NEW.id <> OLD.id OR NEW.entity_id <> OLD.entity_id;
|
|
3041
|
-
END;
|
|
3042
|
-
CREATE TRIGGER IF NOT EXISTS ${ad} AFTER DELETE ON ${p}entries BEGIN
|
|
3043
|
-
INSERT INTO ${p}fts_pending (entity_id, id) VALUES (OLD.entity_id, OLD.id);
|
|
3044
|
-
END;`;
|
|
3045
|
-
}
|
|
3046
|
-
function drainHiSql(p) {
|
|
3047
|
-
return `SELECT max(seq) AS hi FROM (SELECT seq FROM ${p}fts_pending ORDER BY seq LIMIT ${FTS5_DRAIN_CHUNK})`;
|
|
3048
|
-
}
|
|
3049
|
-
function drainChunkSql(p) {
|
|
3050
|
-
const chunkIds = `SELECT id FROM ${p}fts_pending WHERE seq <= ?`;
|
|
3051
|
-
return [
|
|
3052
|
-
`DELETE FROM ${p}entries_fts WHERE rowid IN (SELECT m.fts_rowid FROM ${p}fts_map m WHERE m.id IN (${chunkIds}))`,
|
|
3053
|
-
`DELETE FROM ${p}fts_map WHERE id IN (${chunkIds})`,
|
|
3054
|
-
`INSERT INTO ${p}fts_map (id, entity_id)
|
|
3055
|
-
SELECT e.id, e.entity_id FROM ${p}entries e WHERE e.id IN (${chunkIds}) AND e.deleted_at IS NULL`,
|
|
3056
|
-
`INSERT INTO ${p}entries_fts (rowid, id, entity_id, title, body, tags)
|
|
3057
|
-
SELECT m.fts_rowid, e.id, e.entity_id, e.title, e.body, ${tagsExpr("e.tags")}
|
|
3058
|
-
FROM ${p}fts_map m JOIN ${p}entries e ON e.id = m.id
|
|
3059
|
-
WHERE m.id IN (${chunkIds})`,
|
|
3060
|
-
`DELETE FROM ${p}fts_pending WHERE seq <= ?`
|
|
3061
|
-
];
|
|
3062
|
-
}
|
|
3063
|
-
function rebuildSql(p) {
|
|
3064
|
-
return [
|
|
3065
|
-
`INSERT INTO ${p}fts_map (id, entity_id) SELECT id, entity_id FROM ${p}entries WHERE deleted_at IS NULL`,
|
|
3066
|
-
`INSERT INTO ${p}entries_fts (rowid, id, entity_id, title, body, tags)
|
|
3067
|
-
SELECT m.fts_rowid, e.id, e.entity_id, e.title, e.body, ${tagsExpr("e.tags")}
|
|
3068
|
-
FROM ${p}fts_map m JOIN ${p}entries e ON e.id = m.id`
|
|
3069
|
-
];
|
|
3070
|
-
}
|
|
3071
|
-
function fts5LedgerDropSql(p) {
|
|
3072
|
-
return [
|
|
3073
|
-
...fts5TriggerNames(p).map((n) => `DROP TRIGGER IF EXISTS ${n};`),
|
|
3074
|
-
`DROP TABLE IF EXISTS ${p}fts_pending;`,
|
|
3075
|
-
`DROP TABLE IF EXISTS ${p}fts_map;`
|
|
3076
|
-
].join("\n");
|
|
3077
|
-
}
|
|
2516
|
+
// src/utils/monotonicNow.ts
|
|
2517
|
+
var monotonicNow = () => typeof performance !== "undefined" && typeof performance.now === "function" ? performance.now() : Date.now();
|
|
2518
|
+
|
|
2519
|
+
// src/utils/setupDefaults.ts
|
|
2520
|
+
var DEFAULT_SETUP_BUDGET_MS = 1500;
|
|
3078
2521
|
|
|
3079
2522
|
// src/services/search/ftsQuery.ts
|
|
3080
2523
|
var TOKEN = /[\p{L}\p{N}]+/gu;
|
|
@@ -3087,40 +2530,362 @@ function buildFtsMatchQuery(query) {
|
|
|
3087
2530
|
|
|
3088
2531
|
// src/services/search/Fts5IndexStrategy.ts
|
|
3089
2532
|
var Fts5IndexStrategy = class {
|
|
3090
|
-
constructor(db, prefix, metadataRepo) {
|
|
2533
|
+
constructor(db, prefix, metadataRepo, budgetMs = DEFAULT_SETUP_BUDGET_MS) {
|
|
3091
2534
|
this.db = db;
|
|
3092
2535
|
this.prefix = prefix;
|
|
3093
2536
|
this.metadataRepo = metadataRepo;
|
|
2537
|
+
this.budgetMs = budgetMs;
|
|
2538
|
+
}
|
|
2539
|
+
/**
|
|
2540
|
+
* Resolves the index state (spec #280 Change 3 transition table). Every
|
|
2541
|
+
* read-only probe runs before the one transition transaction, so a probe
|
|
2542
|
+
* failure leaves zero mutation (Change 4 ordering).
|
|
2543
|
+
*/
|
|
2544
|
+
async init(opts = {}) {
|
|
2545
|
+
try {
|
|
2546
|
+
await this.initOnce(opts);
|
|
2547
|
+
} catch (err) {
|
|
2548
|
+
const wrapped = unwrapFtsOnly(err);
|
|
2549
|
+
const cls = ftsCorruptionClass(wrapped.cause);
|
|
2550
|
+
if (cls === "vtab" || cls === "corrupt" && wrapped.ftsOnly) {
|
|
2551
|
+
await this.resetAndRearm();
|
|
2552
|
+
return;
|
|
2553
|
+
}
|
|
2554
|
+
throw wrapped.cause;
|
|
2555
|
+
}
|
|
2556
|
+
}
|
|
2557
|
+
/** Marks errors from statements that touch only FTS tables (entries_fts + shadows). */
|
|
2558
|
+
async ftsOnly(fn) {
|
|
2559
|
+
try {
|
|
2560
|
+
return await fn();
|
|
2561
|
+
} catch (err) {
|
|
2562
|
+
throw new FtsOnlyStatementError(err);
|
|
2563
|
+
}
|
|
3094
2564
|
}
|
|
3095
|
-
async
|
|
2565
|
+
async initOnce(opts) {
|
|
2566
|
+
const p = this.prefix;
|
|
3096
2567
|
const state = await this.metadataRepo.getMeta(FTS5_STATE_KEY);
|
|
3097
|
-
|
|
3098
|
-
|
|
2568
|
+
const schema = await this.readSchema();
|
|
2569
|
+
const ftsExists = schema.tables.has(`${p}entries_fts`);
|
|
2570
|
+
const mapExists = schema.tables.has(`${p}fts_map`);
|
|
2571
|
+
const pendingExists = schema.tables.has(`${p}fts_pending`);
|
|
2572
|
+
const trig = schema.triggersPresent;
|
|
2573
|
+
const ftsDdl = schema.tables.get(`${p}entries_fts`) ?? null;
|
|
2574
|
+
const tokenizerOk = ftsDdl === null || /tokenize\s*=\s*['"]porter\s+unicode61['"]/i.test(ftsDdl);
|
|
2575
|
+
if (state === null && opts.entriesExisted === false) {
|
|
2576
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
2577
|
+
await tx.execAsync(fts5TablesDdl(p) + fts5PendingIndexesDdl(p) + fts5TriggersDdl(p));
|
|
2578
|
+
await this.metadataRepo.setMeta(FTS5_STATE_KEY, "live", tx);
|
|
2579
|
+
});
|
|
2580
|
+
return;
|
|
2581
|
+
}
|
|
2582
|
+
if (state === "detached") return this.resetAndRearm();
|
|
2583
|
+
if (!tokenizerOk) return this.resetAndRearm();
|
|
2584
|
+
const intact = trig && ftsExists && mapExists && pendingExists;
|
|
2585
|
+
if (intact && state === "rebuilding") return;
|
|
2586
|
+
if (intact && state === "live") {
|
|
2587
|
+
await this.ftsOnly(() => this.db.getFirstAsync(`SELECT rowid FROM ${p}entries_fts ORDER BY rowid DESC LIMIT 1`));
|
|
2588
|
+
if (await this.liveRowUnindexed()) return this.resetAndRearm();
|
|
3099
2589
|
return;
|
|
3100
2590
|
}
|
|
2591
|
+
if (ftsExists && mapExists && await this.keepMapConsistent()) {
|
|
2592
|
+
await this.db.withTransactionAsync((tx) => this.rearmInTx(tx));
|
|
2593
|
+
return;
|
|
2594
|
+
}
|
|
2595
|
+
return this.resetAndRearm();
|
|
2596
|
+
}
|
|
2597
|
+
/**
|
|
2598
|
+
* The index was lost (rev 8 m3 / rev 9 m4): the oldest or the newest live entry is neither mapped nor
|
|
2599
|
+
* ledgered, which `live` never allows. The oldest one keeps detection armed after post-loss writes
|
|
2600
|
+
* repopulate the map and the ledger (#284 review). Every-open path: each side reads at most
|
|
2601
|
+
* FTS5_DRAIN_CHUNK entries rows (final review M1), so an all-soft-deleted store is never fully scanned;
|
|
2602
|
+
* a lost index whose oldest and newest rows are all deleted goes undetected.
|
|
2603
|
+
*/
|
|
2604
|
+
async liveRowUnindexed() {
|
|
2605
|
+
const p = this.prefix;
|
|
2606
|
+
const edge = (dir) => `SELECT id FROM (SELECT id FROM (SELECT id, deleted_at FROM ${p}entries ORDER BY rowid ${dir} LIMIT ${FTS5_DRAIN_CHUNK}) WHERE deleted_at IS NULL LIMIT 1)`;
|
|
2607
|
+
return await this.db.getFirstAsync(
|
|
2608
|
+
`SELECT 1 AS x FROM (${edge("ASC")} UNION ALL ${edge("DESC")}) e
|
|
2609
|
+
WHERE NOT EXISTS (SELECT 1 FROM ${p}fts_map m WHERE m.id = e.id)
|
|
2610
|
+
AND NOT EXISTS (SELECT 1 FROM ${p}fts_pending q WHERE q.id = e.id)
|
|
2611
|
+
LIMIT 1`
|
|
2612
|
+
) !== null;
|
|
2613
|
+
}
|
|
2614
|
+
/**
|
|
2615
|
+
* Read-only keep-map check (spec DDL note): every surviving entries_fts
|
|
2616
|
+
* rowid must have a map row. Max-rowid seek, then docsize vs map counts.
|
|
2617
|
+
*/
|
|
2618
|
+
async keepMapConsistent() {
|
|
2619
|
+
const p = this.prefix;
|
|
2620
|
+
const fts = await this.ftsOnly(() => this.db.getFirstAsync(`SELECT rowid FROM ${p}entries_fts ORDER BY rowid DESC LIMIT 1`));
|
|
2621
|
+
const map = await this.db.getFirstAsync(
|
|
2622
|
+
`SELECT coalesce(max(fts_rowid), 0) AS hi, count(*) AS n FROM ${p}fts_map`
|
|
2623
|
+
);
|
|
2624
|
+
if ((fts?.rowid ?? 0) > (map?.hi ?? 0)) return false;
|
|
2625
|
+
const doc = await this.ftsOnly(() => this.db.getFirstAsync(`SELECT count(*) AS n FROM ${p}entries_fts_docsize`));
|
|
2626
|
+
return (doc?.n ?? 0) === (map?.n ?? 0);
|
|
2627
|
+
}
|
|
2628
|
+
/** Keep-map re-arm: triggers, empty ledger, rebuild from 0, state rebuilding. One transaction. */
|
|
2629
|
+
async rearmInTx(tx) {
|
|
2630
|
+
const p = this.prefix;
|
|
2631
|
+
await tx.execAsync(fts5TablesDdl(p));
|
|
2632
|
+
await tx.execAsync(`DELETE FROM ${p}fts_pending;`);
|
|
2633
|
+
await tx.execAsync(fts5PendingIndexesDdl(p) + fts5TriggersDdl(p));
|
|
2634
|
+
await writeRebuildStart(tx, p, this.metadataRepo);
|
|
2635
|
+
await this.metadataRepo.setMeta(FTS5_STATE_KEY, "rebuilding", tx);
|
|
2636
|
+
}
|
|
2637
|
+
/** Reset path: retire entries_fts, recreate it empty, clear map + ledger, then the re-arm. One transaction. */
|
|
2638
|
+
async resetAndRearm() {
|
|
2639
|
+
const p = this.prefix;
|
|
3101
2640
|
await this.db.withTransactionAsync(async (tx) => {
|
|
3102
|
-
await
|
|
3103
|
-
await tx.execAsync(fts5TablesDdl(
|
|
3104
|
-
await
|
|
3105
|
-
await
|
|
2641
|
+
await this.retireFtsTable(tx);
|
|
2642
|
+
await tx.execAsync(fts5TablesDdl(p));
|
|
2643
|
+
await tx.execAsync(`DELETE FROM ${p}fts_map; DELETE FROM ${p}fts_pending;`);
|
|
2644
|
+
await this.rearmInTx(tx);
|
|
2645
|
+
});
|
|
2646
|
+
}
|
|
2647
|
+
/** Rename-retire entries_fts (spec #280 rev 11); DROP only when the RENAME throws. */
|
|
2648
|
+
async retireFtsTable(tx) {
|
|
2649
|
+
const p = this.prefix;
|
|
2650
|
+
if (!await this.metadataRepo.tableExists(`${p}entries_fts`, tx)) return;
|
|
2651
|
+
const n = Number(await this.metadataRepo.getMeta(FTS5_RETIRED_COUNTER_KEY, tx) ?? "0") + 1;
|
|
2652
|
+
const name = retiredTableName(p, n);
|
|
2653
|
+
try {
|
|
2654
|
+
await tx.execAsync(`ALTER TABLE ${p}entries_fts RENAME TO ${name}`);
|
|
2655
|
+
} catch (err) {
|
|
2656
|
+
if (isTransientOrTxFatal(err)) throw err;
|
|
2657
|
+
await tx.execAsync(`DROP TABLE ${p}entries_fts`);
|
|
2658
|
+
return;
|
|
2659
|
+
}
|
|
2660
|
+
await this.metadataRepo.setMeta(FTS5_RETIRED_TABLES_KEY, JSON.stringify([...await this.retiredList(tx), name]), tx);
|
|
2661
|
+
await this.metadataRepo.setMeta(FTS5_RETIRED_COUNTER_KEY, String(n), tx);
|
|
2662
|
+
}
|
|
2663
|
+
async retiredList(tx = this.db) {
|
|
2664
|
+
const raw = parseJsonArray(await this.metadataRepo.getMeta(FTS5_RETIRED_TABLES_KEY, tx), []);
|
|
2665
|
+
return raw.filter((n) => isRetiredTableName(this.prefix, n));
|
|
2666
|
+
}
|
|
2667
|
+
async drain(opts = {}) {
|
|
2668
|
+
try {
|
|
2669
|
+
await this.drainOnce(opts);
|
|
2670
|
+
} catch (err) {
|
|
2671
|
+
if (ftsCorruptionClass(err) !== "vtab") throw err;
|
|
2672
|
+
await this.resetAndRearm();
|
|
2673
|
+
if (opts.ids && opts.ids.length > 0) await this.drainIds(opts.ids, idReindexChunkSql(this.prefix));
|
|
2674
|
+
}
|
|
2675
|
+
}
|
|
2676
|
+
async drainOnce(opts) {
|
|
2677
|
+
const ids = opts.ids ?? [];
|
|
2678
|
+
if (ids.length > 0) await this.drainIds(ids);
|
|
2679
|
+
if (opts.mode === "ledger") {
|
|
2680
|
+
if (ids.length === 0 && opts.entityId !== void 0) await this.drainEntityChunk(opts.entityId);
|
|
2681
|
+
else await this.drainLedgerChunk();
|
|
2682
|
+
return;
|
|
2683
|
+
}
|
|
2684
|
+
await this.fullQuantum(opts.deadline ?? monotonicNow() + this.budgetMs);
|
|
2685
|
+
}
|
|
2686
|
+
/**
|
|
2687
|
+
* One full quantum (spec #280 Change 3): ledger first (at most half the
|
|
2688
|
+
* budget while `rebuilding`); then, while `rebuilding`, ≥1 window unless
|
|
2689
|
+
* the ledger drain overran the deadline, one sweep tx, and the flip check;
|
|
2690
|
+
* then reclamation. A `live` store reads only the state.
|
|
2691
|
+
*/
|
|
2692
|
+
async fullQuantum(deadline) {
|
|
2693
|
+
const rebuilding = await this.metadataRepo.getMeta(FTS5_STATE_KEY) === "rebuilding";
|
|
2694
|
+
const start = monotonicNow();
|
|
2695
|
+
const ledgerRan = await this.drainLedgerUntil(rebuilding ? start + (deadline - start) / 2 : deadline);
|
|
2696
|
+
const ledgerOverran = ledgerRan && monotonicNow() >= deadline;
|
|
2697
|
+
if (rebuilding) {
|
|
2698
|
+
const progress = await readRebuildProgress(this.metadataRepo);
|
|
2699
|
+
if (!progress || !await this.fingerprintValid(progress.wm)) {
|
|
2700
|
+
await this.restartRebuild();
|
|
2701
|
+
} else {
|
|
2702
|
+
if (!ledgerOverran) {
|
|
2703
|
+
let step;
|
|
2704
|
+
do {
|
|
2705
|
+
step = await this.runWindow(progress);
|
|
2706
|
+
} while (step === "more" && monotonicNow() < deadline);
|
|
2707
|
+
if (step !== "stale") await this.runSweepTx(progress);
|
|
2708
|
+
}
|
|
2709
|
+
await this.tryFlip();
|
|
2710
|
+
}
|
|
2711
|
+
}
|
|
2712
|
+
try {
|
|
2713
|
+
await this.reclaimRetired(deadline);
|
|
2714
|
+
} catch (err) {
|
|
2715
|
+
if (!isSqliteError(err)) throw err;
|
|
2716
|
+
}
|
|
2717
|
+
}
|
|
2718
|
+
/** Bounded reclamation steps, each its own tx, until the deadline or nothing is retired. */
|
|
2719
|
+
async reclaimRetired(deadline) {
|
|
2720
|
+
while (monotonicNow() < deadline) {
|
|
2721
|
+
const [name] = await this.retiredList();
|
|
2722
|
+
if (name === void 0) return;
|
|
2723
|
+
try {
|
|
2724
|
+
if (!await this.metadataRepo.tableExists(name)) {
|
|
2725
|
+
await this.delistRetired(name, false);
|
|
2726
|
+
continue;
|
|
2727
|
+
}
|
|
2728
|
+
while (!await this.reclaimStep(name) && monotonicNow() < deadline) ;
|
|
2729
|
+
} catch (err) {
|
|
2730
|
+
if (ftsCorruptionClass(err) === null) throw err;
|
|
2731
|
+
try {
|
|
2732
|
+
await this.delistRetired(name, true);
|
|
2733
|
+
} catch (dropErr) {
|
|
2734
|
+
if (!isSqliteError(dropErr)) throw dropErr;
|
|
2735
|
+
await this.delistRetired(name, false);
|
|
2736
|
+
}
|
|
2737
|
+
}
|
|
2738
|
+
}
|
|
2739
|
+
}
|
|
2740
|
+
/** (1) delete <=500 rows, else (2) one bounded merge, else (3) DROP + delist. True once delisted. */
|
|
2741
|
+
async reclaimStep(name) {
|
|
2742
|
+
if (await this.db.getFirstAsync(`SELECT rowid FROM ${name} LIMIT 1`)) {
|
|
2743
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
2744
|
+
await tx.runAsync(`DELETE FROM ${name} WHERE rowid IN (SELECT rowid FROM ${name} ORDER BY rowid LIMIT ${FTS5_DRAIN_CHUNK})`);
|
|
2745
|
+
});
|
|
2746
|
+
return false;
|
|
2747
|
+
}
|
|
2748
|
+
let merged = false;
|
|
2749
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
2750
|
+
const before = await tx.getFirstAsync(`SELECT total_changes() AS n`);
|
|
2751
|
+
await tx.runAsync(`INSERT INTO ${name}(${name}, rank) VALUES('merge', ${FTS5_RECLAIM_MERGE_PAGES})`);
|
|
2752
|
+
const after = await tx.getFirstAsync(`SELECT total_changes() AS n`);
|
|
2753
|
+
merged = (after?.n ?? 0) - (before?.n ?? 0) >= 2;
|
|
2754
|
+
});
|
|
2755
|
+
if (!merged) await this.delistRetired(name, true);
|
|
2756
|
+
return !merged;
|
|
2757
|
+
}
|
|
2758
|
+
async delistRetired(name, drop) {
|
|
2759
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
2760
|
+
if (drop) await tx.execAsync(`DROP TABLE IF EXISTS ${name}`);
|
|
2761
|
+
const rest = (await this.retiredList(tx)).filter((n) => n !== name);
|
|
2762
|
+
await this.metadataRepo.setMeta(FTS5_RETIRED_TABLES_KEY, JSON.stringify(rest), tx);
|
|
2763
|
+
});
|
|
2764
|
+
}
|
|
2765
|
+
/** The id at the watermark's rowid still matches. A null id (start, or windows complete) has nothing to check. */
|
|
2766
|
+
async fingerprintValid(wm, tx = this.db) {
|
|
2767
|
+
if (wm.id === null) return true;
|
|
2768
|
+
const row = await tx.getFirstAsync(`SELECT id FROM ${this.prefix}entries WHERE rowid = ?`, [wm.rowid]);
|
|
2769
|
+
return row?.id === wm.id;
|
|
2770
|
+
}
|
|
2771
|
+
/** External change under the watermark: start over (same re-arm transaction as a trigger-loss restart). */
|
|
2772
|
+
async restartRebuild() {
|
|
2773
|
+
await this.db.withTransactionAsync((tx) => this.rearmInTx(tx));
|
|
2774
|
+
}
|
|
2775
|
+
/**
|
|
2776
|
+
* Progress as stored, re-read inside a window/sweep tx: false when it no
|
|
2777
|
+
* longer matches what this quantum assumed (final review I1). Writing a
|
|
2778
|
+
* watermark or cursor computed from a stale `pr` would overwrite a
|
|
2779
|
+
* concurrent restart (runPrune VACUUM is not on the sync chain).
|
|
2780
|
+
*/
|
|
2781
|
+
async progressUnchanged(pr, tx) {
|
|
2782
|
+
if (await this.metadataRepo.getMeta(FTS5_STATE_KEY, tx) !== "rebuilding") return false;
|
|
2783
|
+
const cur = await readRebuildProgress(this.metadataRepo, tx);
|
|
2784
|
+
return cur !== null && cur.wm.rowid === pr.wm.rowid && cur.wm.id === pr.wm.id && cur.cursor === pr.cursor && cur.snapshot.entries === pr.snapshot.entries && cur.snapshot.map === pr.snapshot.map;
|
|
2785
|
+
}
|
|
2786
|
+
/** One window tx. 'done' once no rows remain in (wm, snapshot] (the watermark is then pinned at the snapshot, F9); 'stale' when progress moved under the quantum. */
|
|
2787
|
+
async runWindow(pr) {
|
|
2788
|
+
if (pr.wm.rowid >= pr.snapshot.entries) return "done";
|
|
2789
|
+
const p = this.prefix;
|
|
2790
|
+
let next = pr.wm;
|
|
2791
|
+
let more = false;
|
|
2792
|
+
let stale = false;
|
|
2793
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
2794
|
+
if (!await this.progressUnchanged(pr, tx)) {
|
|
2795
|
+
stale = true;
|
|
2796
|
+
return;
|
|
2797
|
+
}
|
|
2798
|
+
const row = await tx.getFirstAsync(windowHiSql(p), [pr.wm.rowid, pr.snapshot.entries]);
|
|
2799
|
+
if (row?.hi == null) {
|
|
2800
|
+
next = { rowid: pr.snapshot.entries, id: null };
|
|
2801
|
+
} else {
|
|
2802
|
+
for (const sql of windowChunkSql(p)) await tx.runAsync(sql, [pr.wm.rowid, row.hi]);
|
|
2803
|
+
const fp = await tx.getFirstAsync(`SELECT id FROM ${p}entries WHERE rowid = ?`, [row.hi]);
|
|
2804
|
+
next = { rowid: row.hi, id: fp.id };
|
|
2805
|
+
more = true;
|
|
2806
|
+
}
|
|
2807
|
+
await this.metadataRepo.setMeta(FTS5_REBUILD_WM_KEY, JSON.stringify(next), tx);
|
|
2808
|
+
});
|
|
2809
|
+
if (stale) return "stale";
|
|
2810
|
+
pr.wm = next;
|
|
2811
|
+
return more ? "more" : "done";
|
|
2812
|
+
}
|
|
2813
|
+
/** At most one bounded sweep tx; the cursor advances in the same tx. */
|
|
2814
|
+
async runSweepTx(pr) {
|
|
2815
|
+
if (pr.cursor >= pr.snapshot.map) return;
|
|
2816
|
+
const p = this.prefix;
|
|
2817
|
+
let next = pr.cursor;
|
|
2818
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
2819
|
+
if (!await this.progressUnchanged(pr, tx)) return;
|
|
2820
|
+
const row = await tx.getFirstAsync(sweepHiSql(p), [pr.cursor, pr.snapshot.map]);
|
|
2821
|
+
const hi = row?.hi ?? pr.snapshot.map;
|
|
2822
|
+
for (const sql of sweepChunkSql(p)) await tx.runAsync(sql, [pr.cursor, hi]);
|
|
2823
|
+
await this.metadataRepo.setMeta(FTS5_SWEEP_CURSOR_KEY, String(hi), tx);
|
|
2824
|
+
next = hi;
|
|
2825
|
+
});
|
|
2826
|
+
pr.cursor = next;
|
|
2827
|
+
}
|
|
2828
|
+
/** Completion, re-checked inside the flipping tx; the residual ledger (≤ one chunk) drains in it. */
|
|
2829
|
+
async tryFlip() {
|
|
2830
|
+
const p = this.prefix;
|
|
2831
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
2832
|
+
if (await this.metadataRepo.getMeta(FTS5_STATE_KEY, tx) !== "rebuilding") return;
|
|
2833
|
+
const pr = await readRebuildProgress(this.metadataRepo, tx);
|
|
2834
|
+
if (!pr || pr.wm.rowid < pr.snapshot.entries || pr.cursor < pr.snapshot.map) return;
|
|
2835
|
+
if (!await this.fingerprintValid(pr.wm, tx)) return;
|
|
2836
|
+
const residual = await tx.getFirstAsync(
|
|
2837
|
+
`SELECT count(*) AS n FROM (SELECT 1 FROM ${p}fts_pending LIMIT ${FTS5_DRAIN_CHUNK + 1})`
|
|
2838
|
+
);
|
|
2839
|
+
if ((residual?.n ?? 0) > FTS5_DRAIN_CHUNK) return;
|
|
2840
|
+
const hi = await tx.getFirstAsync(drainHiSql(p));
|
|
2841
|
+
if (hi?.hi != null) for (const sql of drainChunkSql(p)) await tx.runAsync(sql, [hi.hi]);
|
|
2842
|
+
await this.metadataRepo.deleteMeta(FTS5_REBUILD_WM_KEY, tx);
|
|
2843
|
+
await this.metadataRepo.deleteMeta(FTS5_REBUILD_SNAPSHOT_KEY, tx);
|
|
2844
|
+
await this.metadataRepo.deleteMeta(FTS5_SWEEP_CURSOR_KEY, tx);
|
|
3106
2845
|
await this.metadataRepo.setMeta(FTS5_STATE_KEY, "live", tx);
|
|
3107
2846
|
});
|
|
3108
2847
|
}
|
|
3109
|
-
|
|
3110
|
-
|
|
2848
|
+
/** Seq-ordered ledger chunks until empty or `stopAt`. True when a chunk ran. */
|
|
2849
|
+
async drainLedgerUntil(stopAt) {
|
|
2850
|
+
let ran = false;
|
|
2851
|
+
while (await this.drainLedgerChunk()) {
|
|
2852
|
+
ran = true;
|
|
2853
|
+
if (monotonicNow() >= stopAt) break;
|
|
2854
|
+
}
|
|
2855
|
+
return ran;
|
|
2856
|
+
}
|
|
2857
|
+
/** The caller's own ids, to completion, ≤ FTS5_DRAIN_CHUNK per transaction. */
|
|
2858
|
+
async drainIds(ids, chunk = idDrainChunkSql(this.prefix)) {
|
|
2859
|
+
const unique = [...new Set(ids)];
|
|
2860
|
+
for (let i = 0; i < unique.length; i += FTS5_DRAIN_CHUNK) {
|
|
2861
|
+
const json = JSON.stringify(unique.slice(i, i + FTS5_DRAIN_CHUNK));
|
|
2862
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
2863
|
+
for (const sql of chunk) await tx.runAsync(sql, [json]);
|
|
2864
|
+
});
|
|
2865
|
+
}
|
|
3111
2866
|
}
|
|
3112
|
-
|
|
2867
|
+
/** One seq-ordered chunk. False when the ledger was empty. */
|
|
2868
|
+
async drainLedgerChunk() {
|
|
3113
2869
|
const hiSql = drainHiSql(this.prefix);
|
|
3114
2870
|
const chunk = drainChunkSql(this.prefix);
|
|
3115
|
-
|
|
3116
|
-
|
|
3117
|
-
|
|
3118
|
-
|
|
3119
|
-
|
|
3120
|
-
|
|
3121
|
-
|
|
3122
|
-
|
|
3123
|
-
|
|
2871
|
+
let ran = false;
|
|
2872
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
2873
|
+
const row = await tx.getFirstAsync(hiSql);
|
|
2874
|
+
if (row?.hi == null) return;
|
|
2875
|
+
for (const sql of chunk) await tx.runAsync(sql, [row.hi]);
|
|
2876
|
+
ran = true;
|
|
2877
|
+
});
|
|
2878
|
+
return ran;
|
|
2879
|
+
}
|
|
2880
|
+
/** One entity-targeted chunk (ORDER BY seq, ≤ FTS5_DRAIN_CHUNK). */
|
|
2881
|
+
async drainEntityChunk(entityId) {
|
|
2882
|
+
const hiSql = entityDrainHiSql(this.prefix);
|
|
2883
|
+
const chunk = entityDrainChunkSql(this.prefix);
|
|
2884
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
2885
|
+
const row = await tx.getFirstAsync(hiSql, [entityId]);
|
|
2886
|
+
if (row?.hi == null) return;
|
|
2887
|
+
for (const sql of chunk) await tx.runAsync(sql, [entityId, row.hi]);
|
|
2888
|
+
});
|
|
3124
2889
|
}
|
|
3125
2890
|
async replace(entityId, ids, documents) {
|
|
3126
2891
|
assertEntity(entityId, documents);
|
|
@@ -3199,20 +2964,57 @@ var Fts5IndexStrategy = class {
|
|
|
3199
2964
|
);
|
|
3200
2965
|
}
|
|
3201
2966
|
}
|
|
3202
|
-
|
|
2967
|
+
/** The init probes' schema facts in one sqlite_master read (#284 review): FTS tables with their DDL, and the triggers. */
|
|
2968
|
+
async readSchema() {
|
|
3203
2969
|
const p = this.prefix;
|
|
3204
|
-
|
|
3205
|
-
|
|
3206
|
-
}
|
|
3207
|
-
async triggersPresent() {
|
|
3208
|
-
const names = fts5TriggerNames(this.prefix);
|
|
2970
|
+
const triggers = fts5TriggerNames(p);
|
|
2971
|
+
const names = [`${p}entries_fts`, `${p}fts_map`, `${p}fts_pending`, ...triggers];
|
|
3209
2972
|
const rows = await this.db.getAllAsync(
|
|
3210
|
-
`SELECT name FROM sqlite_master WHERE
|
|
3211
|
-
|
|
2973
|
+
`SELECT type, name, sql FROM sqlite_master WHERE name IN (${names.map(() => "?").join(", ")})`,
|
|
2974
|
+
names
|
|
3212
2975
|
);
|
|
3213
|
-
|
|
2976
|
+
const tables = new Map(rows.filter((r) => r.type === "table").map((r) => [r.name, r.sql]));
|
|
2977
|
+
const present = new Set(rows.filter((r) => r.type === "trigger").map((r) => r.name));
|
|
2978
|
+
return { tables, triggersPresent: triggers.every((n) => present.has(n)) };
|
|
3214
2979
|
}
|
|
3215
2980
|
};
|
|
2981
|
+
var FtsOnlyStatementError = class extends Error {
|
|
2982
|
+
constructor(cause) {
|
|
2983
|
+
super("FTS-only statement failed");
|
|
2984
|
+
this.ftsOnlyCause = cause;
|
|
2985
|
+
}
|
|
2986
|
+
};
|
|
2987
|
+
function unwrapFtsOnly(err) {
|
|
2988
|
+
try {
|
|
2989
|
+
if (err instanceof FtsOnlyStatementError) return { cause: err.ftsOnlyCause, ftsOnly: true };
|
|
2990
|
+
} catch {
|
|
2991
|
+
}
|
|
2992
|
+
return { cause: err, ftsOnly: false };
|
|
2993
|
+
}
|
|
2994
|
+
function safeSqliteCode(err) {
|
|
2995
|
+
try {
|
|
2996
|
+
const code = extractSqliteCode(err);
|
|
2997
|
+
if (code !== void 0) return code;
|
|
2998
|
+
const lifted = err?.sqliteErrorCode;
|
|
2999
|
+
return typeof lifted === "string" ? lifted : void 0;
|
|
3000
|
+
} catch {
|
|
3001
|
+
return void 0;
|
|
3002
|
+
}
|
|
3003
|
+
}
|
|
3004
|
+
var TRANSIENT_OR_TX_FATAL = ["SQLITE_BUSY", "SQLITE_LOCKED", "SQLITE_FULL", "SQLITE_IOERR", "SQLITE_NOMEM", "SQLITE_INTERRUPT", "SQLITE_ABORT", "SQLITE_READONLY"];
|
|
3005
|
+
function isTransientOrTxFatal(err) {
|
|
3006
|
+
const code = safeSqliteCode(err);
|
|
3007
|
+
return code !== void 0 && TRANSIENT_OR_TX_FATAL.some((c) => code === c || code.startsWith(`${c}_`));
|
|
3008
|
+
}
|
|
3009
|
+
function ftsCorruptionClass(err) {
|
|
3010
|
+
const code = safeSqliteCode(err);
|
|
3011
|
+
if (code === "SQLITE_CORRUPT_VTAB") return "vtab";
|
|
3012
|
+
if (code?.startsWith("SQLITE_CORRUPT")) return "corrupt";
|
|
3013
|
+
return null;
|
|
3014
|
+
}
|
|
3015
|
+
function isSqliteError(err) {
|
|
3016
|
+
return safeSqliteCode(err) !== void 0;
|
|
3017
|
+
}
|
|
3216
3018
|
function assertEntity(entityId, documents) {
|
|
3217
3019
|
for (const d of documents) {
|
|
3218
3020
|
if (d.entity_id !== entityId) {
|
|
@@ -3222,14 +3024,34 @@ function assertEntity(entityId, documents) {
|
|
|
3222
3024
|
}
|
|
3223
3025
|
|
|
3224
3026
|
// src/services/search/createIndexStrategy.ts
|
|
3027
|
+
var FTS5_INIT_RETRY_MS = 250;
|
|
3028
|
+
function isModuleAbsentError(err) {
|
|
3029
|
+
let cur = err;
|
|
3030
|
+
for (let depth = 0; depth < 3 && cur != null; depth++) {
|
|
3031
|
+
let msg = "";
|
|
3032
|
+
try {
|
|
3033
|
+
msg = String(cur.message ?? cur);
|
|
3034
|
+
} catch {
|
|
3035
|
+
return false;
|
|
3036
|
+
}
|
|
3037
|
+
if (/no such module: fts5/i.test(msg) || /no such function: json_valid/i.test(msg)) return true;
|
|
3038
|
+
try {
|
|
3039
|
+
cur = cur.cause;
|
|
3040
|
+
} catch {
|
|
3041
|
+
return false;
|
|
3042
|
+
}
|
|
3043
|
+
}
|
|
3044
|
+
return false;
|
|
3045
|
+
}
|
|
3225
3046
|
async function probeFts5(db, prefix) {
|
|
3226
3047
|
const name = `temp.${prefix}fts5_probe`;
|
|
3227
3048
|
try {
|
|
3228
3049
|
await db.execAsync(`CREATE VIRTUAL TABLE IF NOT EXISTS ${name} USING fts5(x)`);
|
|
3229
3050
|
const row = await db.getFirstAsync(`SELECT json_valid('[]') AS ok`);
|
|
3230
3051
|
return row?.ok === 1;
|
|
3231
|
-
} catch {
|
|
3232
|
-
return false;
|
|
3052
|
+
} catch (err) {
|
|
3053
|
+
if (isModuleAbsentError(err)) return false;
|
|
3054
|
+
throw err;
|
|
3233
3055
|
} finally {
|
|
3234
3056
|
try {
|
|
3235
3057
|
await db.execAsync(`DROP TABLE IF EXISTS ${name}`);
|
|
@@ -3250,23 +3072,37 @@ async function detachFts5(db, prefix, metadataRepo) {
|
|
|
3250
3072
|
} catch {
|
|
3251
3073
|
}
|
|
3252
3074
|
}
|
|
3253
|
-
async function createIndexStrategy(db, prefix, metadataRepo, preferred) {
|
|
3254
|
-
if (preferred
|
|
3255
|
-
|
|
3256
|
-
|
|
3257
|
-
|
|
3258
|
-
|
|
3259
|
-
|
|
3260
|
-
|
|
3261
|
-
|
|
3262
|
-
|
|
3263
|
-
|
|
3264
|
-
|
|
3075
|
+
async function createIndexStrategy(db, prefix, metadataRepo, preferred, opts = {}) {
|
|
3076
|
+
if (preferred === "minisearch") {
|
|
3077
|
+
await detachFts5(db, prefix, metadataRepo);
|
|
3078
|
+
return new MiniSearchIndexStrategy();
|
|
3079
|
+
}
|
|
3080
|
+
const attempt = async () => {
|
|
3081
|
+
if (!await probeFts5(db, prefix)) return "absent";
|
|
3082
|
+
const strategy = new Fts5IndexStrategy(db, prefix, metadataRepo, opts.setupBudgetMs);
|
|
3083
|
+
await strategy.init({ entriesExisted: opts.entriesExisted });
|
|
3084
|
+
return strategy;
|
|
3085
|
+
};
|
|
3086
|
+
const absent = async () => {
|
|
3087
|
+
if (preferred === "fts5") {
|
|
3265
3088
|
throw new Error(`indexStrategy 'fts5' was requested, but this SQLite build lacks FTS5 or json1.`);
|
|
3266
3089
|
}
|
|
3090
|
+
await detachFts5(db, prefix, metadataRepo);
|
|
3091
|
+
return new MiniSearchIndexStrategy();
|
|
3092
|
+
};
|
|
3093
|
+
for (let tries = 1; ; tries++) {
|
|
3094
|
+
try {
|
|
3095
|
+
const s = await attempt();
|
|
3096
|
+
return s === "absent" ? await absent() : s;
|
|
3097
|
+
} catch (err) {
|
|
3098
|
+
if (preferred === "fts5") throw err;
|
|
3099
|
+
if (tries === 2) {
|
|
3100
|
+
console.warn("[WikiMemory] FTS5 init failed after retry; keyword search unavailable until a later open or syncSearchIndex():", err);
|
|
3101
|
+
return new UnavailableIndexStrategy(err);
|
|
3102
|
+
}
|
|
3103
|
+
}
|
|
3104
|
+
await new Promise((resolve) => setTimeout(resolve, FTS5_INIT_RETRY_MS));
|
|
3267
3105
|
}
|
|
3268
|
-
await detachFts5(db, prefix, metadataRepo);
|
|
3269
|
-
return new MiniSearchIndexStrategy();
|
|
3270
3106
|
}
|
|
3271
3107
|
|
|
3272
3108
|
// src/services/MaintenanceScheduler.ts
|
|
@@ -3350,8 +3186,14 @@ var MaintenanceScheduler = class {
|
|
|
3350
3186
|
var _a;
|
|
3351
3187
|
((_a = st.report).busy ?? (_a.busy = [])).push(job);
|
|
3352
3188
|
};
|
|
3189
|
+
const markSearchUnavailable = (st, job) => {
|
|
3190
|
+
var _a;
|
|
3191
|
+
((_a = st.report).searchUnavailable ?? (_a.searchUnavailable = [])).push(job);
|
|
3192
|
+
};
|
|
3193
|
+
const openJobs = () => states.reduce((n, s) => n + +s.librarian + +s.heal + +s.reembed, 0);
|
|
3353
3194
|
rounds: for (; ; ) {
|
|
3354
3195
|
let progressed = false;
|
|
3196
|
+
const openAtStart = openJobs();
|
|
3355
3197
|
for (const st of states) {
|
|
3356
3198
|
if (!st.librarian && !st.heal && !st.reembed) continue;
|
|
3357
3199
|
if (Date.now() >= deadline) {
|
|
@@ -3369,6 +3211,11 @@ var MaintenanceScheduler = class {
|
|
|
3369
3211
|
let r;
|
|
3370
3212
|
try {
|
|
3371
3213
|
r = await this.maintenanceService.runLibrarianPass({ entityId, trigger: "call", meter });
|
|
3214
|
+
} catch (err) {
|
|
3215
|
+
if (!isWikiSearchUnavailableError(err)) throw err;
|
|
3216
|
+
st.librarian = false;
|
|
3217
|
+
markSearchUnavailable(st, "librarian");
|
|
3218
|
+
continue;
|
|
3372
3219
|
} finally {
|
|
3373
3220
|
this.jobManager.releaseLock("librarian", entityId);
|
|
3374
3221
|
}
|
|
@@ -3412,6 +3259,11 @@ var MaintenanceScheduler = class {
|
|
|
3412
3259
|
st.heal = false;
|
|
3413
3260
|
continue;
|
|
3414
3261
|
}
|
|
3262
|
+
if (h.scanned === 0 && h.remaining > 0) {
|
|
3263
|
+
st.heal = false;
|
|
3264
|
+
if (h.searchUnavailable) markSearchUnavailable(st, "heal");
|
|
3265
|
+
continue;
|
|
3266
|
+
}
|
|
3415
3267
|
st.report.healPasses++;
|
|
3416
3268
|
progressed = true;
|
|
3417
3269
|
if (h.remaining === 0) {
|
|
@@ -3436,7 +3288,7 @@ var MaintenanceScheduler = class {
|
|
|
3436
3288
|
}
|
|
3437
3289
|
}
|
|
3438
3290
|
}
|
|
3439
|
-
if (!progressed) break;
|
|
3291
|
+
if (!progressed && openJobs() === openAtStart) break;
|
|
3440
3292
|
}
|
|
3441
3293
|
if (stopped.reason === "complete" && tooSmall) stopped = tooSmall;
|
|
3442
3294
|
return {
|
|
@@ -3937,6 +3789,9 @@ var OkfTrustWritesRepository = class {
|
|
|
3937
3789
|
|
|
3938
3790
|
// src/WikiMemory.ts
|
|
3939
3791
|
var TABLE_PREFIX_PATTERN = /^[A-Za-z][A-Za-z0-9_]{0,30}_$/;
|
|
3792
|
+
var SEARCH_REINIT_COOLDOWN_MS = 5e3;
|
|
3793
|
+
var SEARCH_REINIT_COOLDOWN_MAX_MS = 5 * 6e4;
|
|
3794
|
+
var SEARCH_REINIT_COST_FACTOR = 10;
|
|
3940
3795
|
function decodeDraftCursor(cursor) {
|
|
3941
3796
|
const sep = typeof cursor === "string" ? cursor.indexOf(":") : -1;
|
|
3942
3797
|
const createdAt = sep > 0 ? Number(cursor.slice(0, sep)) : Number.NaN;
|
|
@@ -3951,6 +3806,21 @@ var WikiMemory = class {
|
|
|
3951
3806
|
constructor(db, options) {
|
|
3952
3807
|
/** Emits `__testAccess` console warning at most once per instance when NODE_ENV ≠ "test". */
|
|
3953
3808
|
__privateAdd(this, _testAccessNonTestEnvWarned, false);
|
|
3809
|
+
/**
|
|
3810
|
+
* upsertGraph node and superseded-fact ids per entity, awaiting syncSearchIndex(), each with the
|
|
3811
|
+
* generation of its latest stash. Ids leave the stash only once a drain
|
|
3812
|
+
* succeeded, so a failed drain retries them next call.
|
|
3813
|
+
*/
|
|
3814
|
+
this.upsertIdStash = /* @__PURE__ */ new Map();
|
|
3815
|
+
/** Total ids across upsertIdStash, kept in step so the cap check is O(1). */
|
|
3816
|
+
this.upsertIdStashSize = 0;
|
|
3817
|
+
/** Bumped per stashed id; a re-stash during an in-flight drain outlives that drain's drop. */
|
|
3818
|
+
this.upsertIdStashGen = 0;
|
|
3819
|
+
this.reinitSearch = null;
|
|
3820
|
+
/** Monotonic time before which a no-arg syncSearchIndex() skips the re-init (0 = no cooldown). */
|
|
3821
|
+
this.reinitRetryAt = 0;
|
|
3822
|
+
/** Consecutive failed re-inits; drives the exponential backoff. */
|
|
3823
|
+
this.reinitFailures = 0;
|
|
3954
3824
|
this.db = withSerializedTransactions(db);
|
|
3955
3825
|
this.options = options;
|
|
3956
3826
|
this.prefix = options.config?.tablePrefix ?? "llm_wiki_";
|
|
@@ -4082,23 +3952,81 @@ var WikiMemory = class {
|
|
|
4082
3952
|
jobManager: this.jobManager
|
|
4083
3953
|
};
|
|
4084
3954
|
}
|
|
4085
|
-
|
|
3955
|
+
stashUpsertIds(entityId, ids) {
|
|
3956
|
+
if (!this.searchService.hasLedger()) return;
|
|
3957
|
+
const cap = this.constructor.UPSERT_ID_STASH_CAP;
|
|
3958
|
+
const stash = this.upsertIdStash.get(entityId) ?? /* @__PURE__ */ new Map();
|
|
3959
|
+
for (const id of ids) {
|
|
3960
|
+
if (!stash.has(id)) {
|
|
3961
|
+
if (this.upsertIdStashSize >= cap) continue;
|
|
3962
|
+
this.upsertIdStashSize++;
|
|
3963
|
+
}
|
|
3964
|
+
stash.set(id, ++this.upsertIdStashGen);
|
|
3965
|
+
}
|
|
3966
|
+
if (stash.size > 0) this.upsertIdStash.set(entityId, stash);
|
|
3967
|
+
}
|
|
3968
|
+
/** Snapshot of the stash (one entity, or all) with generations, without removing anything. */
|
|
3969
|
+
peekStashedIds(entityId) {
|
|
3970
|
+
const snapshot = /* @__PURE__ */ new Map();
|
|
3971
|
+
for (const [id, stash] of this.upsertIdStash) {
|
|
3972
|
+
if (entityId === void 0 || id === entityId) snapshot.set(id, new Map(stash));
|
|
3973
|
+
}
|
|
3974
|
+
return snapshot;
|
|
3975
|
+
}
|
|
3976
|
+
/**
|
|
3977
|
+
* Removes a drained snapshot. An id re-stashed after the peek has a newer
|
|
3978
|
+
* generation and stays: the drain may have run before that write committed.
|
|
3979
|
+
*/
|
|
3980
|
+
dropStashedIds(snapshot) {
|
|
3981
|
+
for (const [entityId, drained] of snapshot) {
|
|
3982
|
+
const stash = this.upsertIdStash.get(entityId);
|
|
3983
|
+
if (!stash) continue;
|
|
3984
|
+
for (const [id, gen] of drained) {
|
|
3985
|
+
if (stash.get(id) === gen && stash.delete(id)) this.upsertIdStashSize--;
|
|
3986
|
+
}
|
|
3987
|
+
if (stash.size === 0) this.upsertIdStash.delete(entityId);
|
|
3988
|
+
}
|
|
3989
|
+
}
|
|
3990
|
+
/** `config.setupBudgetMs` or the default, validated: a positive finite number. */
|
|
3991
|
+
setupBudgetMs() {
|
|
3992
|
+
const budget = this.options.config?.setupBudgetMs ?? DEFAULT_SETUP_BUDGET_MS;
|
|
3993
|
+
if (typeof budget !== "number" || !Number.isFinite(budget) || budget <= 0) {
|
|
3994
|
+
throw new TypeError(`setupBudgetMs must be a positive finite number; got ${String(budget)}`);
|
|
3995
|
+
}
|
|
3996
|
+
return budget;
|
|
3997
|
+
}
|
|
3998
|
+
/**
|
|
3999
|
+
* Opens the store: migration gate, DDL, schema migrations, then the stamped
|
|
4000
|
+
* source_ref normalize. Await it before any other call on this instance — the
|
|
4001
|
+
* normalize commits in chunks, so a sourceRef-keyed call (forget, upsertGraph,
|
|
4002
|
+
* hasChanged, ingest) issued before setup() resolves can see partially
|
|
4003
|
+
* normalized refs.
|
|
4004
|
+
*
|
|
4005
|
+
* Pass `{ migrations: 'refuse' }` from inside a writer lease: pending one-time
|
|
4006
|
+
* work then throws WikiMigrationRequiredError before any mutation, and the
|
|
4007
|
+
* host runs a plain `setup()` outside the lease.
|
|
4008
|
+
*
|
|
4009
|
+
* Keyword-index catch-up work is bounded by `config.setupBudgetMs`; anything
|
|
4010
|
+
* left converges on later opens and no-arg `syncSearchIndex()` calls. Not
|
|
4011
|
+
* bounded: an index reset (tokenizer change, re-attach after detach, a lost
|
|
4012
|
+
* or inconsistent index), which runs as one unchunked transaction.
|
|
4013
|
+
*/
|
|
4014
|
+
async setup(opts) {
|
|
4015
|
+
const setupBudgetMs = this.setupBudgetMs();
|
|
4016
|
+
const openDeadline = monotonicNow() + setupBudgetMs;
|
|
4086
4017
|
const entriesExistedBeforeSetup = await this.metadataRepo.tableExists(`${this.prefix}entries`);
|
|
4087
|
-
|
|
4018
|
+
const migrationPolicy = opts?.migrations ?? this.options.config?.migrations ?? "allow";
|
|
4019
|
+
await setupDatabaseWithGate(this.db, this.prefix, this.metadataRepo, migrationPolicy);
|
|
4088
4020
|
let currentVersion;
|
|
4089
4021
|
if (!entriesExistedBeforeSetup) {
|
|
4090
|
-
await this.
|
|
4022
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
4023
|
+
await createLegacySourceTypeIndex(tx, this.prefix);
|
|
4024
|
+
await writeNormalizeStamp(this.metadataRepo, tx);
|
|
4025
|
+
await this.metadataRepo.setMeta("schema_version", String(CURRENT_SCHEMA_VERSION), tx);
|
|
4026
|
+
});
|
|
4091
4027
|
currentVersion = CURRENT_SCHEMA_VERSION;
|
|
4092
4028
|
} else {
|
|
4093
|
-
|
|
4094
|
-
if (schemaVersionValue) {
|
|
4095
|
-
currentVersion = parseInt(schemaVersionValue, 10);
|
|
4096
|
-
if (!Number.isFinite(currentVersion)) currentVersion = 0;
|
|
4097
|
-
} else {
|
|
4098
|
-
const ftsDdl = await this.metadataRepo.getTableDdl(`${this.prefix}entries_fts`);
|
|
4099
|
-
const hasPorter = /tokenize\s*=\s*['"]porter\s+unicode61['"]/i.test(ftsDdl ?? "");
|
|
4100
|
-
currentVersion = hasPorter ? 1 : 0;
|
|
4101
|
-
}
|
|
4029
|
+
currentVersion = await detectSchemaVersion(this.metadataRepo, this.prefix);
|
|
4102
4030
|
}
|
|
4103
4031
|
for (const migration of MIGRATIONS) {
|
|
4104
4032
|
if (migration.version > currentVersion) {
|
|
@@ -4112,28 +4040,19 @@ var WikiMemory = class {
|
|
|
4112
4040
|
if (!schemaVersionCheck) {
|
|
4113
4041
|
await this.metadataRepo.setMeta("schema_version", String(currentVersion), this.db);
|
|
4114
4042
|
}
|
|
4115
|
-
}
|
|
4116
|
-
if (entriesExistedBeforeSetup) {
|
|
4117
4043
|
await this.importExportService.assertNoLegacySourceTypes();
|
|
4044
|
+
await runSourceRefNormalize(this.db, this.prefix, this.metadataRepo);
|
|
4118
4045
|
}
|
|
4119
|
-
const rows = await this.entryRepo.findRowsForSourceRefMigration();
|
|
4120
|
-
await this.db.withTransactionAsync(async (tx) => {
|
|
4121
|
-
for (const row of rows) {
|
|
4122
|
-
const normalized = normalizeSourceRef(row.source_ref);
|
|
4123
|
-
if (normalized !== row.source_ref) {
|
|
4124
|
-
await this.entryRepo.updateSourceRefByRowid(row.rowid, normalized, tx);
|
|
4125
|
-
}
|
|
4126
|
-
}
|
|
4127
|
-
});
|
|
4128
4046
|
this.searchService.setIndexStrategy(
|
|
4129
4047
|
await createIndexStrategy(
|
|
4130
4048
|
this.db,
|
|
4131
4049
|
this.prefix,
|
|
4132
4050
|
this.metadataRepo,
|
|
4133
|
-
this.options.config?.indexStrategy ?? "auto"
|
|
4051
|
+
this.options.config?.indexStrategy ?? "auto",
|
|
4052
|
+
{ setupBudgetMs, entriesExisted: entriesExistedBeforeSetup }
|
|
4134
4053
|
)
|
|
4135
4054
|
);
|
|
4136
|
-
await this.searchService.sync();
|
|
4055
|
+
await this.searchService.sync(void 0, { deadline: openDeadline });
|
|
4137
4056
|
}
|
|
4138
4057
|
async hasChanged(entityId, sourceRefOrEntries, sourceHashArg) {
|
|
4139
4058
|
if (typeof sourceRefOrEntries === "string") {
|
|
@@ -4384,11 +4303,18 @@ var WikiMemory = class {
|
|
|
4384
4303
|
* commit, so without this the nodes stay out of keyword, hybrid and
|
|
4385
4304
|
* fallback retrieval until the entity's next core write.
|
|
4386
4305
|
*
|
|
4387
|
-
* With `entityId
|
|
4388
|
-
*
|
|
4389
|
-
*
|
|
4390
|
-
*
|
|
4391
|
-
*
|
|
4306
|
+
* With `entityId`: drains that entity's pending `upsertGraph` ids (its new
|
|
4307
|
+
* nodes and the facts they superseded) first, to completion, then at most
|
|
4308
|
+
* one more ledger chunk — so its new nodes are searchable, and the
|
|
4309
|
+
* superseded facts gone, when the call resolves, under any backlog. Without an
|
|
4310
|
+
* argument: drains every stashed `upsertGraph` id across entities first,
|
|
4311
|
+
* then runs one keyword-index quantum bounded by `config.setupBudgetMs`
|
|
4312
|
+
* (backlog drain, then rebuild work while the index is rebuilding). Hosts
|
|
4313
|
+
* with a long-lived process should call the no-arg form on their own
|
|
4314
|
+
* schedule: that is what converges a rebuilding index. While keyword search
|
|
4315
|
+
* is unavailable it first retries the index init; after a failed attempt it
|
|
4316
|
+
* waits at least 5 s, doubling per consecutive failure up to 5 min (longer
|
|
4317
|
+
* when an attempt is itself slow), before retrying again.
|
|
4392
4318
|
*
|
|
4393
4319
|
* Call only after the host's transaction commits: called inside it, the
|
|
4394
4320
|
* rebuild may index rows that later roll back, or miss them and clear the
|
|
@@ -4405,9 +4331,53 @@ var WikiMemory = class {
|
|
|
4405
4331
|
* @throws {TypeError} when `entityId` is given but is not a non-empty string.
|
|
4406
4332
|
*/
|
|
4407
4333
|
async syncSearchIndex(entityId) {
|
|
4408
|
-
if (entityId === void 0)
|
|
4334
|
+
if (entityId === void 0) {
|
|
4335
|
+
if (!this.searchService.isKeywordAvailable() && monotonicNow() >= this.reinitRetryAt) {
|
|
4336
|
+
this.reinitSearch ?? (this.reinitSearch = (async () => {
|
|
4337
|
+
const started = monotonicNow();
|
|
4338
|
+
try {
|
|
4339
|
+
const strategy = await createIndexStrategy(
|
|
4340
|
+
this.db,
|
|
4341
|
+
this.prefix,
|
|
4342
|
+
this.metadataRepo,
|
|
4343
|
+
this.options.config?.indexStrategy ?? "auto",
|
|
4344
|
+
{ setupBudgetMs: this.setupBudgetMs() }
|
|
4345
|
+
);
|
|
4346
|
+
this.searchService.setIndexStrategy(strategy);
|
|
4347
|
+
if (this.searchService.isKeywordAvailable() && !this.searchService.hasLedger()) await this.searchService.sync();
|
|
4348
|
+
} finally {
|
|
4349
|
+
if (this.searchService.isKeywordAvailable()) {
|
|
4350
|
+
this.reinitFailures = 0;
|
|
4351
|
+
this.reinitRetryAt = 0;
|
|
4352
|
+
} else {
|
|
4353
|
+
const now = monotonicNow();
|
|
4354
|
+
const backoff = Math.min(SEARCH_REINIT_COOLDOWN_MS * 2 ** this.reinitFailures, SEARCH_REINIT_COOLDOWN_MAX_MS);
|
|
4355
|
+
this.reinitFailures++;
|
|
4356
|
+
this.reinitRetryAt = now + Math.max(backoff, SEARCH_REINIT_COST_FACTOR * (now - started));
|
|
4357
|
+
}
|
|
4358
|
+
this.reinitSearch = null;
|
|
4359
|
+
}
|
|
4360
|
+
})());
|
|
4361
|
+
await this.reinitSearch;
|
|
4362
|
+
}
|
|
4363
|
+
const stashed2 = this.peekStashedIds();
|
|
4364
|
+
const ids = [...stashed2.values()].flatMap((m) => [...m.keys()]);
|
|
4365
|
+
if (await this.syncDrained(this.searchService.syncStale({ ids }))) this.dropStashedIds(stashed2);
|
|
4366
|
+
return;
|
|
4367
|
+
}
|
|
4409
4368
|
assertEntityId(entityId);
|
|
4410
|
-
|
|
4369
|
+
const stashed = this.peekStashedIds(entityId);
|
|
4370
|
+
if (await this.syncDrained(this.searchService.syncEntries(entityId, stashed.get(entityId)?.keys() ?? []))) {
|
|
4371
|
+
this.dropStashedIds(stashed);
|
|
4372
|
+
}
|
|
4373
|
+
}
|
|
4374
|
+
/**
|
|
4375
|
+
* True when a sync applied the stashed ids. While keyword search is
|
|
4376
|
+
* unavailable the stand-in strategy's drain is a no-op that reports success,
|
|
4377
|
+
* so the ids stay stashed for the first sync after a re-init (#284 review).
|
|
4378
|
+
*/
|
|
4379
|
+
async syncDrained(sync) {
|
|
4380
|
+
return await sync && this.searchService.isKeywordAvailable();
|
|
4411
4381
|
}
|
|
4412
4382
|
async exportDump(entityIds) {
|
|
4413
4383
|
return this.importExportService.exportDump(entityIds);
|
|
@@ -4458,7 +4428,10 @@ var WikiMemory = class {
|
|
|
4458
4428
|
* post-commit work (search sync, embedding, cache eviction). After the
|
|
4459
4429
|
* transaction commits, call `syncSearchIndex(entityId)` — or
|
|
4460
4430
|
* `syncSearchIndex()` after writing several entities in one transaction —
|
|
4461
|
-
* to make the new nodes keyword-searchable
|
|
4431
|
+
* to make the new nodes keyword-searchable: both forms drain this call's node ids
|
|
4432
|
+
* first, to completion, so the nodes are searchable under any backlog (up to
|
|
4433
|
+
* `WikiMemory.UPSERT_ID_STASH_CAP` pending ids; beyond that they converge with
|
|
4434
|
+
* the backlog). Embeddings still come from the maintenance
|
|
4462
4435
|
* sweep (`runLibrarian` / `runHeal` / `runOntologyBackfill` / `runPrune`,
|
|
4463
4436
|
* scoped via `listEntityIds`).
|
|
4464
4437
|
*
|
|
@@ -4508,12 +4481,13 @@ var WikiMemory = class {
|
|
|
4508
4481
|
});
|
|
4509
4482
|
}
|
|
4510
4483
|
const diagBuffer = new DiagnosticBuffer();
|
|
4511
|
-
const result = await this.ingestionService.upsertGraphCore(
|
|
4484
|
+
const { supersededFactIds, ...result } = await this.ingestionService.upsertGraphCore(
|
|
4512
4485
|
entityId,
|
|
4513
4486
|
{ sourceRef, sourceHash, nodes: params.nodes, edges: params.edges },
|
|
4514
4487
|
adapter,
|
|
4515
4488
|
{ diag: { buffer: diagBuffer, operation: "upsertGraph" } }
|
|
4516
4489
|
);
|
|
4490
|
+
this.stashUpsertIds(entityId, [...supersededFactIds, ...params.nodes.map((n) => n.id)]);
|
|
4517
4491
|
this.searchService.markStale(entityId);
|
|
4518
4492
|
diagBuffer.flush(this.options);
|
|
4519
4493
|
return result;
|
|
@@ -4696,6 +4670,8 @@ var WikiMemory = class {
|
|
|
4696
4670
|
}
|
|
4697
4671
|
};
|
|
4698
4672
|
_testAccessNonTestEnvWarned = new WeakMap();
|
|
4673
|
+
/** Max upsertGraph node ids held for their post-commit sync (spec #280 id threading). */
|
|
4674
|
+
WikiMemory.UPSERT_ID_STASH_CAP = 1e4;
|
|
4699
4675
|
function assertEntityId(entityId) {
|
|
4700
4676
|
if (typeof entityId !== "string" || entityId.length === 0) {
|
|
4701
4677
|
throw new TypeError("Invalid entityId: must be a non-empty string.");
|
|
@@ -5621,6 +5597,6 @@ function createWiki(db, options) {
|
|
|
5621
5597
|
return new WikiMemory(db, options);
|
|
5622
5598
|
}
|
|
5623
5599
|
|
|
5624
|
-
export { DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, WikiMemory, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseOkfBundle, validateLibrarianPromptTemplate };
|
|
5600
|
+
export { DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, DEFAULT_SETUP_BUDGET_MS, WikiMemory, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseOkfBundle, validateLibrarianPromptTemplate };
|
|
5625
5601
|
//# sourceMappingURL=index.mjs.map
|
|
5626
5602
|
//# sourceMappingURL=index.mjs.map
|