@equationalapplications/core-llm-wiki 7.11.2 → 7.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -1,187 +1,25 @@
1
- import { BaseRepository, WikiGraphNodeOwnershipConflict, liveAtSql, __privateAdd, MetadataRepository, EmbeddingService, SearchService, TemporalService, JobManager, PromptService, resolveGrounding, IngestionService, MaintenanceService, ImportExportService, RetrievalService, WriteService, __privateGet, __privateSet, normalizeSourceRef, normalizeSourceHash, entitySummaryMetaKey, WikiSourceRefHashCollision, DiagnosticBuffer, validateManifest, WikiDraftNotFound, generateId, emptyManifest, resolveNodeType, WikiStrictOntologyViolation, validateInlineEdges, resolveEdgeDefinitions, typeSatisfies, normalizeTitleKey, edgeTripleAllowed, MAX_EMBED_ATTEMPTS, UsageMeter, WikiBusyError, WikiInvalidReadOptions, isLiveAt, packFactsByBudget, factTokenCost, WikiTransactionError, extractSqliteCode } from './chunk-522SCN2E.mjs';
2
- export { DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, DEFAULT_MAX_EMBED_CHARS, EMBED_CHARS_CEILING, HEAL_ANCHORS_PER_CANDIDATE, HEAL_BATCH_SIZE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, WikiBusyError, WikiDraftNotFound, WikiDuplicateHashError, WikiGraphNodeOwnershipConflict, WikiIngestEmptyError, WikiInvalidReadOptions, WikiParseError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiSupersedeError, WikiTransactionError, chunkText, configureRandomSource, parseEmbedding, safeSlice, validateManifest } from './chunk-522SCN2E.mjs';
1
+ import { BaseRepository, WikiGraphNodeOwnershipConflict, parseJsonObject, parseJsonArray, liveAtSql, reanchorRebuildFingerprint, __privateAdd, MetadataRepository, EmbeddingService, SearchService, TemporalService, JobManager, PromptService, resolveGrounding, IngestionService, MaintenanceService, ImportExportService, RetrievalService, WriteService, __privateGet, __privateSet, createLegacySourceTypeIndex, writeNormalizeStamp, CURRENT_SCHEMA_VERSION, detectSchemaVersion, MIGRATIONS, runSourceRefNormalize, normalizeSourceRef, normalizeSourceHash, entitySummaryMetaKey, WikiSourceRefHashCollision, DiagnosticBuffer, validateManifest, WikiDraftNotFound, generateId, emptyManifest, resolveNodeType, WikiStrictOntologyViolation, validateInlineEdges, resolveEdgeDefinitions, typeSatisfies, normalizeTitleKey, edgeTripleAllowed, MAX_EMBED_ATTEMPTS, UsageMeter, isWikiSearchUnavailableError, WikiBusyError, WikiInvalidReadOptions, isLiveAt, packFactsByBudget, factTokenCost, probePendingWork, WikiMigrationRequiredError, setupDatabase, UnavailableIndexStrategy, FTS5_STATE_KEY, fts5LedgerDropSql, fts5TablesDdl, fts5PendingIndexesDdl, fts5TriggersDdl, writeRebuildStart, FTS5_RETIRED_COUNTER_KEY, retiredTableName, FTS5_RETIRED_TABLES_KEY, isRetiredTableName, idReindexChunkSql, readRebuildProgress, FTS5_DRAIN_CHUNK, FTS5_RECLAIM_MERGE_PAGES, windowHiSql, windowChunkSql, FTS5_REBUILD_WM_KEY, sweepHiSql, sweepChunkSql, FTS5_SWEEP_CURSOR_KEY, drainHiSql, drainChunkSql, FTS5_REBUILD_SNAPSHOT_KEY, idDrainChunkSql, entityDrainHiSql, entityDrainChunkSql, fts5TriggerNames, extractSqliteCode, WikiTransactionError } from './chunk-PCRYIUV5.mjs';
2
+ export { DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, DEFAULT_MAX_EMBED_CHARS, EMBED_CHARS_CEILING, HEAL_ANCHORS_PER_CANDIDATE, HEAL_BATCH_SIZE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, WikiBusyError, WikiDraftNotFound, WikiDuplicateHashError, WikiGraphNodeOwnershipConflict, WikiIngestEmptyError, WikiInvalidReadOptions, WikiMigrationRequiredError, WikiParseError, WikiSearchUnavailableError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiSupersedeError, WikiTransactionError, chunkText, configureRandomSource, isWikiSearchUnavailableError, parseEmbedding, safeSlice, validateManifest } from './chunk-PCRYIUV5.mjs';
3
3
  import { deriveTrustTier, isStaleAfter, appendRelatedSection, buildConceptDocument, buildLogMd, buildEntityIndexMd, buildRootIndexMd, isAllowedOkfPath, parseRootIndexMd, parseEntityIndexMd, parseConcept, splitRelatedSection, extractMarkdownLinks, parseLogMd, parseEventIdComment, latestVerified, formatSourcesJson, appendEventIdComment, parseVerifiedFlexible, parseCitationsList } from '@equationalapplications/core-okf';
4
4
  import MiniSearch from 'minisearch';
5
5
 
6
- // src/db/schema.ts
7
- async function setupDatabase(db, prefix) {
8
- await db.execAsync(`
9
- CREATE TABLE IF NOT EXISTS ${prefix}entries (
10
- id TEXT PRIMARY KEY,
11
- entity_id TEXT NOT NULL,
12
- title TEXT NOT NULL,
13
- body TEXT NOT NULL,
14
- tags TEXT NOT NULL DEFAULT '[]',
15
- confidence TEXT NOT NULL DEFAULT 'inferred',
16
- source_type TEXT NOT NULL DEFAULT 'librarian_inferred',
17
- source_hash TEXT,
18
- source_ref TEXT,
19
- created_at INTEGER NOT NULL,
20
- updated_at INTEGER NOT NULL,
21
- last_accessed_at INTEGER,
22
- access_count INTEGER NOT NULL DEFAULT 0,
23
- deleted_at INTEGER,
24
- embedding TEXT,
25
- embedding_blob BLOB,
26
- okf_type TEXT,
27
- ontology_checked_at INTEGER,
28
- heal_checked_at INTEGER,
29
- lifecycle_status TEXT NOT NULL DEFAULT 'stable',
30
- stale_after INTEGER,
31
- generated_by TEXT,
32
- last_verified_at INTEGER,
33
- last_verified_by TEXT,
34
- okf_sources TEXT,
35
- okf_verified TEXT,
36
- okf_usage_window TEXT,
37
- embedding_failed_at INTEGER,
38
- embedding_failure_kind TEXT,
39
- embedding_attempts INTEGER NOT NULL DEFAULT 0,
40
- valid_from INTEGER,
41
- valid_to INTEGER,
42
- superseded_by TEXT,
43
- superseded_at INTEGER
44
- );
45
-
46
- CREATE INDEX IF NOT EXISTS ${prefix}entries_entity_idx ON ${prefix}entries(entity_id);
47
- CREATE INDEX IF NOT EXISTS ${prefix}entries_source_ref_idx ON ${prefix}entries(entity_id, source_ref);
48
- CREATE INDEX IF NOT EXISTS ${prefix}entries_source_hash_idx ON ${prefix}entries(entity_id, source_hash) WHERE source_hash IS NOT NULL;
49
- CREATE INDEX IF NOT EXISTS ${prefix}entries_updated_idx ON ${prefix}entries(updated_at DESC);
50
-
51
- -- source_ref_index: per-(entity, source_hash) record of the canonical sourceRef
52
- -- currently holding that hash. The partial UNIQUE index on (entity_id, source_hash)
53
- -- WHERE deleted_at IS NULL enforces the sourceRef-level TOCTOU-race invariant;
54
- -- entries-level uniqueness cannot express it because a single ingestDocument call
55
- -- writes N facts that all share (entity_id, source_ref, source_hash). See
56
- -- docs/superpowers/specs/2026-08-07-dependabot-concurrency-release-hygiene-design.md \xA7B1.
57
- CREATE TABLE IF NOT EXISTS ${prefix}source_ref_index (
58
- id TEXT PRIMARY KEY,
59
- entity_id TEXT NOT NULL,
60
- source_hash TEXT NOT NULL,
61
- source_ref TEXT NOT NULL,
62
- created_at INTEGER NOT NULL,
63
- deleted_at INTEGER
64
- );
65
- CREATE UNIQUE INDEX IF NOT EXISTS ${prefix}idx_source_ref_hash
66
- ON ${prefix}source_ref_index (entity_id, source_hash)
67
- WHERE deleted_at IS NULL;
68
-
69
- CREATE TABLE IF NOT EXISTS ${prefix}tasks (
70
- id TEXT PRIMARY KEY,
71
- entity_id TEXT NOT NULL,
72
- description TEXT NOT NULL,
73
- status TEXT NOT NULL DEFAULT 'pending',
74
- priority INTEGER NOT NULL DEFAULT 0,
75
- created_at INTEGER NOT NULL,
76
- updated_at INTEGER NOT NULL,
77
- resolved_at INTEGER,
78
- deleted_at INTEGER,
79
- okf_type TEXT,
80
- lifecycle_status TEXT NOT NULL DEFAULT 'stable',
81
- stale_after INTEGER,
82
- generated_by TEXT,
83
- last_verified_at INTEGER,
84
- last_verified_by TEXT,
85
- okf_sources TEXT,
86
- okf_verified TEXT,
87
- okf_usage_window TEXT
88
- );
89
-
90
- CREATE INDEX IF NOT EXISTS ${prefix}tasks_entity_idx ON ${prefix}tasks(entity_id, status);
91
-
92
- CREATE TABLE IF NOT EXISTS ${prefix}edges (
93
- id TEXT PRIMARY KEY,
94
- entity_id TEXT NOT NULL,
95
- source_id TEXT NOT NULL,
96
- target_id TEXT NOT NULL,
97
- edge_type TEXT NOT NULL,
98
- created_at INTEGER NOT NULL,
99
- UNIQUE(entity_id, source_id, target_id, edge_type)
100
- );
101
-
102
- CREATE INDEX IF NOT EXISTS ${prefix}edges_entity_id_idx ON ${prefix}edges(entity_id, id);
103
-
104
- CREATE TABLE IF NOT EXISTS ${prefix}events (
105
- id TEXT PRIMARY KEY,
106
- entity_id TEXT NOT NULL,
107
- event_type TEXT NOT NULL,
108
- summary TEXT NOT NULL,
109
- related_entry_id TEXT,
110
- created_at INTEGER NOT NULL,
111
- occurred_at INTEGER
112
- );
113
-
114
- CREATE INDEX IF NOT EXISTS ${prefix}events_entity_idx ON ${prefix}events(entity_id, created_at DESC);
115
-
116
- CREATE TABLE IF NOT EXISTS ${prefix}checkpoints (
117
- entity_id TEXT PRIMARY KEY,
118
- heal_checkpoint INTEGER NOT NULL DEFAULT 0,
119
- memory_checkpoint INTEGER NOT NULL DEFAULT 0,
120
- librarian_watermark_at INTEGER,
121
- librarian_watermark_id TEXT
122
- );
123
-
124
- CREATE TABLE IF NOT EXISTS ${prefix}entity_manifests (
125
- entity_id TEXT PRIMARY KEY,
126
- mode TEXT NOT NULL DEFAULT 'off',
127
- manifest_json TEXT NOT NULL DEFAULT '{"node_types":[],"edge_types":[]}',
128
- updated_at INTEGER NOT NULL
129
- );
130
-
131
- CREATE TABLE IF NOT EXISTS ${prefix}meta (
132
- key TEXT PRIMARY KEY,
133
- value TEXT NOT NULL
134
- );
135
-
136
- CREATE TABLE IF NOT EXISTS ${prefix}outbox (
137
- id TEXT PRIMARY KEY,
138
- entity_id TEXT NOT NULL,
139
- table_name TEXT NOT NULL,
140
- record_id TEXT NOT NULL,
141
- operation TEXT NOT NULL,
142
- payload TEXT NOT NULL,
143
- created_at INTEGER NOT NULL
144
- );
145
-
146
- CREATE INDEX IF NOT EXISTS ${prefix}outbox_entity_id_created_at
147
- ON ${prefix}outbox (entity_id, created_at);
148
-
149
- CREATE INDEX IF NOT EXISTS ${prefix}outbox_created_at
150
- ON ${prefix}outbox (created_at);
151
- `);
152
- await createOkfV02IndexesIfColumnsExist(db, prefix);
153
- await createTemporalIndexesIfColumnsExist(db, prefix);
154
- }
155
- async function createOkfV02IndexesIfColumnsExist(db, prefix) {
156
- for (const table of ["entries", "tasks"]) {
157
- const cols = await db.getAllAsync(
158
- `PRAGMA table_info(${prefix}${table})`
159
- );
160
- const colNames = new Set(cols.map((c) => c.name));
161
- if (colNames.has("lifecycle_status")) {
162
- await db.execAsync(`CREATE INDEX IF NOT EXISTS ${prefix}${table}_lifecycle_status_idx ON ${prefix}${table}(lifecycle_status);`);
163
- }
164
- if (colNames.has("stale_after")) {
165
- await db.execAsync(`CREATE INDEX IF NOT EXISTS ${prefix}${table}_stale_after_idx ON ${prefix}${table}(stale_after);`);
166
- }
167
- if (colNames.has("last_verified_at")) {
168
- await db.execAsync(`CREATE INDEX IF NOT EXISTS ${prefix}${table}_last_verified_at_idx ON ${prefix}${table}(last_verified_at);`);
6
+ // src/db/migrationGate.ts
7
+ var MIGRATION_POLICIES = ["allow", "refuse", "auto"];
8
+ async function setupDatabaseWithGate(db, prefix, meta, migrations = "allow") {
9
+ if (!MIGRATION_POLICIES.includes(migrations)) {
10
+ throw new TypeError(`Invalid migrations policy ${JSON.stringify(migrations)}; expected 'allow', 'refuse' or 'auto'.`);
11
+ }
12
+ if (migrations !== "allow") {
13
+ const pending = await probePendingWork(db, meta, prefix);
14
+ if (pending && (migrations === "refuse" || pending.type === "schema_migration")) {
15
+ throw new WikiMigrationRequiredError(describePending(pending, migrations), pending);
169
16
  }
170
17
  }
18
+ return setupDatabase(db, prefix);
171
19
  }
172
- async function createTemporalIndexesIfColumnsExist(db, prefix) {
173
- const cols = await db.getAllAsync(`PRAGMA table_info(${prefix}entries)`);
174
- const names = new Set(cols.map((c) => c.name));
175
- if (names.has("superseded_by")) {
176
- await db.execAsync(
177
- `CREATE INDEX IF NOT EXISTS ${prefix}entries_superseded_idx ON ${prefix}entries(entity_id, superseded_by) WHERE superseded_by IS NOT NULL;`
178
- );
179
- }
180
- if (names.has("valid_from") && names.has("valid_to")) {
181
- await db.execAsync(
182
- `CREATE INDEX IF NOT EXISTS ${prefix}entries_temporal_idx ON ${prefix}entries(entity_id) WHERE valid_from IS NOT NULL OR valid_to IS NOT NULL;`
183
- );
184
- }
20
+ function describePending(pending, policy) {
21
+ const what = pending.type === "schema_migration" ? `schema migration v${pending.fromVersion} -> v${pending.toVersion} is pending` : pending.entriesWatermark === 0 && pending.sourceRefIndexWatermark === 0 ? "source_ref normalize has not run on this store (normalize stamp missing or outdated; may complete with zero rewrites)" : "source_ref normalize was interrupted and will resume from its watermark";
22
+ return `Migration refused (migrations: '${policy}'): ${what}. Open once with setup({ migrations: 'allow' }) outside any time-critical path to complete it.`;
185
23
  }
186
24
 
187
25
  // src/db/serializedAdapter.ts
@@ -227,314 +65,6 @@ function withSerializedTransactions(db) {
227
65
  }
228
66
  });
229
67
  }
230
-
231
- // src/db/migrations.ts
232
- var MIGRATIONS = [
233
- {
234
- version: 1,
235
- description: "Rebuild FTS5 with porter unicode61 tokenizer (superseded by v2)",
236
- run: async (_db, _prefix) => {
237
- }
238
- },
239
- {
240
- version: 2,
241
- description: "Remove FTS5; add embedding column for semantic retrieval",
242
- run: async (db, prefix) => {
243
- await db.withTransactionAsync(async (tx) => {
244
- await tx.execAsync(`
245
- DROP TRIGGER IF EXISTS ${prefix}entries_ai;
246
- DROP TRIGGER IF EXISTS ${prefix}entries_ad;
247
- DROP TRIGGER IF EXISTS ${prefix}entries_au;
248
- DROP TABLE IF EXISTS ${prefix}entries_fts;
249
- `);
250
- });
251
- const cols = await db.getAllAsync(
252
- `PRAGMA table_info(${prefix}entries)`
253
- );
254
- if (!cols.some((c) => c.name === "embedding")) {
255
- await db.execAsync(`ALTER TABLE ${prefix}entries ADD COLUMN embedding TEXT`);
256
- }
257
- }
258
- },
259
- {
260
- version: 3,
261
- description: "Add embedding_blob BLOB column for Float32Array vector storage",
262
- run: async (db, prefix) => {
263
- const cols = await db.getAllAsync(
264
- `PRAGMA table_info(${prefix}entries)`
265
- );
266
- if (!cols.some((c) => c.name === "embedding_blob")) {
267
- await db.execAsync(
268
- `ALTER TABLE ${prefix}entries ADD COLUMN embedding_blob BLOB`
269
- );
270
- }
271
- }
272
- },
273
- {
274
- version: 4,
275
- description: "Create outbox table for change data capture",
276
- run: async (db, prefix) => {
277
- await db.execAsync(`
278
- CREATE TABLE IF NOT EXISTS ${prefix}outbox (
279
- id TEXT PRIMARY KEY,
280
- entity_id TEXT NOT NULL,
281
- table_name TEXT NOT NULL,
282
- record_id TEXT NOT NULL,
283
- operation TEXT NOT NULL,
284
- payload TEXT NOT NULL,
285
- created_at INTEGER NOT NULL
286
- );
287
- CREATE INDEX IF NOT EXISTS ${prefix}outbox_entity_id_created_at
288
- ON ${prefix}outbox (entity_id, created_at);
289
- `);
290
- }
291
- },
292
- {
293
- version: 5,
294
- description: "Add okf_type to entries/tasks for OKF type fidelity; create edges table for OKF graph import",
295
- run: async (db, prefix) => {
296
- for (const table of ["entries", "tasks"]) {
297
- const cols = await db.getAllAsync(
298
- `PRAGMA table_info(${prefix}${table})`
299
- );
300
- if (!cols.some((c) => c.name === "okf_type")) {
301
- await db.execAsync(`ALTER TABLE ${prefix}${table} ADD COLUMN okf_type TEXT`);
302
- }
303
- }
304
- await db.execAsync(`
305
- CREATE TABLE IF NOT EXISTS ${prefix}edges (
306
- id TEXT PRIMARY KEY,
307
- entity_id TEXT NOT NULL,
308
- source_id TEXT NOT NULL,
309
- target_id TEXT NOT NULL,
310
- edge_type TEXT NOT NULL,
311
- created_at INTEGER NOT NULL,
312
- UNIQUE(entity_id, source_id, target_id, edge_type)
313
- );
314
- CREATE INDEX IF NOT EXISTS ${prefix}edges_entity_idx ON ${prefix}edges (entity_id);
315
- `);
316
- }
317
- },
318
- {
319
- version: 6,
320
- description: "Add entity_manifests table for per-entity ontology state",
321
- run: async (db, prefix) => {
322
- await db.execAsync(`
323
- CREATE TABLE IF NOT EXISTS ${prefix}entity_manifests (
324
- entity_id TEXT PRIMARY KEY,
325
- mode TEXT NOT NULL DEFAULT 'off',
326
- manifest_json TEXT NOT NULL DEFAULT '{"node_types":[],"edge_types":[]}',
327
- updated_at INTEGER NOT NULL
328
- );
329
- `);
330
- }
331
- },
332
- {
333
- version: 7,
334
- description: "Add ontology_checked_at to entries for ontology backfill recheck cooldown",
335
- run: async (db, prefix) => {
336
- const cols = await db.getAllAsync(
337
- `PRAGMA table_info(${prefix}entries)`
338
- );
339
- if (!cols.some((c) => c.name === "ontology_checked_at")) {
340
- await db.execAsync(
341
- `ALTER TABLE ${prefix}entries ADD COLUMN ontology_checked_at INTEGER`
342
- );
343
- }
344
- }
345
- },
346
- {
347
- version: 8,
348
- description: "Add heal_checked_at to entries for heal recheck cooldown",
349
- run: async (db, prefix) => {
350
- const cols = await db.getAllAsync(
351
- `PRAGMA table_info(${prefix}entries)`
352
- );
353
- if (!cols.some((c) => c.name === "heal_checked_at")) {
354
- await db.execAsync(
355
- `ALTER TABLE ${prefix}entries ADD COLUMN heal_checked_at INTEGER`
356
- );
357
- }
358
- }
359
- },
360
- {
361
- version: 9,
362
- description: "add_source_ref_index",
363
- run: async (db, prefix) => {
364
- const duplicates = await db.getAllAsync(
365
- `SELECT entity_id, source_hash, COUNT(DISTINCT source_ref) AS n_refs
366
- FROM ${prefix}entries
367
- WHERE deleted_at IS NULL AND source_hash IS NOT NULL
368
- GROUP BY entity_id, source_hash
369
- HAVING COUNT(DISTINCT source_ref) > 1`
370
- );
371
- if (duplicates.length > 0) {
372
- const sample = duplicates.slice(0, 5).map((d) => `(entity_id=${d.entity_id}, source_hash=${d.source_hash.slice(0, 12)}\u2026, n_refs=${d.n_refs})`).join(", ");
373
- throw new Error(
374
- `Migration v9 (add_source_ref_index) failed: existing live rows have multiple sourceRefs sharing a hash. Found ${duplicates.length} duplicate (entity_id, source_hash) groups. First ${Math.min(5, duplicates.length)}: ${sample}. Resolve each by calling forget({ sourceRef: <loser> }) for the offending sourceRef, then re-run setup.`
375
- );
376
- }
377
- await db.execAsync(
378
- `CREATE TABLE IF NOT EXISTS ${prefix}source_ref_index (
379
- id TEXT PRIMARY KEY,
380
- entity_id TEXT NOT NULL,
381
- source_hash TEXT NOT NULL,
382
- source_ref TEXT NOT NULL,
383
- created_at INTEGER NOT NULL,
384
- deleted_at INTEGER
385
- );
386
- CREATE UNIQUE INDEX IF NOT EXISTS ${prefix}idx_source_ref_hash
387
- ON ${prefix}source_ref_index (entity_id, source_hash)
388
- WHERE deleted_at IS NULL;`
389
- );
390
- await db.execAsync(
391
- `INSERT OR IGNORE INTO ${prefix}source_ref_index (id, entity_id, source_hash, source_ref, created_at, deleted_at)
392
- SELECT
393
- 'sri:' || entity_id || ':' || source_hash,
394
- entity_id,
395
- source_hash,
396
- source_ref,
397
- updated_at,
398
- NULL
399
- FROM (
400
- SELECT
401
- entity_id, source_hash, source_ref, updated_at,
402
- ROW_NUMBER() OVER (
403
- PARTITION BY entity_id, source_hash
404
- ORDER BY updated_at ASC, id ASC
405
- ) AS rn
406
- FROM ${prefix}entries
407
- WHERE deleted_at IS NULL
408
- AND source_hash IS NOT NULL
409
- AND source_ref IS NOT NULL
410
- ) ranked
411
- WHERE rn = 1;`
412
- );
413
- }
414
- },
415
- {
416
- version: 10,
417
- description: "OKF v0.2: add lifecycle_status, stale_after, generated_by, last_verified_*, okf_sources, okf_verified, okf_usage_window to entries and tasks",
418
- run: async (db, prefix) => {
419
- for (const table of ["entries", "tasks"]) {
420
- const cols = await db.getAllAsync(
421
- `PRAGMA table_info(${prefix}${table})`
422
- );
423
- const existing = new Set(cols.map((c) => c.name));
424
- const adds = [
425
- ["lifecycle_status", `ALTER TABLE ${prefix}${table} ADD COLUMN lifecycle_status TEXT NOT NULL DEFAULT 'stable'`],
426
- ["stale_after", `ALTER TABLE ${prefix}${table} ADD COLUMN stale_after INTEGER`],
427
- ["generated_by", `ALTER TABLE ${prefix}${table} ADD COLUMN generated_by TEXT`],
428
- ["last_verified_at", `ALTER TABLE ${prefix}${table} ADD COLUMN last_verified_at INTEGER`],
429
- ["last_verified_by", `ALTER TABLE ${prefix}${table} ADD COLUMN last_verified_by TEXT`],
430
- ["okf_sources", `ALTER TABLE ${prefix}${table} ADD COLUMN okf_sources TEXT`],
431
- ["okf_verified", `ALTER TABLE ${prefix}${table} ADD COLUMN okf_verified TEXT`],
432
- ["okf_usage_window", `ALTER TABLE ${prefix}${table} ADD COLUMN okf_usage_window TEXT`]
433
- ];
434
- for (const [name, sql] of adds) {
435
- if (!existing.has(name)) await db.execAsync(sql);
436
- }
437
- }
438
- await db.execAsync(`
439
- CREATE INDEX IF NOT EXISTS ${prefix}entries_lifecycle_status_idx ON ${prefix}entries(lifecycle_status);
440
- CREATE INDEX IF NOT EXISTS ${prefix}entries_stale_after_idx ON ${prefix}entries(stale_after);
441
- CREATE INDEX IF NOT EXISTS ${prefix}entries_last_verified_at_idx ON ${prefix}entries(last_verified_at);
442
- CREATE INDEX IF NOT EXISTS ${prefix}tasks_lifecycle_status_idx ON ${prefix}tasks(lifecycle_status);
443
- CREATE INDEX IF NOT EXISTS ${prefix}tasks_stale_after_idx ON ${prefix}tasks(stale_after);
444
- CREATE INDEX IF NOT EXISTS ${prefix}tasks_last_verified_at_idx ON ${prefix}tasks(last_verified_at);
445
- `);
446
- }
447
- },
448
- {
449
- version: 11,
450
- description: "Add embedding_failed_at, embedding_failure_kind, embedding_attempts to entries for embed retry state",
451
- run: async (db, prefix) => {
452
- const cols = await db.getAllAsync(
453
- `PRAGMA table_info(${prefix}entries)`
454
- );
455
- const existing = new Set(cols.map((c) => c.name));
456
- const adds = [
457
- ["embedding_failed_at", `ALTER TABLE ${prefix}entries ADD COLUMN embedding_failed_at INTEGER`],
458
- ["embedding_failure_kind", `ALTER TABLE ${prefix}entries ADD COLUMN embedding_failure_kind TEXT`],
459
- ["embedding_attempts", `ALTER TABLE ${prefix}entries ADD COLUMN embedding_attempts INTEGER NOT NULL DEFAULT 0`]
460
- ];
461
- for (const [name, sql] of adds) {
462
- if (!existing.has(name)) await db.execAsync(sql);
463
- }
464
- }
465
- },
466
- {
467
- version: 12,
468
- description: "Replace edges_entity_idx with composite edges(entity_id, id) for keyset paging",
469
- run: async (db, prefix) => {
470
- await db.execAsync(`
471
- CREATE INDEX IF NOT EXISTS ${prefix}edges_entity_id_idx ON ${prefix}edges(entity_id, id);
472
- DROP INDEX IF EXISTS ${prefix}edges_entity_idx;
473
- `);
474
- }
475
- },
476
- {
477
- version: 13,
478
- description: "Temporal facts: valid_from/valid_to/superseded_by/superseded_at on entries, occurred_at on events, librarian watermark on checkpoints",
479
- run: async (db, prefix) => {
480
- const adds = [
481
- ["entries", "valid_from", "INTEGER"],
482
- ["entries", "valid_to", "INTEGER"],
483
- ["entries", "superseded_by", "TEXT"],
484
- ["entries", "superseded_at", "INTEGER"],
485
- ["events", "occurred_at", "INTEGER"],
486
- ["checkpoints", "librarian_watermark_at", "INTEGER"],
487
- ["checkpoints", "librarian_watermark_id", "TEXT"]
488
- ];
489
- const existingByTable = /* @__PURE__ */ new Map();
490
- for (const [table, column, type] of adds) {
491
- let existing = existingByTable.get(table);
492
- if (!existing) {
493
- const cols = await db.getAllAsync(`PRAGMA table_info(${prefix}${table})`);
494
- existing = new Set(cols.map((c) => c.name));
495
- existingByTable.set(table, existing);
496
- }
497
- if (!existing.has(column)) {
498
- await db.execAsync(`ALTER TABLE ${prefix}${table} ADD COLUMN ${column} ${type}`);
499
- existing.add(column);
500
- }
501
- }
502
- await createTemporalIndexesIfColumnsExist(db, prefix);
503
- }
504
- }
505
- ];
506
- for (let i = 1; i < MIGRATIONS.length; i++) {
507
- if (MIGRATIONS[i].version <= MIGRATIONS[i - 1].version) {
508
- throw new Error(
509
- `migrations.ts: MIGRATIONS must be in strictly ascending version order. Found version ${MIGRATIONS[i].version} after ${MIGRATIONS[i - 1].version} at index ${i}.`
510
- );
511
- }
512
- }
513
- var CURRENT_SCHEMA_VERSION = MIGRATIONS.length > 0 ? MIGRATIONS[MIGRATIONS.length - 1].version : 0;
514
-
515
- // src/repositories/rowMappers.ts
516
- function parseJsonArray(s, fallback) {
517
- if (Array.isArray(s)) return s;
518
- if (typeof s === "string" && s.length > 0) {
519
- try {
520
- const p = JSON.parse(s);
521
- if (Array.isArray(p)) return p;
522
- } catch {
523
- }
524
- }
525
- return fallback;
526
- }
527
- function parseJsonObject(s, fallback = null) {
528
- if (s && typeof s === "object" && !Array.isArray(s)) return s;
529
- if (typeof s === "string" && s.length > 0) {
530
- try {
531
- const p = JSON.parse(s);
532
- if (p && typeof p === "object" && !Array.isArray(p)) return p;
533
- } catch {
534
- }
535
- }
536
- return fallback;
537
- }
538
68
  function mapRowToFact(row) {
539
69
  const tags = (() => {
540
70
  if (Array.isArray(row.tags)) return row.tags;
@@ -612,6 +142,15 @@ var _EntryRepository = class _EntryRepository extends BaseRepository {
612
142
  this.outbox = outbox;
613
143
  this.chunkSize = 500;
614
144
  }
145
+ /** Whether the entity has any non-deleted row (librarian gate: keyword ranking over zero facts is trivially empty). */
146
+ async hasLiveEntry(entityId, tx) {
147
+ const executor = this.getExecutor(tx);
148
+ const row = await executor.getFirstAsync(
149
+ `SELECT 1 AS one FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL LIMIT 1`,
150
+ [entityId]
151
+ );
152
+ return row != null;
153
+ }
615
154
  /**
616
155
  * Fetch facts by IDs, optionally scoped to entity IDs.
617
156
  * Returns facts in the order of the input IDs (first match wins).
@@ -1156,6 +695,7 @@ var _EntryRepository = class _EntryRepository extends BaseRepository {
1156
695
  }
1157
696
  }
1158
697
  }
698
+ if (totalChanges > 0) await reanchorRebuildFingerprint(executor, this.prefix, ids);
1159
699
  return totalChanges;
1160
700
  }
1161
701
  /**
@@ -1498,27 +1038,6 @@ var _EntryRepository = class _EntryRepository extends BaseRepository {
1498
1038
  `SELECT * FROM ${this.prefix}entries WHERE deleted_at IS NULL`
1499
1039
  );
1500
1040
  }
1501
- async findRowsForSourceRefMigration(tx) {
1502
- const executor = this.getExecutor(tx);
1503
- return executor.getAllAsync(
1504
- `SELECT rowid, source_ref FROM ${this.prefix}entries
1505
- WHERE source_ref IS NOT NULL
1506
- AND (
1507
- TRIM(source_ref) != source_ref
1508
- OR INSTR(source_ref, '/') > 0
1509
- OR INSTR(source_ref, '\\') > 0
1510
- OR INSTR(source_ref, CHAR(0)) > 0
1511
- OR source_ref GLOB '*[^-A-Za-z0-9._ ]*'
1512
- )`
1513
- );
1514
- }
1515
- async updateSourceRefByRowid(rowid, sourceRef, tx) {
1516
- const executor = this.getExecutor(tx);
1517
- await executor.runAsync(
1518
- `UPDATE ${this.prefix}entries SET source_ref = ? WHERE rowid = ?`,
1519
- [sourceRef, rowid]
1520
- );
1521
- }
1522
1041
  async findLatestSourceHash(entityId, sourceRef, tx) {
1523
1042
  const executor = this.getExecutor(tx);
1524
1043
  const row = await executor.getFirstAsync(
@@ -2994,87 +2513,11 @@ function assertOwnedBy(entityId, documents) {
2994
2513
  }
2995
2514
  }
2996
2515
 
2997
- // src/services/search/fts5Sql.ts
2998
- var FTS5_STATE_KEY = "fts5_index_state";
2999
- var FTS5_DRAIN_CHUNK = 500;
3000
- function fts5TriggerNames(prefix) {
3001
- return [`${prefix}entries_fts_ai`, `${prefix}entries_fts_au`, `${prefix}entries_fts_ad`];
3002
- }
3003
- function tagsExpr(col) {
3004
- return `CASE WHEN json_valid(${col}) THEN
3005
- CASE WHEN json_type(${col}) = 'array' THEN
3006
- (SELECT group_concat(CASE type WHEN 'true' THEN 'true' WHEN 'false' THEN 'false'
3007
- WHEN 'null' THEN '' ELSE value END, ' ')
3008
- FROM json_each(${col}))
3009
- ELSE ${col} END
3010
- ELSE ${col} END`;
3011
- }
3012
- function fts5TablesDdl(p) {
3013
- return `
3014
- CREATE VIRTUAL TABLE IF NOT EXISTS ${p}entries_fts USING fts5(
3015
- id UNINDEXED, entity_id UNINDEXED, title, body, tags,
3016
- tokenize = 'porter unicode61'
3017
- );
3018
- CREATE TABLE IF NOT EXISTS ${p}fts_map (
3019
- fts_rowid INTEGER PRIMARY KEY,
3020
- id TEXT NOT NULL UNIQUE,
3021
- entity_id TEXT NOT NULL
3022
- );
3023
- CREATE INDEX IF NOT EXISTS ${p}fts_map_entity ON ${p}fts_map(entity_id);
3024
- CREATE TABLE IF NOT EXISTS ${p}fts_pending (
3025
- seq INTEGER PRIMARY KEY,
3026
- entity_id TEXT NOT NULL,
3027
- id TEXT NOT NULL
3028
- );`;
3029
- }
3030
- function fts5TriggersDdl(p) {
3031
- const [ai, au, ad] = fts5TriggerNames(p);
3032
- return `
3033
- CREATE TRIGGER IF NOT EXISTS ${ai} AFTER INSERT ON ${p}entries BEGIN
3034
- INSERT INTO ${p}fts_pending (entity_id, id) VALUES (NEW.entity_id, NEW.id);
3035
- END;
3036
- CREATE TRIGGER IF NOT EXISTS ${au}
3037
- AFTER UPDATE OF id, entity_id, title, body, tags, deleted_at ON ${p}entries BEGIN
3038
- INSERT INTO ${p}fts_pending (entity_id, id) VALUES (OLD.entity_id, OLD.id);
3039
- INSERT INTO ${p}fts_pending (entity_id, id)
3040
- SELECT NEW.entity_id, NEW.id WHERE NEW.id <> OLD.id OR NEW.entity_id <> OLD.entity_id;
3041
- END;
3042
- CREATE TRIGGER IF NOT EXISTS ${ad} AFTER DELETE ON ${p}entries BEGIN
3043
- INSERT INTO ${p}fts_pending (entity_id, id) VALUES (OLD.entity_id, OLD.id);
3044
- END;`;
3045
- }
3046
- function drainHiSql(p) {
3047
- return `SELECT max(seq) AS hi FROM (SELECT seq FROM ${p}fts_pending ORDER BY seq LIMIT ${FTS5_DRAIN_CHUNK})`;
3048
- }
3049
- function drainChunkSql(p) {
3050
- const chunkIds = `SELECT id FROM ${p}fts_pending WHERE seq <= ?`;
3051
- return [
3052
- `DELETE FROM ${p}entries_fts WHERE rowid IN (SELECT m.fts_rowid FROM ${p}fts_map m WHERE m.id IN (${chunkIds}))`,
3053
- `DELETE FROM ${p}fts_map WHERE id IN (${chunkIds})`,
3054
- `INSERT INTO ${p}fts_map (id, entity_id)
3055
- SELECT e.id, e.entity_id FROM ${p}entries e WHERE e.id IN (${chunkIds}) AND e.deleted_at IS NULL`,
3056
- `INSERT INTO ${p}entries_fts (rowid, id, entity_id, title, body, tags)
3057
- SELECT m.fts_rowid, e.id, e.entity_id, e.title, e.body, ${tagsExpr("e.tags")}
3058
- FROM ${p}fts_map m JOIN ${p}entries e ON e.id = m.id
3059
- WHERE m.id IN (${chunkIds})`,
3060
- `DELETE FROM ${p}fts_pending WHERE seq <= ?`
3061
- ];
3062
- }
3063
- function rebuildSql(p) {
3064
- return [
3065
- `INSERT INTO ${p}fts_map (id, entity_id) SELECT id, entity_id FROM ${p}entries WHERE deleted_at IS NULL`,
3066
- `INSERT INTO ${p}entries_fts (rowid, id, entity_id, title, body, tags)
3067
- SELECT m.fts_rowid, e.id, e.entity_id, e.title, e.body, ${tagsExpr("e.tags")}
3068
- FROM ${p}fts_map m JOIN ${p}entries e ON e.id = m.id`
3069
- ];
3070
- }
3071
- function fts5LedgerDropSql(p) {
3072
- return [
3073
- ...fts5TriggerNames(p).map((n) => `DROP TRIGGER IF EXISTS ${n};`),
3074
- `DROP TABLE IF EXISTS ${p}fts_pending;`,
3075
- `DROP TABLE IF EXISTS ${p}fts_map;`
3076
- ].join("\n");
3077
- }
2516
+ // src/utils/monotonicNow.ts
2517
+ var monotonicNow = () => typeof performance !== "undefined" && typeof performance.now === "function" ? performance.now() : Date.now();
2518
+
2519
+ // src/utils/setupDefaults.ts
2520
+ var DEFAULT_SETUP_BUDGET_MS = 1500;
3078
2521
 
3079
2522
  // src/services/search/ftsQuery.ts
3080
2523
  var TOKEN = /[\p{L}\p{N}]+/gu;
@@ -3087,40 +2530,362 @@ function buildFtsMatchQuery(query) {
3087
2530
 
3088
2531
  // src/services/search/Fts5IndexStrategy.ts
3089
2532
  var Fts5IndexStrategy = class {
3090
- constructor(db, prefix, metadataRepo) {
2533
+ constructor(db, prefix, metadataRepo, budgetMs = DEFAULT_SETUP_BUDGET_MS) {
3091
2534
  this.db = db;
3092
2535
  this.prefix = prefix;
3093
2536
  this.metadataRepo = metadataRepo;
2537
+ this.budgetMs = budgetMs;
2538
+ }
2539
+ /**
2540
+ * Resolves the index state (spec #280 Change 3 transition table). Every
2541
+ * read-only probe runs before the one transition transaction, so a probe
2542
+ * failure leaves zero mutation (Change 4 ordering).
2543
+ */
2544
+ async init(opts = {}) {
2545
+ try {
2546
+ await this.initOnce(opts);
2547
+ } catch (err) {
2548
+ const wrapped = unwrapFtsOnly(err);
2549
+ const cls = ftsCorruptionClass(wrapped.cause);
2550
+ if (cls === "vtab" || cls === "corrupt" && wrapped.ftsOnly) {
2551
+ await this.resetAndRearm();
2552
+ return;
2553
+ }
2554
+ throw wrapped.cause;
2555
+ }
2556
+ }
2557
+ /** Marks errors from statements that touch only FTS tables (entries_fts + shadows). */
2558
+ async ftsOnly(fn) {
2559
+ try {
2560
+ return await fn();
2561
+ } catch (err) {
2562
+ throw new FtsOnlyStatementError(err);
2563
+ }
3094
2564
  }
3095
- async init() {
2565
+ async initOnce(opts) {
2566
+ const p = this.prefix;
3096
2567
  const state = await this.metadataRepo.getMeta(FTS5_STATE_KEY);
3097
- if (state === "live" && await this.triggersPresent()) {
3098
- await this.db.execAsync(fts5TablesDdl(this.prefix));
2568
+ const schema = await this.readSchema();
2569
+ const ftsExists = schema.tables.has(`${p}entries_fts`);
2570
+ const mapExists = schema.tables.has(`${p}fts_map`);
2571
+ const pendingExists = schema.tables.has(`${p}fts_pending`);
2572
+ const trig = schema.triggersPresent;
2573
+ const ftsDdl = schema.tables.get(`${p}entries_fts`) ?? null;
2574
+ const tokenizerOk = ftsDdl === null || /tokenize\s*=\s*['"]porter\s+unicode61['"]/i.test(ftsDdl);
2575
+ if (state === null && opts.entriesExisted === false) {
2576
+ await this.db.withTransactionAsync(async (tx) => {
2577
+ await tx.execAsync(fts5TablesDdl(p) + fts5PendingIndexesDdl(p) + fts5TriggersDdl(p));
2578
+ await this.metadataRepo.setMeta(FTS5_STATE_KEY, "live", tx);
2579
+ });
2580
+ return;
2581
+ }
2582
+ if (state === "detached") return this.resetAndRearm();
2583
+ if (!tokenizerOk) return this.resetAndRearm();
2584
+ const intact = trig && ftsExists && mapExists && pendingExists;
2585
+ if (intact && state === "rebuilding") return;
2586
+ if (intact && state === "live") {
2587
+ await this.ftsOnly(() => this.db.getFirstAsync(`SELECT rowid FROM ${p}entries_fts ORDER BY rowid DESC LIMIT 1`));
2588
+ if (await this.liveRowUnindexed()) return this.resetAndRearm();
3099
2589
  return;
3100
2590
  }
2591
+ if (ftsExists && mapExists && await this.keepMapConsistent()) {
2592
+ await this.db.withTransactionAsync((tx) => this.rearmInTx(tx));
2593
+ return;
2594
+ }
2595
+ return this.resetAndRearm();
2596
+ }
2597
+ /**
2598
+ * The index was lost (rev 8 m3 / rev 9 m4): the oldest or the newest live entry is neither mapped nor
2599
+ * ledgered, which `live` never allows. The oldest one keeps detection armed after post-loss writes
2600
+ * repopulate the map and the ledger (#284 review). Every-open path: each side reads at most
2601
+ * FTS5_DRAIN_CHUNK entries rows (final review M1), so an all-soft-deleted store is never fully scanned;
2602
+ * a lost index whose oldest and newest rows are all deleted goes undetected.
2603
+ */
2604
+ async liveRowUnindexed() {
2605
+ const p = this.prefix;
2606
+ const edge = (dir) => `SELECT id FROM (SELECT id FROM (SELECT id, deleted_at FROM ${p}entries ORDER BY rowid ${dir} LIMIT ${FTS5_DRAIN_CHUNK}) WHERE deleted_at IS NULL LIMIT 1)`;
2607
+ return await this.db.getFirstAsync(
2608
+ `SELECT 1 AS x FROM (${edge("ASC")} UNION ALL ${edge("DESC")}) e
2609
+ WHERE NOT EXISTS (SELECT 1 FROM ${p}fts_map m WHERE m.id = e.id)
2610
+ AND NOT EXISTS (SELECT 1 FROM ${p}fts_pending q WHERE q.id = e.id)
2611
+ LIMIT 1`
2612
+ ) !== null;
2613
+ }
2614
+ /**
2615
+ * Read-only keep-map check (spec DDL note): every surviving entries_fts
2616
+ * rowid must have a map row. Max-rowid seek, then docsize vs map counts.
2617
+ */
2618
+ async keepMapConsistent() {
2619
+ const p = this.prefix;
2620
+ const fts = await this.ftsOnly(() => this.db.getFirstAsync(`SELECT rowid FROM ${p}entries_fts ORDER BY rowid DESC LIMIT 1`));
2621
+ const map = await this.db.getFirstAsync(
2622
+ `SELECT coalesce(max(fts_rowid), 0) AS hi, count(*) AS n FROM ${p}fts_map`
2623
+ );
2624
+ if ((fts?.rowid ?? 0) > (map?.hi ?? 0)) return false;
2625
+ const doc = await this.ftsOnly(() => this.db.getFirstAsync(`SELECT count(*) AS n FROM ${p}entries_fts_docsize`));
2626
+ return (doc?.n ?? 0) === (map?.n ?? 0);
2627
+ }
2628
+ /** Keep-map re-arm: triggers, empty ledger, rebuild from 0, state rebuilding. One transaction. */
2629
+ async rearmInTx(tx) {
2630
+ const p = this.prefix;
2631
+ await tx.execAsync(fts5TablesDdl(p));
2632
+ await tx.execAsync(`DELETE FROM ${p}fts_pending;`);
2633
+ await tx.execAsync(fts5PendingIndexesDdl(p) + fts5TriggersDdl(p));
2634
+ await writeRebuildStart(tx, p, this.metadataRepo);
2635
+ await this.metadataRepo.setMeta(FTS5_STATE_KEY, "rebuilding", tx);
2636
+ }
2637
+ /** Reset path: retire entries_fts, recreate it empty, clear map + ledger, then the re-arm. One transaction. */
2638
+ async resetAndRearm() {
2639
+ const p = this.prefix;
3101
2640
  await this.db.withTransactionAsync(async (tx) => {
3102
- await tx.execAsync(`DROP TABLE IF EXISTS ${this.prefix}entries_fts;`);
3103
- await tx.execAsync(fts5TablesDdl(this.prefix));
3104
- await this.rebuildInTx(tx);
3105
- await tx.execAsync(fts5TriggersDdl(this.prefix));
2641
+ await this.retireFtsTable(tx);
2642
+ await tx.execAsync(fts5TablesDdl(p));
2643
+ await tx.execAsync(`DELETE FROM ${p}fts_map; DELETE FROM ${p}fts_pending;`);
2644
+ await this.rearmInTx(tx);
2645
+ });
2646
+ }
2647
+ /** Rename-retire entries_fts (spec #280 rev 11); DROP only when the RENAME throws. */
2648
+ async retireFtsTable(tx) {
2649
+ const p = this.prefix;
2650
+ if (!await this.metadataRepo.tableExists(`${p}entries_fts`, tx)) return;
2651
+ const n = Number(await this.metadataRepo.getMeta(FTS5_RETIRED_COUNTER_KEY, tx) ?? "0") + 1;
2652
+ const name = retiredTableName(p, n);
2653
+ try {
2654
+ await tx.execAsync(`ALTER TABLE ${p}entries_fts RENAME TO ${name}`);
2655
+ } catch (err) {
2656
+ if (isTransientOrTxFatal(err)) throw err;
2657
+ await tx.execAsync(`DROP TABLE ${p}entries_fts`);
2658
+ return;
2659
+ }
2660
+ await this.metadataRepo.setMeta(FTS5_RETIRED_TABLES_KEY, JSON.stringify([...await this.retiredList(tx), name]), tx);
2661
+ await this.metadataRepo.setMeta(FTS5_RETIRED_COUNTER_KEY, String(n), tx);
2662
+ }
2663
+ async retiredList(tx = this.db) {
2664
+ const raw = parseJsonArray(await this.metadataRepo.getMeta(FTS5_RETIRED_TABLES_KEY, tx), []);
2665
+ return raw.filter((n) => isRetiredTableName(this.prefix, n));
2666
+ }
2667
+ async drain(opts = {}) {
2668
+ try {
2669
+ await this.drainOnce(opts);
2670
+ } catch (err) {
2671
+ if (ftsCorruptionClass(err) !== "vtab") throw err;
2672
+ await this.resetAndRearm();
2673
+ if (opts.ids && opts.ids.length > 0) await this.drainIds(opts.ids, idReindexChunkSql(this.prefix));
2674
+ }
2675
+ }
2676
+ async drainOnce(opts) {
2677
+ const ids = opts.ids ?? [];
2678
+ if (ids.length > 0) await this.drainIds(ids);
2679
+ if (opts.mode === "ledger") {
2680
+ if (ids.length === 0 && opts.entityId !== void 0) await this.drainEntityChunk(opts.entityId);
2681
+ else await this.drainLedgerChunk();
2682
+ return;
2683
+ }
2684
+ await this.fullQuantum(opts.deadline ?? monotonicNow() + this.budgetMs);
2685
+ }
2686
+ /**
2687
+ * One full quantum (spec #280 Change 3): ledger first (at most half the
2688
+ * budget while `rebuilding`); then, while `rebuilding`, ≥1 window unless
2689
+ * the ledger drain overran the deadline, one sweep tx, and the flip check;
2690
+ * then reclamation. A `live` store reads only the state.
2691
+ */
2692
+ async fullQuantum(deadline) {
2693
+ const rebuilding = await this.metadataRepo.getMeta(FTS5_STATE_KEY) === "rebuilding";
2694
+ const start = monotonicNow();
2695
+ const ledgerRan = await this.drainLedgerUntil(rebuilding ? start + (deadline - start) / 2 : deadline);
2696
+ const ledgerOverran = ledgerRan && monotonicNow() >= deadline;
2697
+ if (rebuilding) {
2698
+ const progress = await readRebuildProgress(this.metadataRepo);
2699
+ if (!progress || !await this.fingerprintValid(progress.wm)) {
2700
+ await this.restartRebuild();
2701
+ } else {
2702
+ if (!ledgerOverran) {
2703
+ let step;
2704
+ do {
2705
+ step = await this.runWindow(progress);
2706
+ } while (step === "more" && monotonicNow() < deadline);
2707
+ if (step !== "stale") await this.runSweepTx(progress);
2708
+ }
2709
+ await this.tryFlip();
2710
+ }
2711
+ }
2712
+ try {
2713
+ await this.reclaimRetired(deadline);
2714
+ } catch (err) {
2715
+ if (!isSqliteError(err)) throw err;
2716
+ }
2717
+ }
2718
+ /** Bounded reclamation steps, each its own tx, until the deadline or nothing is retired. */
2719
+ async reclaimRetired(deadline) {
2720
+ while (monotonicNow() < deadline) {
2721
+ const [name] = await this.retiredList();
2722
+ if (name === void 0) return;
2723
+ try {
2724
+ if (!await this.metadataRepo.tableExists(name)) {
2725
+ await this.delistRetired(name, false);
2726
+ continue;
2727
+ }
2728
+ while (!await this.reclaimStep(name) && monotonicNow() < deadline) ;
2729
+ } catch (err) {
2730
+ if (ftsCorruptionClass(err) === null) throw err;
2731
+ try {
2732
+ await this.delistRetired(name, true);
2733
+ } catch (dropErr) {
2734
+ if (!isSqliteError(dropErr)) throw dropErr;
2735
+ await this.delistRetired(name, false);
2736
+ }
2737
+ }
2738
+ }
2739
+ }
2740
+ /** (1) delete <=500 rows, else (2) one bounded merge, else (3) DROP + delist. True once delisted. */
2741
+ async reclaimStep(name) {
2742
+ if (await this.db.getFirstAsync(`SELECT rowid FROM ${name} LIMIT 1`)) {
2743
+ await this.db.withTransactionAsync(async (tx) => {
2744
+ await tx.runAsync(`DELETE FROM ${name} WHERE rowid IN (SELECT rowid FROM ${name} ORDER BY rowid LIMIT ${FTS5_DRAIN_CHUNK})`);
2745
+ });
2746
+ return false;
2747
+ }
2748
+ let merged = false;
2749
+ await this.db.withTransactionAsync(async (tx) => {
2750
+ const before = await tx.getFirstAsync(`SELECT total_changes() AS n`);
2751
+ await tx.runAsync(`INSERT INTO ${name}(${name}, rank) VALUES('merge', ${FTS5_RECLAIM_MERGE_PAGES})`);
2752
+ const after = await tx.getFirstAsync(`SELECT total_changes() AS n`);
2753
+ merged = (after?.n ?? 0) - (before?.n ?? 0) >= 2;
2754
+ });
2755
+ if (!merged) await this.delistRetired(name, true);
2756
+ return !merged;
2757
+ }
2758
+ async delistRetired(name, drop) {
2759
+ await this.db.withTransactionAsync(async (tx) => {
2760
+ if (drop) await tx.execAsync(`DROP TABLE IF EXISTS ${name}`);
2761
+ const rest = (await this.retiredList(tx)).filter((n) => n !== name);
2762
+ await this.metadataRepo.setMeta(FTS5_RETIRED_TABLES_KEY, JSON.stringify(rest), tx);
2763
+ });
2764
+ }
2765
+ /** The id at the watermark's rowid still matches. A null id (start, or windows complete) has nothing to check. */
2766
+ async fingerprintValid(wm, tx = this.db) {
2767
+ if (wm.id === null) return true;
2768
+ const row = await tx.getFirstAsync(`SELECT id FROM ${this.prefix}entries WHERE rowid = ?`, [wm.rowid]);
2769
+ return row?.id === wm.id;
2770
+ }
2771
+ /** External change under the watermark: start over (same re-arm transaction as a trigger-loss restart). */
2772
+ async restartRebuild() {
2773
+ await this.db.withTransactionAsync((tx) => this.rearmInTx(tx));
2774
+ }
2775
+ /**
2776
+ * Progress as stored, re-read inside a window/sweep tx: false when it no
2777
+ * longer matches what this quantum assumed (final review I1). Writing a
2778
+ * watermark or cursor computed from a stale `pr` would overwrite a
2779
+ * concurrent restart (runPrune VACUUM is not on the sync chain).
2780
+ */
2781
+ async progressUnchanged(pr, tx) {
2782
+ if (await this.metadataRepo.getMeta(FTS5_STATE_KEY, tx) !== "rebuilding") return false;
2783
+ const cur = await readRebuildProgress(this.metadataRepo, tx);
2784
+ return cur !== null && cur.wm.rowid === pr.wm.rowid && cur.wm.id === pr.wm.id && cur.cursor === pr.cursor && cur.snapshot.entries === pr.snapshot.entries && cur.snapshot.map === pr.snapshot.map;
2785
+ }
2786
+ /** One window tx. 'done' once no rows remain in (wm, snapshot] (the watermark is then pinned at the snapshot, F9); 'stale' when progress moved under the quantum. */
2787
+ async runWindow(pr) {
2788
+ if (pr.wm.rowid >= pr.snapshot.entries) return "done";
2789
+ const p = this.prefix;
2790
+ let next = pr.wm;
2791
+ let more = false;
2792
+ let stale = false;
2793
+ await this.db.withTransactionAsync(async (tx) => {
2794
+ if (!await this.progressUnchanged(pr, tx)) {
2795
+ stale = true;
2796
+ return;
2797
+ }
2798
+ const row = await tx.getFirstAsync(windowHiSql(p), [pr.wm.rowid, pr.snapshot.entries]);
2799
+ if (row?.hi == null) {
2800
+ next = { rowid: pr.snapshot.entries, id: null };
2801
+ } else {
2802
+ for (const sql of windowChunkSql(p)) await tx.runAsync(sql, [pr.wm.rowid, row.hi]);
2803
+ const fp = await tx.getFirstAsync(`SELECT id FROM ${p}entries WHERE rowid = ?`, [row.hi]);
2804
+ next = { rowid: row.hi, id: fp.id };
2805
+ more = true;
2806
+ }
2807
+ await this.metadataRepo.setMeta(FTS5_REBUILD_WM_KEY, JSON.stringify(next), tx);
2808
+ });
2809
+ if (stale) return "stale";
2810
+ pr.wm = next;
2811
+ return more ? "more" : "done";
2812
+ }
2813
+ /** At most one bounded sweep tx; the cursor advances in the same tx. */
2814
+ async runSweepTx(pr) {
2815
+ if (pr.cursor >= pr.snapshot.map) return;
2816
+ const p = this.prefix;
2817
+ let next = pr.cursor;
2818
+ await this.db.withTransactionAsync(async (tx) => {
2819
+ if (!await this.progressUnchanged(pr, tx)) return;
2820
+ const row = await tx.getFirstAsync(sweepHiSql(p), [pr.cursor, pr.snapshot.map]);
2821
+ const hi = row?.hi ?? pr.snapshot.map;
2822
+ for (const sql of sweepChunkSql(p)) await tx.runAsync(sql, [pr.cursor, hi]);
2823
+ await this.metadataRepo.setMeta(FTS5_SWEEP_CURSOR_KEY, String(hi), tx);
2824
+ next = hi;
2825
+ });
2826
+ pr.cursor = next;
2827
+ }
2828
+ /** Completion, re-checked inside the flipping tx; the residual ledger (≤ one chunk) drains in it. */
2829
+ async tryFlip() {
2830
+ const p = this.prefix;
2831
+ await this.db.withTransactionAsync(async (tx) => {
2832
+ if (await this.metadataRepo.getMeta(FTS5_STATE_KEY, tx) !== "rebuilding") return;
2833
+ const pr = await readRebuildProgress(this.metadataRepo, tx);
2834
+ if (!pr || pr.wm.rowid < pr.snapshot.entries || pr.cursor < pr.snapshot.map) return;
2835
+ if (!await this.fingerprintValid(pr.wm, tx)) return;
2836
+ const residual = await tx.getFirstAsync(
2837
+ `SELECT count(*) AS n FROM (SELECT 1 FROM ${p}fts_pending LIMIT ${FTS5_DRAIN_CHUNK + 1})`
2838
+ );
2839
+ if ((residual?.n ?? 0) > FTS5_DRAIN_CHUNK) return;
2840
+ const hi = await tx.getFirstAsync(drainHiSql(p));
2841
+ if (hi?.hi != null) for (const sql of drainChunkSql(p)) await tx.runAsync(sql, [hi.hi]);
2842
+ await this.metadataRepo.deleteMeta(FTS5_REBUILD_WM_KEY, tx);
2843
+ await this.metadataRepo.deleteMeta(FTS5_REBUILD_SNAPSHOT_KEY, tx);
2844
+ await this.metadataRepo.deleteMeta(FTS5_SWEEP_CURSOR_KEY, tx);
3106
2845
  await this.metadataRepo.setMeta(FTS5_STATE_KEY, "live", tx);
3107
2846
  });
3108
2847
  }
3109
- async rebuildFromSource() {
3110
- await this.db.withTransactionAsync((tx) => this.rebuildInTx(tx));
2848
+ /** Seq-ordered ledger chunks until empty or `stopAt`. True when a chunk ran. */
2849
+ async drainLedgerUntil(stopAt) {
2850
+ let ran = false;
2851
+ while (await this.drainLedgerChunk()) {
2852
+ ran = true;
2853
+ if (monotonicNow() >= stopAt) break;
2854
+ }
2855
+ return ran;
2856
+ }
2857
+ /** The caller's own ids, to completion, ≤ FTS5_DRAIN_CHUNK per transaction. */
2858
+ async drainIds(ids, chunk = idDrainChunkSql(this.prefix)) {
2859
+ const unique = [...new Set(ids)];
2860
+ for (let i = 0; i < unique.length; i += FTS5_DRAIN_CHUNK) {
2861
+ const json = JSON.stringify(unique.slice(i, i + FTS5_DRAIN_CHUNK));
2862
+ await this.db.withTransactionAsync(async (tx) => {
2863
+ for (const sql of chunk) await tx.runAsync(sql, [json]);
2864
+ });
2865
+ }
3111
2866
  }
3112
- async drain() {
2867
+ /** One seq-ordered chunk. False when the ledger was empty. */
2868
+ async drainLedgerChunk() {
3113
2869
  const hiSql = drainHiSql(this.prefix);
3114
2870
  const chunk = drainChunkSql(this.prefix);
3115
- for (; ; ) {
3116
- const done = await this.db.withTransactionAsync(async (tx) => {
3117
- const row = await tx.getFirstAsync(hiSql);
3118
- if (row?.hi == null) return true;
3119
- for (const sql of chunk) await tx.runAsync(sql, [row.hi]);
3120
- return false;
3121
- });
3122
- if (done) return;
3123
- }
2871
+ let ran = false;
2872
+ await this.db.withTransactionAsync(async (tx) => {
2873
+ const row = await tx.getFirstAsync(hiSql);
2874
+ if (row?.hi == null) return;
2875
+ for (const sql of chunk) await tx.runAsync(sql, [row.hi]);
2876
+ ran = true;
2877
+ });
2878
+ return ran;
2879
+ }
2880
+ /** One entity-targeted chunk (ORDER BY seq, ≤ FTS5_DRAIN_CHUNK). */
2881
+ async drainEntityChunk(entityId) {
2882
+ const hiSql = entityDrainHiSql(this.prefix);
2883
+ const chunk = entityDrainChunkSql(this.prefix);
2884
+ await this.db.withTransactionAsync(async (tx) => {
2885
+ const row = await tx.getFirstAsync(hiSql, [entityId]);
2886
+ if (row?.hi == null) return;
2887
+ for (const sql of chunk) await tx.runAsync(sql, [entityId, row.hi]);
2888
+ });
3124
2889
  }
3125
2890
  async replace(entityId, ids, documents) {
3126
2891
  assertEntity(entityId, documents);
@@ -3199,20 +2964,57 @@ var Fts5IndexStrategy = class {
3199
2964
  );
3200
2965
  }
3201
2966
  }
3202
- async rebuildInTx(tx) {
2967
+ /** The init probes' schema facts in one sqlite_master read (#284 review): FTS tables with their DDL, and the triggers. */
2968
+ async readSchema() {
3203
2969
  const p = this.prefix;
3204
- await tx.execAsync(`DELETE FROM ${p}entries_fts; DELETE FROM ${p}fts_map; DELETE FROM ${p}fts_pending;`);
3205
- for (const sql of rebuildSql(p)) await tx.runAsync(sql);
3206
- }
3207
- async triggersPresent() {
3208
- const names = fts5TriggerNames(this.prefix);
2970
+ const triggers = fts5TriggerNames(p);
2971
+ const names = [`${p}entries_fts`, `${p}fts_map`, `${p}fts_pending`, ...triggers];
3209
2972
  const rows = await this.db.getAllAsync(
3210
- `SELECT name FROM sqlite_master WHERE type = 'trigger' AND name IN (?, ?, ?)`,
3211
- [...names]
2973
+ `SELECT type, name, sql FROM sqlite_master WHERE name IN (${names.map(() => "?").join(", ")})`,
2974
+ names
3212
2975
  );
3213
- return rows.length === names.length;
2976
+ const tables = new Map(rows.filter((r) => r.type === "table").map((r) => [r.name, r.sql]));
2977
+ const present = new Set(rows.filter((r) => r.type === "trigger").map((r) => r.name));
2978
+ return { tables, triggersPresent: triggers.every((n) => present.has(n)) };
3214
2979
  }
3215
2980
  };
2981
+ var FtsOnlyStatementError = class extends Error {
2982
+ constructor(cause) {
2983
+ super("FTS-only statement failed");
2984
+ this.ftsOnlyCause = cause;
2985
+ }
2986
+ };
2987
+ function unwrapFtsOnly(err) {
2988
+ try {
2989
+ if (err instanceof FtsOnlyStatementError) return { cause: err.ftsOnlyCause, ftsOnly: true };
2990
+ } catch {
2991
+ }
2992
+ return { cause: err, ftsOnly: false };
2993
+ }
2994
+ function safeSqliteCode(err) {
2995
+ try {
2996
+ const code = extractSqliteCode(err);
2997
+ if (code !== void 0) return code;
2998
+ const lifted = err?.sqliteErrorCode;
2999
+ return typeof lifted === "string" ? lifted : void 0;
3000
+ } catch {
3001
+ return void 0;
3002
+ }
3003
+ }
3004
+ var TRANSIENT_OR_TX_FATAL = ["SQLITE_BUSY", "SQLITE_LOCKED", "SQLITE_FULL", "SQLITE_IOERR", "SQLITE_NOMEM", "SQLITE_INTERRUPT", "SQLITE_ABORT", "SQLITE_READONLY"];
3005
+ function isTransientOrTxFatal(err) {
3006
+ const code = safeSqliteCode(err);
3007
+ return code !== void 0 && TRANSIENT_OR_TX_FATAL.some((c) => code === c || code.startsWith(`${c}_`));
3008
+ }
3009
+ function ftsCorruptionClass(err) {
3010
+ const code = safeSqliteCode(err);
3011
+ if (code === "SQLITE_CORRUPT_VTAB") return "vtab";
3012
+ if (code?.startsWith("SQLITE_CORRUPT")) return "corrupt";
3013
+ return null;
3014
+ }
3015
+ function isSqliteError(err) {
3016
+ return safeSqliteCode(err) !== void 0;
3017
+ }
3216
3018
  function assertEntity(entityId, documents) {
3217
3019
  for (const d of documents) {
3218
3020
  if (d.entity_id !== entityId) {
@@ -3222,14 +3024,34 @@ function assertEntity(entityId, documents) {
3222
3024
  }
3223
3025
 
3224
3026
  // src/services/search/createIndexStrategy.ts
3027
+ var FTS5_INIT_RETRY_MS = 250;
3028
+ function isModuleAbsentError(err) {
3029
+ let cur = err;
3030
+ for (let depth = 0; depth < 3 && cur != null; depth++) {
3031
+ let msg = "";
3032
+ try {
3033
+ msg = String(cur.message ?? cur);
3034
+ } catch {
3035
+ return false;
3036
+ }
3037
+ if (/no such module: fts5/i.test(msg) || /no such function: json_valid/i.test(msg)) return true;
3038
+ try {
3039
+ cur = cur.cause;
3040
+ } catch {
3041
+ return false;
3042
+ }
3043
+ }
3044
+ return false;
3045
+ }
3225
3046
  async function probeFts5(db, prefix) {
3226
3047
  const name = `temp.${prefix}fts5_probe`;
3227
3048
  try {
3228
3049
  await db.execAsync(`CREATE VIRTUAL TABLE IF NOT EXISTS ${name} USING fts5(x)`);
3229
3050
  const row = await db.getFirstAsync(`SELECT json_valid('[]') AS ok`);
3230
3051
  return row?.ok === 1;
3231
- } catch {
3232
- return false;
3052
+ } catch (err) {
3053
+ if (isModuleAbsentError(err)) return false;
3054
+ throw err;
3233
3055
  } finally {
3234
3056
  try {
3235
3057
  await db.execAsync(`DROP TABLE IF EXISTS ${name}`);
@@ -3250,23 +3072,37 @@ async function detachFts5(db, prefix, metadataRepo) {
3250
3072
  } catch {
3251
3073
  }
3252
3074
  }
3253
- async function createIndexStrategy(db, prefix, metadataRepo, preferred) {
3254
- if (preferred !== "minisearch") {
3255
- if (await probeFts5(db, prefix)) {
3256
- const strategy = new Fts5IndexStrategy(db, prefix, metadataRepo);
3257
- try {
3258
- await strategy.init();
3259
- return strategy;
3260
- } catch (err) {
3261
- if (preferred === "fts5") throw err;
3262
- console.warn("[WikiMemory] FTS5 init failed; falling back to MiniSearch:", err);
3263
- }
3264
- } else if (preferred === "fts5") {
3075
+ async function createIndexStrategy(db, prefix, metadataRepo, preferred, opts = {}) {
3076
+ if (preferred === "minisearch") {
3077
+ await detachFts5(db, prefix, metadataRepo);
3078
+ return new MiniSearchIndexStrategy();
3079
+ }
3080
+ const attempt = async () => {
3081
+ if (!await probeFts5(db, prefix)) return "absent";
3082
+ const strategy = new Fts5IndexStrategy(db, prefix, metadataRepo, opts.setupBudgetMs);
3083
+ await strategy.init({ entriesExisted: opts.entriesExisted });
3084
+ return strategy;
3085
+ };
3086
+ const absent = async () => {
3087
+ if (preferred === "fts5") {
3265
3088
  throw new Error(`indexStrategy 'fts5' was requested, but this SQLite build lacks FTS5 or json1.`);
3266
3089
  }
3090
+ await detachFts5(db, prefix, metadataRepo);
3091
+ return new MiniSearchIndexStrategy();
3092
+ };
3093
+ for (let tries = 1; ; tries++) {
3094
+ try {
3095
+ const s = await attempt();
3096
+ return s === "absent" ? await absent() : s;
3097
+ } catch (err) {
3098
+ if (preferred === "fts5") throw err;
3099
+ if (tries === 2) {
3100
+ console.warn("[WikiMemory] FTS5 init failed after retry; keyword search unavailable until a later open or syncSearchIndex():", err);
3101
+ return new UnavailableIndexStrategy(err);
3102
+ }
3103
+ }
3104
+ await new Promise((resolve) => setTimeout(resolve, FTS5_INIT_RETRY_MS));
3267
3105
  }
3268
- await detachFts5(db, prefix, metadataRepo);
3269
- return new MiniSearchIndexStrategy();
3270
3106
  }
3271
3107
 
3272
3108
  // src/services/MaintenanceScheduler.ts
@@ -3350,8 +3186,14 @@ var MaintenanceScheduler = class {
3350
3186
  var _a;
3351
3187
  ((_a = st.report).busy ?? (_a.busy = [])).push(job);
3352
3188
  };
3189
+ const markSearchUnavailable = (st, job) => {
3190
+ var _a;
3191
+ ((_a = st.report).searchUnavailable ?? (_a.searchUnavailable = [])).push(job);
3192
+ };
3193
+ const openJobs = () => states.reduce((n, s) => n + +s.librarian + +s.heal + +s.reembed, 0);
3353
3194
  rounds: for (; ; ) {
3354
3195
  let progressed = false;
3196
+ const openAtStart = openJobs();
3355
3197
  for (const st of states) {
3356
3198
  if (!st.librarian && !st.heal && !st.reembed) continue;
3357
3199
  if (Date.now() >= deadline) {
@@ -3369,6 +3211,11 @@ var MaintenanceScheduler = class {
3369
3211
  let r;
3370
3212
  try {
3371
3213
  r = await this.maintenanceService.runLibrarianPass({ entityId, trigger: "call", meter });
3214
+ } catch (err) {
3215
+ if (!isWikiSearchUnavailableError(err)) throw err;
3216
+ st.librarian = false;
3217
+ markSearchUnavailable(st, "librarian");
3218
+ continue;
3372
3219
  } finally {
3373
3220
  this.jobManager.releaseLock("librarian", entityId);
3374
3221
  }
@@ -3412,6 +3259,11 @@ var MaintenanceScheduler = class {
3412
3259
  st.heal = false;
3413
3260
  continue;
3414
3261
  }
3262
+ if (h.scanned === 0 && h.remaining > 0) {
3263
+ st.heal = false;
3264
+ if (h.searchUnavailable) markSearchUnavailable(st, "heal");
3265
+ continue;
3266
+ }
3415
3267
  st.report.healPasses++;
3416
3268
  progressed = true;
3417
3269
  if (h.remaining === 0) {
@@ -3436,7 +3288,7 @@ var MaintenanceScheduler = class {
3436
3288
  }
3437
3289
  }
3438
3290
  }
3439
- if (!progressed) break;
3291
+ if (!progressed && openJobs() === openAtStart) break;
3440
3292
  }
3441
3293
  if (stopped.reason === "complete" && tooSmall) stopped = tooSmall;
3442
3294
  return {
@@ -3937,6 +3789,9 @@ var OkfTrustWritesRepository = class {
3937
3789
 
3938
3790
  // src/WikiMemory.ts
3939
3791
  var TABLE_PREFIX_PATTERN = /^[A-Za-z][A-Za-z0-9_]{0,30}_$/;
3792
+ var SEARCH_REINIT_COOLDOWN_MS = 5e3;
3793
+ var SEARCH_REINIT_COOLDOWN_MAX_MS = 5 * 6e4;
3794
+ var SEARCH_REINIT_COST_FACTOR = 10;
3940
3795
  function decodeDraftCursor(cursor) {
3941
3796
  const sep = typeof cursor === "string" ? cursor.indexOf(":") : -1;
3942
3797
  const createdAt = sep > 0 ? Number(cursor.slice(0, sep)) : Number.NaN;
@@ -3951,6 +3806,21 @@ var WikiMemory = class {
3951
3806
  constructor(db, options) {
3952
3807
  /** Emits `__testAccess` console warning at most once per instance when NODE_ENV ≠ "test". */
3953
3808
  __privateAdd(this, _testAccessNonTestEnvWarned, false);
3809
+ /**
3810
+ * upsertGraph node and superseded-fact ids per entity, awaiting syncSearchIndex(), each with the
3811
+ * generation of its latest stash. Ids leave the stash only once a drain
3812
+ * succeeded, so a failed drain retries them next call.
3813
+ */
3814
+ this.upsertIdStash = /* @__PURE__ */ new Map();
3815
+ /** Total ids across upsertIdStash, kept in step so the cap check is O(1). */
3816
+ this.upsertIdStashSize = 0;
3817
+ /** Bumped per stashed id; a re-stash during an in-flight drain outlives that drain's drop. */
3818
+ this.upsertIdStashGen = 0;
3819
+ this.reinitSearch = null;
3820
+ /** Monotonic time before which a no-arg syncSearchIndex() skips the re-init (0 = no cooldown). */
3821
+ this.reinitRetryAt = 0;
3822
+ /** Consecutive failed re-inits; drives the exponential backoff. */
3823
+ this.reinitFailures = 0;
3954
3824
  this.db = withSerializedTransactions(db);
3955
3825
  this.options = options;
3956
3826
  this.prefix = options.config?.tablePrefix ?? "llm_wiki_";
@@ -4082,23 +3952,81 @@ var WikiMemory = class {
4082
3952
  jobManager: this.jobManager
4083
3953
  };
4084
3954
  }
4085
- async setup() {
3955
+ stashUpsertIds(entityId, ids) {
3956
+ if (!this.searchService.hasLedger()) return;
3957
+ const cap = this.constructor.UPSERT_ID_STASH_CAP;
3958
+ const stash = this.upsertIdStash.get(entityId) ?? /* @__PURE__ */ new Map();
3959
+ for (const id of ids) {
3960
+ if (!stash.has(id)) {
3961
+ if (this.upsertIdStashSize >= cap) continue;
3962
+ this.upsertIdStashSize++;
3963
+ }
3964
+ stash.set(id, ++this.upsertIdStashGen);
3965
+ }
3966
+ if (stash.size > 0) this.upsertIdStash.set(entityId, stash);
3967
+ }
3968
+ /** Snapshot of the stash (one entity, or all) with generations, without removing anything. */
3969
+ peekStashedIds(entityId) {
3970
+ const snapshot = /* @__PURE__ */ new Map();
3971
+ for (const [id, stash] of this.upsertIdStash) {
3972
+ if (entityId === void 0 || id === entityId) snapshot.set(id, new Map(stash));
3973
+ }
3974
+ return snapshot;
3975
+ }
3976
+ /**
3977
+ * Removes a drained snapshot. An id re-stashed after the peek has a newer
3978
+ * generation and stays: the drain may have run before that write committed.
3979
+ */
3980
+ dropStashedIds(snapshot) {
3981
+ for (const [entityId, drained] of snapshot) {
3982
+ const stash = this.upsertIdStash.get(entityId);
3983
+ if (!stash) continue;
3984
+ for (const [id, gen] of drained) {
3985
+ if (stash.get(id) === gen && stash.delete(id)) this.upsertIdStashSize--;
3986
+ }
3987
+ if (stash.size === 0) this.upsertIdStash.delete(entityId);
3988
+ }
3989
+ }
3990
+ /** `config.setupBudgetMs` or the default, validated: a positive finite number. */
3991
+ setupBudgetMs() {
3992
+ const budget = this.options.config?.setupBudgetMs ?? DEFAULT_SETUP_BUDGET_MS;
3993
+ if (typeof budget !== "number" || !Number.isFinite(budget) || budget <= 0) {
3994
+ throw new TypeError(`setupBudgetMs must be a positive finite number; got ${String(budget)}`);
3995
+ }
3996
+ return budget;
3997
+ }
3998
+ /**
3999
+ * Opens the store: migration gate, DDL, schema migrations, then the stamped
4000
+ * source_ref normalize. Await it before any other call on this instance — the
4001
+ * normalize commits in chunks, so a sourceRef-keyed call (forget, upsertGraph,
4002
+ * hasChanged, ingest) issued before setup() resolves can see partially
4003
+ * normalized refs.
4004
+ *
4005
+ * Pass `{ migrations: 'refuse' }` from inside a writer lease: pending one-time
4006
+ * work then throws WikiMigrationRequiredError before any mutation, and the
4007
+ * host runs a plain `setup()` outside the lease.
4008
+ *
4009
+ * Keyword-index catch-up work is bounded by `config.setupBudgetMs`; anything
4010
+ * left converges on later opens and no-arg `syncSearchIndex()` calls. Not
4011
+ * bounded: an index reset (tokenizer change, re-attach after detach, a lost
4012
+ * or inconsistent index), which runs as one unchunked transaction.
4013
+ */
4014
+ async setup(opts) {
4015
+ const setupBudgetMs = this.setupBudgetMs();
4016
+ const openDeadline = monotonicNow() + setupBudgetMs;
4086
4017
  const entriesExistedBeforeSetup = await this.metadataRepo.tableExists(`${this.prefix}entries`);
4087
- await setupDatabase(this.db, this.prefix);
4018
+ const migrationPolicy = opts?.migrations ?? this.options.config?.migrations ?? "allow";
4019
+ await setupDatabaseWithGate(this.db, this.prefix, this.metadataRepo, migrationPolicy);
4088
4020
  let currentVersion;
4089
4021
  if (!entriesExistedBeforeSetup) {
4090
- await this.metadataRepo.setMeta("schema_version", String(CURRENT_SCHEMA_VERSION), this.db);
4022
+ await this.db.withTransactionAsync(async (tx) => {
4023
+ await createLegacySourceTypeIndex(tx, this.prefix);
4024
+ await writeNormalizeStamp(this.metadataRepo, tx);
4025
+ await this.metadataRepo.setMeta("schema_version", String(CURRENT_SCHEMA_VERSION), tx);
4026
+ });
4091
4027
  currentVersion = CURRENT_SCHEMA_VERSION;
4092
4028
  } else {
4093
- const schemaVersionValue = await this.metadataRepo.getMeta("schema_version");
4094
- if (schemaVersionValue) {
4095
- currentVersion = parseInt(schemaVersionValue, 10);
4096
- if (!Number.isFinite(currentVersion)) currentVersion = 0;
4097
- } else {
4098
- const ftsDdl = await this.metadataRepo.getTableDdl(`${this.prefix}entries_fts`);
4099
- const hasPorter = /tokenize\s*=\s*['"]porter\s+unicode61['"]/i.test(ftsDdl ?? "");
4100
- currentVersion = hasPorter ? 1 : 0;
4101
- }
4029
+ currentVersion = await detectSchemaVersion(this.metadataRepo, this.prefix);
4102
4030
  }
4103
4031
  for (const migration of MIGRATIONS) {
4104
4032
  if (migration.version > currentVersion) {
@@ -4112,28 +4040,19 @@ var WikiMemory = class {
4112
4040
  if (!schemaVersionCheck) {
4113
4041
  await this.metadataRepo.setMeta("schema_version", String(currentVersion), this.db);
4114
4042
  }
4115
- }
4116
- if (entriesExistedBeforeSetup) {
4117
4043
  await this.importExportService.assertNoLegacySourceTypes();
4044
+ await runSourceRefNormalize(this.db, this.prefix, this.metadataRepo);
4118
4045
  }
4119
- const rows = await this.entryRepo.findRowsForSourceRefMigration();
4120
- await this.db.withTransactionAsync(async (tx) => {
4121
- for (const row of rows) {
4122
- const normalized = normalizeSourceRef(row.source_ref);
4123
- if (normalized !== row.source_ref) {
4124
- await this.entryRepo.updateSourceRefByRowid(row.rowid, normalized, tx);
4125
- }
4126
- }
4127
- });
4128
4046
  this.searchService.setIndexStrategy(
4129
4047
  await createIndexStrategy(
4130
4048
  this.db,
4131
4049
  this.prefix,
4132
4050
  this.metadataRepo,
4133
- this.options.config?.indexStrategy ?? "auto"
4051
+ this.options.config?.indexStrategy ?? "auto",
4052
+ { setupBudgetMs, entriesExisted: entriesExistedBeforeSetup }
4134
4053
  )
4135
4054
  );
4136
- await this.searchService.sync();
4055
+ await this.searchService.sync(void 0, { deadline: openDeadline });
4137
4056
  }
4138
4057
  async hasChanged(entityId, sourceRefOrEntries, sourceHashArg) {
4139
4058
  if (typeof sourceRefOrEntries === "string") {
@@ -4384,11 +4303,18 @@ var WikiMemory = class {
4384
4303
  * commit, so without this the nodes stay out of keyword, hybrid and
4385
4304
  * fallback retrieval until the entity's next core write.
4386
4305
  *
4387
- * With `entityId`, rebuilds that entity if it is stale or has never been
4388
- * indexed; otherwise it is a no-op that still waits for rebuilds already
4389
- * queued. Without an argument, rebuilds every entity currently marked
4390
- * stale; when none is, it resolves immediately and does not wait for rebuilds
4391
- * already queued.
4306
+ * With `entityId`: drains that entity's pending `upsertGraph` ids (its new
4307
+ * nodes and the facts they superseded) first, to completion, then at most
4308
+ * one more ledger chunk — so its new nodes are searchable, and the
4309
+ * superseded facts gone, when the call resolves, under any backlog. Without an
4310
+ * argument: drains every stashed `upsertGraph` id across entities first,
4311
+ * then runs one keyword-index quantum bounded by `config.setupBudgetMs`
4312
+ * (backlog drain, then rebuild work while the index is rebuilding). Hosts
4313
+ * with a long-lived process should call the no-arg form on their own
4314
+ * schedule: that is what converges a rebuilding index. While keyword search
4315
+ * is unavailable it first retries the index init; after a failed attempt it
4316
+ * waits at least 5 s, doubling per consecutive failure up to 5 min (longer
4317
+ * when an attempt is itself slow), before retrying again.
4392
4318
  *
4393
4319
  * Call only after the host's transaction commits: called inside it, the
4394
4320
  * rebuild may index rows that later roll back, or miss them and clear the
@@ -4405,9 +4331,53 @@ var WikiMemory = class {
4405
4331
  * @throws {TypeError} when `entityId` is given but is not a non-empty string.
4406
4332
  */
4407
4333
  async syncSearchIndex(entityId) {
4408
- if (entityId === void 0) return this.searchService.syncStale();
4334
+ if (entityId === void 0) {
4335
+ if (!this.searchService.isKeywordAvailable() && monotonicNow() >= this.reinitRetryAt) {
4336
+ this.reinitSearch ?? (this.reinitSearch = (async () => {
4337
+ const started = monotonicNow();
4338
+ try {
4339
+ const strategy = await createIndexStrategy(
4340
+ this.db,
4341
+ this.prefix,
4342
+ this.metadataRepo,
4343
+ this.options.config?.indexStrategy ?? "auto",
4344
+ { setupBudgetMs: this.setupBudgetMs() }
4345
+ );
4346
+ this.searchService.setIndexStrategy(strategy);
4347
+ if (this.searchService.isKeywordAvailable() && !this.searchService.hasLedger()) await this.searchService.sync();
4348
+ } finally {
4349
+ if (this.searchService.isKeywordAvailable()) {
4350
+ this.reinitFailures = 0;
4351
+ this.reinitRetryAt = 0;
4352
+ } else {
4353
+ const now = monotonicNow();
4354
+ const backoff = Math.min(SEARCH_REINIT_COOLDOWN_MS * 2 ** this.reinitFailures, SEARCH_REINIT_COOLDOWN_MAX_MS);
4355
+ this.reinitFailures++;
4356
+ this.reinitRetryAt = now + Math.max(backoff, SEARCH_REINIT_COST_FACTOR * (now - started));
4357
+ }
4358
+ this.reinitSearch = null;
4359
+ }
4360
+ })());
4361
+ await this.reinitSearch;
4362
+ }
4363
+ const stashed2 = this.peekStashedIds();
4364
+ const ids = [...stashed2.values()].flatMap((m) => [...m.keys()]);
4365
+ if (await this.syncDrained(this.searchService.syncStale({ ids }))) this.dropStashedIds(stashed2);
4366
+ return;
4367
+ }
4409
4368
  assertEntityId(entityId);
4410
- return this.searchService.syncEntries(entityId, []);
4369
+ const stashed = this.peekStashedIds(entityId);
4370
+ if (await this.syncDrained(this.searchService.syncEntries(entityId, stashed.get(entityId)?.keys() ?? []))) {
4371
+ this.dropStashedIds(stashed);
4372
+ }
4373
+ }
4374
+ /**
4375
+ * True when a sync applied the stashed ids. While keyword search is
4376
+ * unavailable the stand-in strategy's drain is a no-op that reports success,
4377
+ * so the ids stay stashed for the first sync after a re-init (#284 review).
4378
+ */
4379
+ async syncDrained(sync) {
4380
+ return await sync && this.searchService.isKeywordAvailable();
4411
4381
  }
4412
4382
  async exportDump(entityIds) {
4413
4383
  return this.importExportService.exportDump(entityIds);
@@ -4458,7 +4428,10 @@ var WikiMemory = class {
4458
4428
  * post-commit work (search sync, embedding, cache eviction). After the
4459
4429
  * transaction commits, call `syncSearchIndex(entityId)` — or
4460
4430
  * `syncSearchIndex()` after writing several entities in one transaction —
4461
- * to make the new nodes keyword-searchable. Embeddings still come from the maintenance
4431
+ * to make the new nodes keyword-searchable: both forms drain this call's node ids
4432
+ * first, to completion, so the nodes are searchable under any backlog (up to
4433
+ * `WikiMemory.UPSERT_ID_STASH_CAP` pending ids; beyond that they converge with
4434
+ * the backlog). Embeddings still come from the maintenance
4462
4435
  * sweep (`runLibrarian` / `runHeal` / `runOntologyBackfill` / `runPrune`,
4463
4436
  * scoped via `listEntityIds`).
4464
4437
  *
@@ -4508,12 +4481,13 @@ var WikiMemory = class {
4508
4481
  });
4509
4482
  }
4510
4483
  const diagBuffer = new DiagnosticBuffer();
4511
- const result = await this.ingestionService.upsertGraphCore(
4484
+ const { supersededFactIds, ...result } = await this.ingestionService.upsertGraphCore(
4512
4485
  entityId,
4513
4486
  { sourceRef, sourceHash, nodes: params.nodes, edges: params.edges },
4514
4487
  adapter,
4515
4488
  { diag: { buffer: diagBuffer, operation: "upsertGraph" } }
4516
4489
  );
4490
+ this.stashUpsertIds(entityId, [...supersededFactIds, ...params.nodes.map((n) => n.id)]);
4517
4491
  this.searchService.markStale(entityId);
4518
4492
  diagBuffer.flush(this.options);
4519
4493
  return result;
@@ -4696,6 +4670,8 @@ var WikiMemory = class {
4696
4670
  }
4697
4671
  };
4698
4672
  _testAccessNonTestEnvWarned = new WeakMap();
4673
+ /** Max upsertGraph node ids held for their post-commit sync (spec #280 id threading). */
4674
+ WikiMemory.UPSERT_ID_STASH_CAP = 1e4;
4699
4675
  function assertEntityId(entityId) {
4700
4676
  if (typeof entityId !== "string" || entityId.length === 0) {
4701
4677
  throw new TypeError("Invalid entityId: must be a non-empty string.");
@@ -5621,6 +5597,6 @@ function createWiki(db, options) {
5621
5597
  return new WikiMemory(db, options);
5622
5598
  }
5623
5599
 
5624
- export { DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, WikiMemory, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseOkfBundle, validateLibrarianPromptTemplate };
5600
+ export { DEFAULT_LIBRARIAN_SYNTHESIS_PROMPT, DEFAULT_SETUP_BUDGET_MS, WikiMemory, createWiki, formatContext, formatGraphContext, formatMemoryDump, formatOkfBundle, hydrateLibrarianPrompt, mapLibrarianOptionsToReadOptions, parseOkfBundle, validateLibrarianPromptTemplate };
5625
5601
  //# sourceMappingURL=index.mjs.map
5626
5602
  //# sourceMappingURL=index.mjs.map