@equationalapplications/core-llm-wiki 7.9.1 → 7.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -0
- package/dist/{chunk-7FRLQ7ZO.mjs → chunk-LOVBNRCV.mjs} +33 -2
- package/dist/chunk-LOVBNRCV.mjs.map +1 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +318 -0
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +289 -2
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-Bf6etdUy.d.mts → testing-SxCsMt1q.d.mts} +45 -0
- package/dist/{testing-Bf6etdUy.d.ts → testing-SxCsMt1q.d.ts} +45 -0
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +31 -0
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +4 -2
- package/dist/chunk-7FRLQ7ZO.mjs.map +0 -1
package/dist/index.d.mts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-
|
|
2
|
-
export { C as ChunkFailure, d as ClassifierAnswer, e as ClassifierQuestion, f as ClassifyRequest, g as ClassifyResponse, D as DegradedRecord, h as DraftPage, E as EmbedFactResult, i as EmbedFailureKind, j as EmbeddingMarkerKind, k as EntityMaintenanceReport, l as EntityStatus, m as ExtractedFact, n as ExtractedFactEdge, o as ExtractedFactWithOntology, p as ExtractedTask, q as GraphTraversalOptions, r as GroundingConfig, H as HEAL_BATCH_SIZE, s as HEAL_RECHECK_MS, t as HOOK_TIMEOUT_MARKER, u as HealResult, I as IngestDocumentResult, L as LLMProvider, v as LibrarianConfig, w as LibrarianGateConfig, x as MaintenanceJob, y as MaintenanceStopReason, O as ONTOLOGY_BACKFILL_BATCH_SIZE, z as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, A as ONTOLOGY_BACKFILL_RECHECK_MS, B as OntologyBackfillResult, J as OntologyConfig, K as OntologyEdgeType, N as OntologyManifest, P as OntologyMode, Q as OntologyNodeType, T as OntologyPromptContext, U as OntologyUpdates, V as PendingMaintenance, X as PendingSourceStatus, Y as PromptOverrides, Z as PromptService, _ as PrunePartialFailureError, $ as ReembedResult, a0 as RunPendingMaintenanceOptions, a1 as RunPendingMaintenanceResult, a2 as SupersedeReplacement, a3 as VectorRanker, a4 as VectorRankerFallback, a5 as VectorRankerRankArgs, a6 as VectorRankerSemanticResult, a7 as WikiBusyError, a8 as WikiBusyOperation, a9 as WikiCheckpoint, aa as WikiConfig, ab as WikiDiagnostic, ac as WikiDiagnosticCode, ad as WikiDiagnosticDetail, ae as WikiDiagnosticOperation, af as WikiDiagnosticSeverity, ag as WikiDiagnosticTrigger, ah as WikiDraftNotFound, ai as WikiDuplicateHashError, aj as WikiEdge, ak as WikiEvent, al as WikiFact, am as WikiGraphNodeOwnershipConflict, an as WikiIngestEmptyError, ao as WikiInstructions, ap as WikiInvalidReadOptions, aq as WikiLintReport, ar as WikiMemoryTestAccess, as as WikiOutboxEvent, at as WikiParseError, au as WikiSourceRefHashCollision, av as WikiStrictOntologyViolation, aw as WikiSupersedeError, ax as WikiSupersedeReason, ay as WikiTask, az as WikiTransactionError, aA as validateManifest } from './testing-
|
|
1
|
+
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-SxCsMt1q.mjs';
|
|
2
|
+
export { C as ChunkFailure, d as ClassifierAnswer, e as ClassifierQuestion, f as ClassifyRequest, g as ClassifyResponse, D as DegradedRecord, h as DraftPage, E as EmbedFactResult, i as EmbedFailureKind, j as EmbeddingMarkerKind, k as EntityMaintenanceReport, l as EntityStatus, m as ExtractedFact, n as ExtractedFactEdge, o as ExtractedFactWithOntology, p as ExtractedTask, q as GraphTraversalOptions, r as GroundingConfig, H as HEAL_BATCH_SIZE, s as HEAL_RECHECK_MS, t as HOOK_TIMEOUT_MARKER, u as HealResult, I as IngestDocumentResult, L as LLMProvider, v as LibrarianConfig, w as LibrarianGateConfig, x as MaintenanceJob, y as MaintenanceStopReason, O as ONTOLOGY_BACKFILL_BATCH_SIZE, z as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, A as ONTOLOGY_BACKFILL_RECHECK_MS, B as OntologyBackfillResult, J as OntologyConfig, K as OntologyEdgeType, N as OntologyManifest, P as OntologyMode, Q as OntologyNodeType, T as OntologyPromptContext, U as OntologyUpdates, V as PendingMaintenance, X as PendingSourceStatus, Y as PromptOverrides, Z as PromptService, _ as PrunePartialFailureError, $ as ReembedResult, a0 as RunPendingMaintenanceOptions, a1 as RunPendingMaintenanceResult, a2 as SupersedeReplacement, a3 as VectorRanker, a4 as VectorRankerFallback, a5 as VectorRankerRankArgs, a6 as VectorRankerSemanticResult, a7 as WikiBusyError, a8 as WikiBusyOperation, a9 as WikiCheckpoint, aa as WikiConfig, ab as WikiDiagnostic, ac as WikiDiagnosticCode, ad as WikiDiagnosticDetail, ae as WikiDiagnosticOperation, af as WikiDiagnosticSeverity, ag as WikiDiagnosticTrigger, ah as WikiDraftNotFound, ai as WikiDuplicateHashError, aj as WikiEdge, ak as WikiEvent, al as WikiFact, am as WikiGraphNodeOwnershipConflict, an as WikiIngestEmptyError, ao as WikiInstructions, ap as WikiInvalidReadOptions, aq as WikiLintReport, ar as WikiMemoryTestAccess, as as WikiOutboxEvent, at as WikiParseError, au as WikiSourceRefHashCollision, av as WikiStrictOntologyViolation, aw as WikiSupersedeError, ax as WikiSupersedeReason, ay as WikiTask, az as WikiTransactionError, aA as validateManifest } from './testing-SxCsMt1q.mjs';
|
|
3
3
|
import { OkfFile } from '@equationalapplications/core-okf';
|
|
4
4
|
|
|
5
5
|
/**
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-
|
|
2
|
-
export { C as ChunkFailure, d as ClassifierAnswer, e as ClassifierQuestion, f as ClassifyRequest, g as ClassifyResponse, D as DegradedRecord, h as DraftPage, E as EmbedFactResult, i as EmbedFailureKind, j as EmbeddingMarkerKind, k as EntityMaintenanceReport, l as EntityStatus, m as ExtractedFact, n as ExtractedFactEdge, o as ExtractedFactWithOntology, p as ExtractedTask, q as GraphTraversalOptions, r as GroundingConfig, H as HEAL_BATCH_SIZE, s as HEAL_RECHECK_MS, t as HOOK_TIMEOUT_MARKER, u as HealResult, I as IngestDocumentResult, L as LLMProvider, v as LibrarianConfig, w as LibrarianGateConfig, x as MaintenanceJob, y as MaintenanceStopReason, O as ONTOLOGY_BACKFILL_BATCH_SIZE, z as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, A as ONTOLOGY_BACKFILL_RECHECK_MS, B as OntologyBackfillResult, J as OntologyConfig, K as OntologyEdgeType, N as OntologyManifest, P as OntologyMode, Q as OntologyNodeType, T as OntologyPromptContext, U as OntologyUpdates, V as PendingMaintenance, X as PendingSourceStatus, Y as PromptOverrides, Z as PromptService, _ as PrunePartialFailureError, $ as ReembedResult, a0 as RunPendingMaintenanceOptions, a1 as RunPendingMaintenanceResult, a2 as SupersedeReplacement, a3 as VectorRanker, a4 as VectorRankerFallback, a5 as VectorRankerRankArgs, a6 as VectorRankerSemanticResult, a7 as WikiBusyError, a8 as WikiBusyOperation, a9 as WikiCheckpoint, aa as WikiConfig, ab as WikiDiagnostic, ac as WikiDiagnosticCode, ad as WikiDiagnosticDetail, ae as WikiDiagnosticOperation, af as WikiDiagnosticSeverity, ag as WikiDiagnosticTrigger, ah as WikiDraftNotFound, ai as WikiDuplicateHashError, aj as WikiEdge, ak as WikiEvent, al as WikiFact, am as WikiGraphNodeOwnershipConflict, an as WikiIngestEmptyError, ao as WikiInstructions, ap as WikiInvalidReadOptions, aq as WikiLintReport, ar as WikiMemoryTestAccess, as as WikiOutboxEvent, at as WikiParseError, au as WikiSourceRefHashCollision, av as WikiStrictOntologyViolation, aw as WikiSupersedeError, ax as WikiSupersedeReason, ay as WikiTask, az as WikiTransactionError, aA as validateManifest } from './testing-
|
|
1
|
+
import { M as MemoryBundle, F as FormatContextOptions, G as GraphNeighborhood, a as MemoryDump, b as FormattedMemoryDump, R as ReadOptions, S as SQLiteAdapter, W as WikiOptions, c as WikiMemory } from './testing-SxCsMt1q.js';
|
|
2
|
+
export { C as ChunkFailure, d as ClassifierAnswer, e as ClassifierQuestion, f as ClassifyRequest, g as ClassifyResponse, D as DegradedRecord, h as DraftPage, E as EmbedFactResult, i as EmbedFailureKind, j as EmbeddingMarkerKind, k as EntityMaintenanceReport, l as EntityStatus, m as ExtractedFact, n as ExtractedFactEdge, o as ExtractedFactWithOntology, p as ExtractedTask, q as GraphTraversalOptions, r as GroundingConfig, H as HEAL_BATCH_SIZE, s as HEAL_RECHECK_MS, t as HOOK_TIMEOUT_MARKER, u as HealResult, I as IngestDocumentResult, L as LLMProvider, v as LibrarianConfig, w as LibrarianGateConfig, x as MaintenanceJob, y as MaintenanceStopReason, O as ONTOLOGY_BACKFILL_BATCH_SIZE, z as ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, A as ONTOLOGY_BACKFILL_RECHECK_MS, B as OntologyBackfillResult, J as OntologyConfig, K as OntologyEdgeType, N as OntologyManifest, P as OntologyMode, Q as OntologyNodeType, T as OntologyPromptContext, U as OntologyUpdates, V as PendingMaintenance, X as PendingSourceStatus, Y as PromptOverrides, Z as PromptService, _ as PrunePartialFailureError, $ as ReembedResult, a0 as RunPendingMaintenanceOptions, a1 as RunPendingMaintenanceResult, a2 as SupersedeReplacement, a3 as VectorRanker, a4 as VectorRankerFallback, a5 as VectorRankerRankArgs, a6 as VectorRankerSemanticResult, a7 as WikiBusyError, a8 as WikiBusyOperation, a9 as WikiCheckpoint, aa as WikiConfig, ab as WikiDiagnostic, ac as WikiDiagnosticCode, ad as WikiDiagnosticDetail, ae as WikiDiagnosticOperation, af as WikiDiagnosticSeverity, ag as WikiDiagnosticTrigger, ah as WikiDraftNotFound, ai as WikiDuplicateHashError, aj as WikiEdge, ak as WikiEvent, al as WikiFact, am as WikiGraphNodeOwnershipConflict, an as WikiIngestEmptyError, ao as WikiInstructions, ap as WikiInvalidReadOptions, aq as WikiLintReport, ar as WikiMemoryTestAccess, as as WikiOutboxEvent, at as WikiParseError, au as WikiSourceRefHashCollision, av as WikiStrictOntologyViolation, aw as WikiSupersedeError, ax as WikiSupersedeReason, ay as WikiTask, az as WikiTransactionError, aA as validateManifest } from './testing-SxCsMt1q.js';
|
|
3
3
|
import { OkfFile } from '@equationalapplications/core-okf';
|
|
4
4
|
|
|
5
5
|
/**
|
package/dist/index.js
CHANGED
|
@@ -3518,6 +3518,33 @@ var _SearchService = class _SearchService {
|
|
|
3518
3518
|
this.staleEpochs = /* @__PURE__ */ new Map();
|
|
3519
3519
|
this.indexStrategy = indexStrategy;
|
|
3520
3520
|
}
|
|
3521
|
+
/**
|
|
3522
|
+
* Install the strategy resolved by `WikiMemory.setup()`. Before setup, the
|
|
3523
|
+
* service runs on an empty MiniSearch placeholder, as it always has.
|
|
3524
|
+
*/
|
|
3525
|
+
setIndexStrategy(strategy) {
|
|
3526
|
+
this.indexStrategy = strategy;
|
|
3527
|
+
}
|
|
3528
|
+
/**
|
|
3529
|
+
* One serialized drain turn for strategies with a durable ledger. Like
|
|
3530
|
+
* sync(), it never rejects: the keyword index is rebuildable, and a failed
|
|
3531
|
+
* drain leaves the ledger rows in place for the next turn.
|
|
3532
|
+
*/
|
|
3533
|
+
drainTurn(entityId) {
|
|
3534
|
+
const work = this.syncChain.then(async () => {
|
|
3535
|
+
try {
|
|
3536
|
+
try {
|
|
3537
|
+
await this.indexStrategy.drain();
|
|
3538
|
+
} finally {
|
|
3539
|
+
this.evictCache(entityId);
|
|
3540
|
+
}
|
|
3541
|
+
} catch (err) {
|
|
3542
|
+
console.warn(`[WikiMemory] search index drain failed for ${entityId ?? "*"}:`, err);
|
|
3543
|
+
}
|
|
3544
|
+
});
|
|
3545
|
+
this.syncChain = work;
|
|
3546
|
+
return work;
|
|
3547
|
+
}
|
|
3521
3548
|
/**
|
|
3522
3549
|
* Rebuilds the search index and clears the vector cache for a given entity.
|
|
3523
3550
|
* A direct replacement for manually syncing state after a DB transaction.
|
|
@@ -3527,6 +3554,7 @@ var _SearchService = class _SearchService {
|
|
|
3527
3554
|
* correct failure mode and killing the host process is not.
|
|
3528
3555
|
*/
|
|
3529
3556
|
async sync(entityId) {
|
|
3557
|
+
if (this.indexStrategy.drain) return this.drainTurn(entityId);
|
|
3530
3558
|
const work = this.syncChain.then(async () => {
|
|
3531
3559
|
try {
|
|
3532
3560
|
try {
|
|
@@ -3569,6 +3597,7 @@ var _SearchService = class _SearchService {
|
|
|
3569
3597
|
* already queued on the chain.
|
|
3570
3598
|
*/
|
|
3571
3599
|
async syncEntries(entityId, ids) {
|
|
3600
|
+
if (this.indexStrategy.drain) return this.drainTurn(entityId);
|
|
3572
3601
|
const uniqueIds = [...new Set(ids)];
|
|
3573
3602
|
const indexed = this.indexStrategy.hasIndexedEntity?.(entityId);
|
|
3574
3603
|
if (uniqueIds.length === 0 && !this.staleEntities.has(entityId) && indexed === true) {
|
|
@@ -3608,6 +3637,7 @@ var _SearchService = class _SearchService {
|
|
|
3608
3637
|
* cannot index itself, such as upsertGraph inside a host transaction.
|
|
3609
3638
|
*/
|
|
3610
3639
|
markStale(entityId) {
|
|
3640
|
+
if (this.indexStrategy.drain) return;
|
|
3611
3641
|
this.staleEntities.add(entityId);
|
|
3612
3642
|
this.staleEpochs.set(entityId, (this.staleEpochs.get(entityId) ?? 0) + 1);
|
|
3613
3643
|
}
|
|
@@ -3618,6 +3648,7 @@ var _SearchService = class _SearchService {
|
|
|
3618
3648
|
* stale it resolves at once, without waiting on work already on the chain.
|
|
3619
3649
|
*/
|
|
3620
3650
|
async syncStale() {
|
|
3651
|
+
if (this.indexStrategy.drain) return this.drainTurn();
|
|
3621
3652
|
const ids = [...this.staleEntities];
|
|
3622
3653
|
await Promise.all(ids.map((id) => this.syncEntries(id, [])));
|
|
3623
3654
|
}
|
|
@@ -3960,6 +3991,280 @@ function assertOwnedBy(entityId, documents) {
|
|
|
3960
3991
|
}
|
|
3961
3992
|
}
|
|
3962
3993
|
|
|
3994
|
+
// src/services/search/fts5Sql.ts
|
|
3995
|
+
var FTS5_STATE_KEY = "fts5_index_state";
|
|
3996
|
+
var FTS5_DRAIN_CHUNK = 500;
|
|
3997
|
+
function fts5TriggerNames(prefix) {
|
|
3998
|
+
return [`${prefix}entries_fts_ai`, `${prefix}entries_fts_au`, `${prefix}entries_fts_ad`];
|
|
3999
|
+
}
|
|
4000
|
+
function tagsExpr(col) {
|
|
4001
|
+
return `CASE WHEN json_valid(${col}) THEN
|
|
4002
|
+
CASE WHEN json_type(${col}) = 'array' THEN
|
|
4003
|
+
(SELECT group_concat(CASE type WHEN 'true' THEN 'true' WHEN 'false' THEN 'false'
|
|
4004
|
+
WHEN 'null' THEN '' ELSE value END, ' ')
|
|
4005
|
+
FROM json_each(${col}))
|
|
4006
|
+
ELSE ${col} END
|
|
4007
|
+
ELSE ${col} END`;
|
|
4008
|
+
}
|
|
4009
|
+
function fts5TablesDdl(p) {
|
|
4010
|
+
return `
|
|
4011
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS ${p}entries_fts USING fts5(
|
|
4012
|
+
id UNINDEXED, entity_id UNINDEXED, title, body, tags,
|
|
4013
|
+
tokenize = 'porter unicode61'
|
|
4014
|
+
);
|
|
4015
|
+
CREATE TABLE IF NOT EXISTS ${p}fts_map (
|
|
4016
|
+
fts_rowid INTEGER PRIMARY KEY,
|
|
4017
|
+
id TEXT NOT NULL UNIQUE,
|
|
4018
|
+
entity_id TEXT NOT NULL
|
|
4019
|
+
);
|
|
4020
|
+
CREATE INDEX IF NOT EXISTS ${p}fts_map_entity ON ${p}fts_map(entity_id);
|
|
4021
|
+
CREATE TABLE IF NOT EXISTS ${p}fts_pending (
|
|
4022
|
+
seq INTEGER PRIMARY KEY,
|
|
4023
|
+
entity_id TEXT NOT NULL,
|
|
4024
|
+
id TEXT NOT NULL
|
|
4025
|
+
);`;
|
|
4026
|
+
}
|
|
4027
|
+
function fts5TriggersDdl(p) {
|
|
4028
|
+
const [ai, au, ad] = fts5TriggerNames(p);
|
|
4029
|
+
return `
|
|
4030
|
+
CREATE TRIGGER IF NOT EXISTS ${ai} AFTER INSERT ON ${p}entries BEGIN
|
|
4031
|
+
INSERT INTO ${p}fts_pending (entity_id, id) VALUES (NEW.entity_id, NEW.id);
|
|
4032
|
+
END;
|
|
4033
|
+
CREATE TRIGGER IF NOT EXISTS ${au}
|
|
4034
|
+
AFTER UPDATE OF id, entity_id, title, body, tags, deleted_at ON ${p}entries BEGIN
|
|
4035
|
+
INSERT INTO ${p}fts_pending (entity_id, id) VALUES (OLD.entity_id, OLD.id);
|
|
4036
|
+
INSERT INTO ${p}fts_pending (entity_id, id)
|
|
4037
|
+
SELECT NEW.entity_id, NEW.id WHERE NEW.id <> OLD.id OR NEW.entity_id <> OLD.entity_id;
|
|
4038
|
+
END;
|
|
4039
|
+
CREATE TRIGGER IF NOT EXISTS ${ad} AFTER DELETE ON ${p}entries BEGIN
|
|
4040
|
+
INSERT INTO ${p}fts_pending (entity_id, id) VALUES (OLD.entity_id, OLD.id);
|
|
4041
|
+
END;`;
|
|
4042
|
+
}
|
|
4043
|
+
function drainHiSql(p) {
|
|
4044
|
+
return `SELECT max(seq) AS hi FROM (SELECT seq FROM ${p}fts_pending ORDER BY seq LIMIT ${FTS5_DRAIN_CHUNK})`;
|
|
4045
|
+
}
|
|
4046
|
+
function drainChunkSql(p) {
|
|
4047
|
+
const chunkIds = `SELECT id FROM ${p}fts_pending WHERE seq <= ?`;
|
|
4048
|
+
return [
|
|
4049
|
+
`DELETE FROM ${p}entries_fts WHERE rowid IN (SELECT m.fts_rowid FROM ${p}fts_map m WHERE m.id IN (${chunkIds}))`,
|
|
4050
|
+
`DELETE FROM ${p}fts_map WHERE id IN (${chunkIds})`,
|
|
4051
|
+
`INSERT INTO ${p}fts_map (id, entity_id)
|
|
4052
|
+
SELECT e.id, e.entity_id FROM ${p}entries e WHERE e.id IN (${chunkIds}) AND e.deleted_at IS NULL`,
|
|
4053
|
+
`INSERT INTO ${p}entries_fts (rowid, id, entity_id, title, body, tags)
|
|
4054
|
+
SELECT m.fts_rowid, e.id, e.entity_id, e.title, e.body, ${tagsExpr("e.tags")}
|
|
4055
|
+
FROM ${p}fts_map m JOIN ${p}entries e ON e.id = m.id
|
|
4056
|
+
WHERE m.id IN (${chunkIds})`,
|
|
4057
|
+
`DELETE FROM ${p}fts_pending WHERE seq <= ?`
|
|
4058
|
+
];
|
|
4059
|
+
}
|
|
4060
|
+
function rebuildSql(p) {
|
|
4061
|
+
return [
|
|
4062
|
+
`INSERT INTO ${p}fts_map (id, entity_id) SELECT id, entity_id FROM ${p}entries WHERE deleted_at IS NULL`,
|
|
4063
|
+
`INSERT INTO ${p}entries_fts (rowid, id, entity_id, title, body, tags)
|
|
4064
|
+
SELECT m.fts_rowid, e.id, e.entity_id, e.title, e.body, ${tagsExpr("e.tags")}
|
|
4065
|
+
FROM ${p}fts_map m JOIN ${p}entries e ON e.id = m.id`
|
|
4066
|
+
];
|
|
4067
|
+
}
|
|
4068
|
+
function fts5LedgerDropSql(p) {
|
|
4069
|
+
return [
|
|
4070
|
+
...fts5TriggerNames(p).map((n) => `DROP TRIGGER IF EXISTS ${n};`),
|
|
4071
|
+
`DROP TABLE IF EXISTS ${p}fts_pending;`,
|
|
4072
|
+
`DROP TABLE IF EXISTS ${p}fts_map;`
|
|
4073
|
+
].join("\n");
|
|
4074
|
+
}
|
|
4075
|
+
|
|
4076
|
+
// src/services/search/ftsQuery.ts
|
|
4077
|
+
var TOKEN = /[\p{L}\p{N}]+/gu;
|
|
4078
|
+
function buildFtsMatchQuery(query) {
|
|
4079
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4080
|
+
for (const m of query.toLowerCase().matchAll(TOKEN)) seen.add(m[0]);
|
|
4081
|
+
if (seen.size === 0) return null;
|
|
4082
|
+
return [...seen].map((t) => `"${t}"*`).join(" OR ");
|
|
4083
|
+
}
|
|
4084
|
+
|
|
4085
|
+
// src/services/search/Fts5IndexStrategy.ts
|
|
4086
|
+
var Fts5IndexStrategy = class {
|
|
4087
|
+
constructor(db, prefix, metadataRepo) {
|
|
4088
|
+
this.db = db;
|
|
4089
|
+
this.prefix = prefix;
|
|
4090
|
+
this.metadataRepo = metadataRepo;
|
|
4091
|
+
}
|
|
4092
|
+
async init() {
|
|
4093
|
+
const state = await this.metadataRepo.getMeta(FTS5_STATE_KEY);
|
|
4094
|
+
if (state === "live" && await this.triggersPresent()) {
|
|
4095
|
+
await this.db.execAsync(fts5TablesDdl(this.prefix));
|
|
4096
|
+
return;
|
|
4097
|
+
}
|
|
4098
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
4099
|
+
await tx.execAsync(`DROP TABLE IF EXISTS ${this.prefix}entries_fts;`);
|
|
4100
|
+
await tx.execAsync(fts5TablesDdl(this.prefix));
|
|
4101
|
+
await this.rebuildInTx(tx);
|
|
4102
|
+
await tx.execAsync(fts5TriggersDdl(this.prefix));
|
|
4103
|
+
await this.metadataRepo.setMeta(FTS5_STATE_KEY, "live", tx);
|
|
4104
|
+
});
|
|
4105
|
+
}
|
|
4106
|
+
async rebuildFromSource() {
|
|
4107
|
+
await this.db.withTransactionAsync((tx) => this.rebuildInTx(tx));
|
|
4108
|
+
}
|
|
4109
|
+
async drain() {
|
|
4110
|
+
const hiSql = drainHiSql(this.prefix);
|
|
4111
|
+
const chunk = drainChunkSql(this.prefix);
|
|
4112
|
+
for (; ; ) {
|
|
4113
|
+
const done = await this.db.withTransactionAsync(async (tx) => {
|
|
4114
|
+
const row = await tx.getFirstAsync(hiSql);
|
|
4115
|
+
if (row?.hi == null) return true;
|
|
4116
|
+
for (const sql of chunk) await tx.runAsync(sql, [row.hi]);
|
|
4117
|
+
return false;
|
|
4118
|
+
});
|
|
4119
|
+
if (done) return;
|
|
4120
|
+
}
|
|
4121
|
+
}
|
|
4122
|
+
async replace(entityId, ids, documents) {
|
|
4123
|
+
assertEntity(entityId, documents);
|
|
4124
|
+
const docIds = documents.map((d) => d.id);
|
|
4125
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
4126
|
+
await this.deleteByIds(tx, entityId, ids);
|
|
4127
|
+
await this.deleteByIds(tx, null, docIds);
|
|
4128
|
+
await this.insertDocs(tx, documents);
|
|
4129
|
+
});
|
|
4130
|
+
}
|
|
4131
|
+
async replaceEntity(entityId, documents) {
|
|
4132
|
+
assertEntity(entityId, documents);
|
|
4133
|
+
const p = this.prefix;
|
|
4134
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
4135
|
+
await tx.runAsync(
|
|
4136
|
+
`DELETE FROM ${p}entries_fts WHERE rowid IN (SELECT fts_rowid FROM ${p}fts_map WHERE entity_id = ?)`,
|
|
4137
|
+
[entityId]
|
|
4138
|
+
);
|
|
4139
|
+
await tx.runAsync(`DELETE FROM ${p}fts_map WHERE entity_id = ?`, [entityId]);
|
|
4140
|
+
await this.insertDocs(tx, documents);
|
|
4141
|
+
});
|
|
4142
|
+
}
|
|
4143
|
+
async replaceAll(documents) {
|
|
4144
|
+
const p = this.prefix;
|
|
4145
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
4146
|
+
await tx.execAsync(`DELETE FROM ${p}entries_fts; DELETE FROM ${p}fts_map; DELETE FROM ${p}fts_pending;`);
|
|
4147
|
+
await this.insertDocs(tx, documents);
|
|
4148
|
+
});
|
|
4149
|
+
}
|
|
4150
|
+
async search(query, options) {
|
|
4151
|
+
if (options.entityIds.length === 0) return [];
|
|
4152
|
+
const match = buildFtsMatchQuery(query);
|
|
4153
|
+
if (match === null) return [];
|
|
4154
|
+
const cap = options.preFilterLimit ?? options.limit;
|
|
4155
|
+
const limit = cap >= Number.MAX_SAFE_INTEGER ? -1 : cap;
|
|
4156
|
+
const t = `${this.prefix}entries_fts`;
|
|
4157
|
+
const bm25 = `bm25(${t}, 0.0, 0.0, 2.0, 1.0, 1.0)`;
|
|
4158
|
+
return this.db.getAllAsync(
|
|
4159
|
+
`SELECT id, entity_id, max(0.0, -${bm25}) AS score
|
|
4160
|
+
FROM ${t}
|
|
4161
|
+
WHERE ${t} MATCH ? AND entity_id IN (SELECT value FROM json_each(?))
|
|
4162
|
+
ORDER BY ${bm25}, id
|
|
4163
|
+
LIMIT ?`,
|
|
4164
|
+
[match, JSON.stringify(options.entityIds), limit]
|
|
4165
|
+
);
|
|
4166
|
+
}
|
|
4167
|
+
async deleteByIds(tx, entityId, ids) {
|
|
4168
|
+
if (ids.length === 0) return;
|
|
4169
|
+
const p = this.prefix;
|
|
4170
|
+
const json = JSON.stringify(ids);
|
|
4171
|
+
const whereEntity = entityId === null ? "" : " AND entity_id = ?";
|
|
4172
|
+
const args = [json];
|
|
4173
|
+
if (entityId !== null) args.push(entityId);
|
|
4174
|
+
await tx.runAsync(
|
|
4175
|
+
`DELETE FROM ${p}entries_fts WHERE rowid IN (
|
|
4176
|
+
SELECT fts_rowid FROM ${p}fts_map WHERE id IN (SELECT value FROM json_each(?))${whereEntity})`,
|
|
4177
|
+
args
|
|
4178
|
+
);
|
|
4179
|
+
await tx.runAsync(
|
|
4180
|
+
`DELETE FROM ${p}fts_map WHERE id IN (SELECT value FROM json_each(?))${whereEntity}`,
|
|
4181
|
+
args
|
|
4182
|
+
);
|
|
4183
|
+
}
|
|
4184
|
+
/** One INSERT pair per document: a throw part-way rolls back the whole transaction. */
|
|
4185
|
+
async insertDocs(tx, documents) {
|
|
4186
|
+
const p = this.prefix;
|
|
4187
|
+
for (const d of documents) {
|
|
4188
|
+
const { lastInsertRowId } = await tx.runAsync(
|
|
4189
|
+
`INSERT INTO ${p}fts_map (id, entity_id) VALUES (?, ?)`,
|
|
4190
|
+
[d.id, d.entity_id]
|
|
4191
|
+
);
|
|
4192
|
+
await tx.runAsync(
|
|
4193
|
+
`INSERT INTO ${p}entries_fts (rowid, id, entity_id, title, body, tags) VALUES (?, ?, ?, ?, ?, ?)`,
|
|
4194
|
+
[lastInsertRowId, d.id, d.entity_id, d.title, d.body, d.tags]
|
|
4195
|
+
);
|
|
4196
|
+
}
|
|
4197
|
+
}
|
|
4198
|
+
async rebuildInTx(tx) {
|
|
4199
|
+
const p = this.prefix;
|
|
4200
|
+
await tx.execAsync(`DELETE FROM ${p}entries_fts; DELETE FROM ${p}fts_map; DELETE FROM ${p}fts_pending;`);
|
|
4201
|
+
for (const sql of rebuildSql(p)) await tx.runAsync(sql);
|
|
4202
|
+
}
|
|
4203
|
+
async triggersPresent() {
|
|
4204
|
+
const names = fts5TriggerNames(this.prefix);
|
|
4205
|
+
const rows = await this.db.getAllAsync(
|
|
4206
|
+
`SELECT name FROM sqlite_master WHERE type = 'trigger' AND name IN (?, ?, ?)`,
|
|
4207
|
+
[...names]
|
|
4208
|
+
);
|
|
4209
|
+
return rows.length === names.length;
|
|
4210
|
+
}
|
|
4211
|
+
};
|
|
4212
|
+
function assertEntity(entityId, documents) {
|
|
4213
|
+
for (const d of documents) {
|
|
4214
|
+
if (d.entity_id !== entityId) {
|
|
4215
|
+
throw new TypeError(`IndexDocument ${d.id} has entity_id ${d.entity_id}, expected ${entityId}`);
|
|
4216
|
+
}
|
|
4217
|
+
}
|
|
4218
|
+
}
|
|
4219
|
+
|
|
4220
|
+
// src/services/search/createIndexStrategy.ts
|
|
4221
|
+
async function probeFts5(db, prefix) {
|
|
4222
|
+
const name = `temp.${prefix}fts5_probe`;
|
|
4223
|
+
try {
|
|
4224
|
+
await db.execAsync(`CREATE VIRTUAL TABLE IF NOT EXISTS ${name} USING fts5(x)`);
|
|
4225
|
+
const row = await db.getFirstAsync(`SELECT json_valid('[]') AS ok`);
|
|
4226
|
+
return row?.ok === 1;
|
|
4227
|
+
} catch {
|
|
4228
|
+
return false;
|
|
4229
|
+
} finally {
|
|
4230
|
+
try {
|
|
4231
|
+
await db.execAsync(`DROP TABLE IF EXISTS ${name}`);
|
|
4232
|
+
} catch {
|
|
4233
|
+
}
|
|
4234
|
+
}
|
|
4235
|
+
}
|
|
4236
|
+
async function detachFts5(db, prefix, metadataRepo) {
|
|
4237
|
+
const hasLedger = await metadataRepo.tableExists(`${prefix}fts_pending`);
|
|
4238
|
+
const state = await metadataRepo.getMeta(FTS5_STATE_KEY);
|
|
4239
|
+
if (!hasLedger && state !== "live") return;
|
|
4240
|
+
await db.withTransactionAsync(async (tx) => {
|
|
4241
|
+
await tx.execAsync(fts5LedgerDropSql(prefix));
|
|
4242
|
+
await metadataRepo.setMeta(FTS5_STATE_KEY, "detached", tx);
|
|
4243
|
+
});
|
|
4244
|
+
try {
|
|
4245
|
+
await db.execAsync(`DROP TABLE IF EXISTS ${prefix}entries_fts`);
|
|
4246
|
+
} catch {
|
|
4247
|
+
}
|
|
4248
|
+
}
|
|
4249
|
+
async function createIndexStrategy(db, prefix, metadataRepo, preferred) {
|
|
4250
|
+
if (preferred !== "minisearch") {
|
|
4251
|
+
if (await probeFts5(db, prefix)) {
|
|
4252
|
+
const strategy = new Fts5IndexStrategy(db, prefix, metadataRepo);
|
|
4253
|
+
try {
|
|
4254
|
+
await strategy.init();
|
|
4255
|
+
return strategy;
|
|
4256
|
+
} catch (err) {
|
|
4257
|
+
if (preferred === "fts5") throw err;
|
|
4258
|
+
console.warn("[WikiMemory] FTS5 init failed; falling back to MiniSearch:", err);
|
|
4259
|
+
}
|
|
4260
|
+
} else if (preferred === "fts5") {
|
|
4261
|
+
throw new Error(`indexStrategy 'fts5' was requested, but this SQLite build lacks FTS5 or json1.`);
|
|
4262
|
+
}
|
|
4263
|
+
}
|
|
4264
|
+
await detachFts5(db, prefix, metadataRepo);
|
|
4265
|
+
return new MiniSearchIndexStrategy();
|
|
4266
|
+
}
|
|
4267
|
+
|
|
3963
4268
|
// src/services/JobManager.ts
|
|
3964
4269
|
var JobManager = class {
|
|
3965
4270
|
constructor(prefix) {
|
|
@@ -10593,6 +10898,14 @@ var WikiMemory = class {
|
|
|
10593
10898
|
}
|
|
10594
10899
|
}
|
|
10595
10900
|
});
|
|
10901
|
+
this.searchService.setIndexStrategy(
|
|
10902
|
+
await createIndexStrategy(
|
|
10903
|
+
this.db,
|
|
10904
|
+
this.prefix,
|
|
10905
|
+
this.metadataRepo,
|
|
10906
|
+
this.options.config?.indexStrategy ?? "minisearch"
|
|
10907
|
+
)
|
|
10908
|
+
);
|
|
10596
10909
|
await this.searchService.sync();
|
|
10597
10910
|
}
|
|
10598
10911
|
async hasChanged(entityId, sourceRefOrEntries, sourceHashArg) {
|
|
@@ -10857,6 +11170,11 @@ var WikiMemory = class {
|
|
|
10857
11170
|
* entity stays stale and the next call retries it. Does not compute
|
|
10858
11171
|
* embeddings; that remains the maintenance sweep's job.
|
|
10859
11172
|
*
|
|
11173
|
+
* With the FTS5 index strategy this drains the index ledger in its own
|
|
11174
|
+
* transactions, so it MUST be called after the host transaction commits,
|
|
11175
|
+
* never from inside a `withTransactionAsync` callback. Calling it inside
|
|
11176
|
+
* one waits on the transaction mutex that the callback holds.
|
|
11177
|
+
*
|
|
10860
11178
|
* @throws {TypeError} when `entityId` is given but is not a non-empty string.
|
|
10861
11179
|
*/
|
|
10862
11180
|
async syncSearchIndex(entityId) {
|