@3sln/trove 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (162) hide show
  1. package/README.md +1227 -0
  2. package/package.json +75 -0
  3. package/packages/core/src/collections/index.js +249 -0
  4. package/packages/core/src/errors.js +186 -0
  5. package/packages/core/src/identity/discovery.js +210 -0
  6. package/packages/core/src/identity/index.js +188 -0
  7. package/packages/core/src/identity/jwt.js +199 -0
  8. package/packages/core/src/index.js +104 -0
  9. package/packages/core/src/indexers/contribution.js +115 -0
  10. package/packages/core/src/indexers/registry.js +162 -0
  11. package/packages/core/src/indexing.js +340 -0
  12. package/packages/core/src/issues.js +150 -0
  13. package/packages/core/src/kv.js +0 -0
  14. package/packages/core/src/links.js +141 -0
  15. package/packages/core/src/metadata/cursor.js +73 -0
  16. package/packages/core/src/metadata/interface.js +244 -0
  17. package/packages/core/src/metadata/memory.js +270 -0
  18. package/packages/core/src/metadata/sqlite.js +412 -0
  19. package/packages/core/src/notifications/index.js +139 -0
  20. package/packages/core/src/notifications/webpush.js +217 -0
  21. package/packages/core/src/plugins/contributions.js +177 -0
  22. package/packages/core/src/plugins/identity.js +98 -0
  23. package/packages/core/src/plugins/index.js +225 -0
  24. package/packages/core/src/plugins/indexers.js +142 -0
  25. package/packages/core/src/plugins/installStore.js +134 -0
  26. package/packages/core/src/plugins/package.js +102 -0
  27. package/packages/core/src/plugins/packageStore.js +61 -0
  28. package/packages/core/src/plugins/runtime.js +101 -0
  29. package/packages/core/src/plugins/sql.js +52 -0
  30. package/packages/core/src/retry.js +74 -0
  31. package/packages/core/src/scan.js +302 -0
  32. package/packages/core/src/search/embeddings.js +128 -0
  33. package/packages/core/src/search/index.js +200 -0
  34. package/packages/core/src/search/keywordStore.js +107 -0
  35. package/packages/core/src/search/sqliteStores.js +455 -0
  36. package/packages/core/src/search/tagMatch.js +59 -0
  37. package/packages/core/src/search/transformer.js +195 -0
  38. package/packages/core/src/search/vectorStore.js +274 -0
  39. package/packages/core/src/search/vectorize.js +249 -0
  40. package/packages/core/src/sidecar/document.js +213 -0
  41. package/packages/core/src/sidecar/index.js +174 -0
  42. package/packages/core/src/sidecar/manager.js +239 -0
  43. package/packages/core/src/sidecar/store.js +46 -0
  44. package/packages/core/src/signedUrls.js +170 -0
  45. package/packages/core/src/sqlite-d1.js +162 -0
  46. package/packages/core/src/sqlite-driver.js +42 -0
  47. package/packages/core/src/sqlite.js +162 -0
  48. package/packages/core/src/storage/filesystem.js +283 -0
  49. package/packages/core/src/storage/interface.js +222 -0
  50. package/packages/core/src/storage/memory.js +113 -0
  51. package/packages/core/src/storage/prefixed.js +75 -0
  52. package/packages/core/src/storage/s3.js +316 -0
  53. package/packages/core/src/storage/s3sigv4.js +185 -0
  54. package/packages/core/src/tasks.js +228 -0
  55. package/packages/core/src/uploads.js +386 -0
  56. package/packages/core/src/util.js +125 -0
  57. package/packages/core/src/vfs.js +666 -0
  58. package/packages/plugin-sdk/src/browser.js +316 -0
  59. package/packages/plugin-sdk/src/index.js +32 -0
  60. package/packages/plugin-sdk/src/protocol.js +59 -0
  61. package/packages/plugin-sdk/src/rpc.js +95 -0
  62. package/packages/server/src/adapters/bun.js +78 -0
  63. package/packages/server/src/adapters/node.js +115 -0
  64. package/packages/server/src/adapters/staticAssets.js +123 -0
  65. package/packages/server/src/adapters/webDist.js +70 -0
  66. package/packages/server/src/adapters/worker-tasks.js +206 -0
  67. package/packages/server/src/adapters/worker.js +159 -0
  68. package/packages/server/src/cachePolicy.js +34 -0
  69. package/packages/server/src/engine/README.md +88 -0
  70. package/packages/server/src/engine/actions/scanCollection.js +114 -0
  71. package/packages/server/src/engine/index.js +95 -0
  72. package/packages/server/src/engine/lazy.js +25 -0
  73. package/packages/server/src/engine/providers/access.js +363 -0
  74. package/packages/server/src/engine/providers/core.js +405 -0
  75. package/packages/server/src/engine/providers/scan.js +67 -0
  76. package/packages/server/src/index.js +698 -0
  77. package/packages/server/src/manifest.js +98 -0
  78. package/packages/server/src/mcp/auth.js +40 -0
  79. package/packages/server/src/mcp/index.js +213 -0
  80. package/packages/server/src/mcp/protocol.js +181 -0
  81. package/packages/server/src/mcp/tools.js +351 -0
  82. package/packages/server/src/router.js +229 -0
  83. package/packages/server/src/routes.js +1066 -0
  84. package/packages/server/src/scope.js +43 -0
  85. package/packages/web/dist/assets/chunk-4xqbzebh.js +5 -0
  86. package/packages/web/dist/assets/chunk-4xqbzebh.js.map +9 -0
  87. package/packages/web/dist/assets/chunk-h05bxfbs.js +5 -0
  88. package/packages/web/dist/assets/chunk-h05bxfbs.js.map +10 -0
  89. package/packages/web/dist/assets/main-4cxs7prw.js +356 -0
  90. package/packages/web/dist/assets/main-4cxs7prw.js.map +103 -0
  91. package/packages/web/dist/assets/styles-kcx1x337.css +1 -0
  92. package/packages/web/dist/icon.svg +11 -0
  93. package/packages/web/dist/index.html +16 -0
  94. package/packages/web/dist/sql-wasm.wasm +0 -0
  95. package/packages/web/dist/sw.js +186 -0
  96. package/packages/web/src/bl/actions.js +410 -0
  97. package/packages/web/src/bl/activity.js +306 -0
  98. package/packages/web/src/bl/commands.js +274 -0
  99. package/packages/web/src/bl/fileType.js +49 -0
  100. package/packages/web/src/bl/index.js +70 -0
  101. package/packages/web/src/bl/links.js +54 -0
  102. package/packages/web/src/bl/offline.js +268 -0
  103. package/packages/web/src/bl/openers.js +71 -0
  104. package/packages/web/src/bl/pluginInstall.js +59 -0
  105. package/packages/web/src/bl/services.js +143 -0
  106. package/packages/web/src/bl/social.js +234 -0
  107. package/packages/web/src/bl/tagQuery.js +44 -0
  108. package/packages/web/src/main.js +10 -0
  109. package/packages/web/src/platform/api.js +529 -0
  110. package/packages/web/src/platform/commands.js +89 -0
  111. package/packages/web/src/platform/context.js +77 -0
  112. package/packages/web/src/platform/contributions.js +156 -0
  113. package/packages/web/src/platform/index.js +150 -0
  114. package/packages/web/src/platform/keybindings.js +199 -0
  115. package/packages/web/src/platform/mediaUrls.js +137 -0
  116. package/packages/web/src/platform/navigation.js +131 -0
  117. package/packages/web/src/platform/notifications.js +50 -0
  118. package/packages/web/src/platform/overlay.js +81 -0
  119. package/packages/web/src/platform/pluginClientDb.js +132 -0
  120. package/packages/web/src/platform/pluginDock.js +141 -0
  121. package/packages/web/src/platform/pluginFrames.js +194 -0
  122. package/packages/web/src/platform/pluginHost.js +648 -0
  123. package/packages/web/src/platform/pluginMedia.js +62 -0
  124. package/packages/web/src/platform/pluginModules.js +90 -0
  125. package/packages/web/src/platform/pluginNet.js +71 -0
  126. package/packages/web/src/platform/pluginPackage.js +247 -0
  127. package/packages/web/src/platform/pluginRpc.js +377 -0
  128. package/packages/web/src/platform/pluginSigning.js +168 -0
  129. package/packages/web/src/platform/pluginStore.js +67 -0
  130. package/packages/web/src/platform/settings.js +101 -0
  131. package/packages/web/src/platform/spatialNav.js +286 -0
  132. package/packages/web/src/platform/viewport.js +123 -0
  133. package/packages/web/src/platform/voice.js +133 -0
  134. package/packages/web/src/platform/voiceSearch.js +155 -0
  135. package/packages/web/src/platform/whenclause.js +162 -0
  136. package/packages/web/src/platform/workbench.js +156 -0
  137. package/packages/web/src/runtime.js +73 -0
  138. package/packages/web/src/styles.css +1382 -0
  139. package/packages/web/src/ui/components/activityBar.js +35 -0
  140. package/packages/web/src/ui/components/activityPanel.js +132 -0
  141. package/packages/web/src/ui/components/commandPalette.js +154 -0
  142. package/packages/web/src/ui/components/editorArea.js +75 -0
  143. package/packages/web/src/ui/components/launcher.js +392 -0
  144. package/packages/web/src/ui/components/openers/index.js +212 -0
  145. package/packages/web/src/ui/components/openers/markdown.js +222 -0
  146. package/packages/web/src/ui/components/overlays.js +255 -0
  147. package/packages/web/src/ui/components/phoneChrome.js +188 -0
  148. package/packages/web/src/ui/components/pluginReview.js +151 -0
  149. package/packages/web/src/ui/components/pluginsView.js +120 -0
  150. package/packages/web/src/ui/components/settingsView.js +258 -0
  151. package/packages/web/src/ui/components/social.js +290 -0
  152. package/packages/web/src/ui/components/statusBar.js +198 -0
  153. package/packages/web/src/ui/components/views/grid.js +115 -0
  154. package/packages/web/src/ui/components/views/index.js +155 -0
  155. package/packages/web/src/ui/components/views/list.js +50 -0
  156. package/packages/web/src/ui/components/views/parts.js +58 -0
  157. package/packages/web/src/ui/compositions/workbench.js +125 -0
  158. package/packages/web/src/ui/format.js +33 -0
  159. package/packages/web/src/ui/icon.js +81 -0
  160. package/packages/web/src/ui/media.js +114 -0
  161. package/packages/web/src/ui/sanitize.js +86 -0
  162. package/packages/web/src/workbench.js +205 -0
@@ -0,0 +1,455 @@
1
+ // Durable search: a VectorStore over sqlite-vec and a KeywordStore over FTS5.
2
+ //
3
+ // The in-memory defaults were the single biggest gap between "works" and "runs": files
4
+ // and metadata persisted, the search index did not, and nothing rebuilt it — so a
5
+ // restart left a drive whose contents were all present and none of it findable. In an
6
+ // app where search IS the navigation, that is data loss in everything but name.
7
+ //
8
+ // Both stores live in the SAME SQLite file the metadata already uses (the `search` key,
9
+ // which LocalSqliteProvider co-locates with the main db), so the single-file self-host
10
+ // story holds: one file to back up, one file to move.
11
+ //
12
+ // They are constructed through a SqliteProvider rather than a raw handle, so nothing
13
+ // here is bound to bun:sqlite or node:sqlite — the provider is the seam, and a
14
+ // deployment on something else supplies its own.
15
+ //
16
+ // Availability differs between the two on purpose:
17
+ // • FTS5 is compiled into both bun:sqlite and node:sqlite — always available.
18
+ // • sqlite-vec is a loadable extension (a prebuilt native artifact per platform), so
19
+ // it can be absent. `SqliteVectorStore.open()` returns null rather than throwing,
20
+ // and the caller falls back to memory with a warning. A drive that can't do
21
+ // semantic search should still start and still do keyword search.
22
+
23
+ import { TroveError } from '../errors.js';
24
+ import { tokenize, KeywordStore } from './keywordStore.js';
25
+ import { VectorStore } from './vectorStore.js';
26
+
27
+ /** The provider key both stores share; co-located with metadata in the main db file. */
28
+ export const SEARCH_DB_KEY = 'search';
29
+
30
+ // --- vectors -----------------------------------------------------------------
31
+
32
+ /**
33
+ * Nearest-neighbour search over a sqlite-vec `vec0` table.
34
+ *
35
+ * sqlite-vec 0.1.x is exact brute force — no ANN index — so query cost is linear in
36
+ * the number of chunks. That is a deliberate trade: it is ~3× faster than the JS
37
+ * in-memory store (SIMD C vs a JS loop), holds no heap, and survives a restart.
38
+ * Measured at 384 dimensions: 50k chunks → ~40 ms per query, ~76 MB on disk. Past a
39
+ * few hundred thousand chunks the answer is an ANN store (Qdrant/Vectorize ship as
40
+ * adapters already), not a bigger brute force.
41
+ */
42
+ export class SqliteVectorStore extends VectorStore {
43
+ /**
44
+ * Open the store, or return null when the sqlite-vec extension isn't installed on
45
+ * this platform. Null rather than a throw: missing semantic search is a degraded
46
+ * deployment, not a broken one, and the caller decides what to fall back to.
47
+ * @param {{provider: object, dimensions: number, key?: string}} opts
48
+ * @returns {Promise<SqliteVectorStore|null>}
49
+ */
50
+ static async open({ provider, dimensions, key = SEARCH_DB_KEY }) {
51
+ if (!provider) throw TroveError.invalid('SqliteVectorStore needs a SqliteProvider');
52
+ if (!dimensions) throw TroveError.invalid('SqliteVectorStore needs the embedding dimensions');
53
+ const db = await provider.obtain({ key });
54
+ // The raw handle is needed to load an extension — that is not part of the
55
+ // SqliteDatabase interface, and deliberately so: it is a local-backend concern.
56
+ const raw = db.raw;
57
+ if (!raw?.loadExtension && !raw?.enableLoadExtension) return null;
58
+ let vec;
59
+ try {
60
+ vec = await import('sqlite-vec');
61
+ } catch {
62
+ return null; // dependency not installed
63
+ }
64
+ try {
65
+ if (raw.enableLoadExtension) raw.enableLoadExtension(true);
66
+ if (typeof vec.load === 'function') vec.load(raw);
67
+ else raw.loadExtension(vec.getLoadablePath());
68
+ await db.get('SELECT vec_version() AS v');
69
+ } catch {
70
+ return null; // extension present but unloadable here (wrong arch, disabled, …)
71
+ }
72
+ const store = new SqliteVectorStore(db, dimensions);
73
+ await store.init();
74
+ return store;
75
+ }
76
+
77
+ constructor(db, dimensions) {
78
+ super();
79
+ this.db = db;
80
+ this._dimensions = dimensions;
81
+ }
82
+
83
+ async init() {
84
+ // A vec0 table's dimension is fixed at CREATE time, so an index built for one
85
+ // embedding provider is meaningless to another. Switching providers (or their
86
+ // `dimensions`) has to drop the vectors — otherwise every insert would fail with a
87
+ // dimension error and the drive would look broken rather than merely unindexed.
88
+ // Dropping leaves the index empty, which is exactly the state the startup rebuild
89
+ // watches for, so the vectors come back under the new model.
90
+ await this.db.exec('CREATE TABLE IF NOT EXISTS vec_config (k TEXT PRIMARY KEY, v TEXT)');
91
+ const prior = Number((await this.db.get("SELECT v FROM vec_config WHERE k = 'dimensions'"))?.v);
92
+ if (prior && prior !== this._dimensions) {
93
+ await this.db.exec('DROP TABLE IF EXISTS vec_docs; DROP TABLE IF EXISTS vec_meta;');
94
+ console.warn(`[trove] embedding dimensions changed (${prior} → ${this._dimensions}) — the vector index was dropped and will be rebuilt`);
95
+ }
96
+
97
+ // vec0 holds the vectors; a plain sidecar table holds what we filter and return by.
98
+ // They are kept in step by always writing/deleting through this class, and the
99
+ // sidecar is what `removeByNode`/`removeByIndexer` scan — a vec0 table can't be
100
+ // queried by a non-vector predicate in 0.1.x.
101
+ await this.db.exec(`
102
+ CREATE VIRTUAL TABLE IF NOT EXISTS vec_docs USING vec0(
103
+ doc_id TEXT PRIMARY KEY,
104
+ embedding float[${this._dimensions}]
105
+ );
106
+ CREATE TABLE IF NOT EXISTS vec_meta (
107
+ doc_id TEXT PRIMARY KEY,
108
+ nodeId TEXT NOT NULL,
109
+ indexerId TEXT NOT NULL,
110
+ fields TEXT NOT NULL DEFAULT '{}'
111
+ );
112
+ CREATE INDEX IF NOT EXISTS idx_vec_meta_node ON vec_meta(nodeId);
113
+ CREATE INDEX IF NOT EXISTS idx_vec_meta_indexer ON vec_meta(indexerId);
114
+ `);
115
+ await this.db.run("INSERT INTO vec_config(k, v) VALUES ('dimensions', ?) ON CONFLICT(k) DO UPDATE SET v = excluded.v", String(this._dimensions));
116
+ }
117
+
118
+ /**
119
+ * Add documents — serialized, one caller at a time.
120
+ *
121
+ * The INSERT and the `SELECT last_insert_rowid()` that reads its rowid are two
122
+ * separate round trips on a SHARED connection, and FTS5 cannot return the rowid
123
+ * inline (`RETURNING rowid` on a virtual table yields -1). So two concurrent
124
+ * `indexNode` calls — two simultaneous uploads — interleaved between them, and
125
+ * document A recorded document B's rowid in `kw_meta`. Re-indexing A then deleted
126
+ * B's row: B disappeared from keyword search with nothing having touched it, and A's
127
+ * superseded chunk was orphaned in the index for good.
128
+ *
129
+ * A promise chain is enough because the hazard is within one process on one
130
+ * connection; across processes SQLite's own locking applies.
131
+ */
132
+ add(docs) {
133
+ const run = () => this.#addLocked(docs);
134
+ const next = (this._writes || Promise.resolve()).then(run, run);
135
+ this._writes = next.catch(() => {});
136
+ return next;
137
+ }
138
+
139
+ async #addLocked(docs) {
140
+ for (const doc of docs) {
141
+ const vector = toF32(doc.vector);
142
+ if (vector.length !== this._dimensions) {
143
+ throw TroveError.invalid(`Vector dim ${vector.length} != index dim ${this._dimensions}`);
144
+ }
145
+ // vec0 has no UPSERT, so a re-index deletes then inserts.
146
+ await this.remove(doc.id);
147
+ await this.db.batch([
148
+ { sql: 'INSERT INTO vec_docs(doc_id, embedding) VALUES (?, ?)', params: [doc.id, bytesOf(vector)] },
149
+ {
150
+ sql: 'INSERT INTO vec_meta(doc_id, nodeId, indexerId, fields) VALUES (?,?,?,?)',
151
+ params: [doc.id, doc.nodeId, doc.indexerId, JSON.stringify(doc.fields || {})],
152
+ },
153
+ ]);
154
+ }
155
+ }
156
+
157
+ async remove(docId) {
158
+ await this.db.batch([
159
+ { sql: 'DELETE FROM vec_docs WHERE doc_id = ?', params: [docId] },
160
+ { sql: 'DELETE FROM vec_meta WHERE doc_id = ?', params: [docId] },
161
+ ]);
162
+ }
163
+
164
+ async removeByNode(nodeId) {
165
+ await this.#removeWhere('nodeId = ?', [nodeId]);
166
+ }
167
+ async removeByIndexer(indexerId) {
168
+ await this.#removeWhere('indexerId = ?', [indexerId]);
169
+ }
170
+ async removeByNodeIndexer(nodeId, indexerId) {
171
+ await this.#removeWhere('nodeId = ? AND indexerId = ?', [nodeId, indexerId]);
172
+ }
173
+ async #removeWhere(where, params) {
174
+ const rows = await this.db.all(`SELECT doc_id FROM vec_meta WHERE ${where}`, ...params);
175
+ if (!rows.length) return;
176
+ await this.db.batch([
177
+ ...rows.map((r) => ({ sql: 'DELETE FROM vec_docs WHERE doc_id = ?', params: [r.doc_id] })),
178
+ { sql: `DELETE FROM vec_meta WHERE ${where}`, params },
179
+ ]);
180
+ }
181
+
182
+ async query(vector, opts = {}) {
183
+ const limit = opts.limit ?? 40;
184
+ const allow = opts.indexers?.length ? new Set(opts.indexers) : null;
185
+ // KNN runs over every vector, so an indexer filter can't narrow the scan in 0.1.x.
186
+ // Over-fetch and filter after, rather than returning fewer than `limit` — a caller
187
+ // that asked for 40 and got 3 because 37 belonged to another indexer would look
188
+ // like a drive with nothing in it.
189
+ const k = allow ? Math.min(limit * 8, 1000) : limit;
190
+ const rows = await this.db.all(
191
+ `SELECT d.doc_id AS docId, d.distance AS distance, m.nodeId, m.indexerId, m.fields
192
+ FROM vec_docs d JOIN vec_meta m ON m.doc_id = d.doc_id
193
+ WHERE d.embedding MATCH ? AND k = ?
194
+ ORDER BY d.distance`,
195
+ bytesOf(toF32(vector)), k,
196
+ );
197
+ const out = [];
198
+ for (const r of rows) {
199
+ if (allow && !allow.has(r.indexerId)) continue;
200
+ out.push({
201
+ docId: r.docId, nodeId: r.nodeId, indexerId: r.indexerId,
202
+ // vec0 returns L2 distance over normalised vectors; cosine similarity is
203
+ // 1 - d²/2, which keeps scores comparable with the other stores' 0..1.
204
+ score: 1 - (r.distance * r.distance) / 2,
205
+ fields: parseJson(r.fields),
206
+ });
207
+ if (out.length >= limit) break;
208
+ }
209
+ return out;
210
+ }
211
+
212
+ async count() {
213
+ return (await this.db.get('SELECT COUNT(*) AS n FROM vec_meta'))?.n ?? 0;
214
+ }
215
+ }
216
+
217
+ // --- keywords ----------------------------------------------------------------
218
+
219
+ /**
220
+ * Lexical search over FTS5, which is compiled into both bun:sqlite and node:sqlite —
221
+ * no extension, no dependency, nothing to install. This is the half that makes
222
+ * "find the file I named X" work, and it was being lost on restart just like the
223
+ * vectors.
224
+ */
225
+ export class SqliteKeywordStore extends KeywordStore {
226
+ static async open({ provider, key = SEARCH_DB_KEY }) {
227
+ if (!provider) throw TroveError.invalid('SqliteKeywordStore needs a SqliteProvider');
228
+ const store = new SqliteKeywordStore(await provider.obtain({ key }));
229
+ await store.init();
230
+ return store;
231
+ }
232
+
233
+ constructor(db) {
234
+ super();
235
+ this.db = db;
236
+ }
237
+
238
+ async init() {
239
+ // `content` is indexed; the rest are UNINDEXED so they're stored and returnable
240
+ // without polluting the term index.
241
+ //
242
+ // The `kw_meta` sidecar exists for DELETES, and it is the difference between a
243
+ // drive that stays fast and one that doesn't. An FTS5 table can only be searched by
244
+ // its term index — a predicate on an UNINDEXED column is a full scan of every row.
245
+ // Every single write re-indexes a node, which means deleting its old rows first, so
246
+ // without this each upload costs a scan of the entire index and the drive slows
247
+ // down in proportion to how much is in it. Measured: uploading 2,000 files took
248
+ // 14s for the first 500 and 40s for the last 500, purely from this.
249
+ //
250
+ // With the sidecar, a delete is an indexed lookup followed by `WHERE rowid = ?`,
251
+ // which is the one predicate FTS5 answers directly.
252
+ await this.db.exec(`
253
+ CREATE VIRTUAL TABLE IF NOT EXISTS kw_docs USING fts5(
254
+ content,
255
+ body UNINDEXED,
256
+ doc_id UNINDEXED,
257
+ nodeId UNINDEXED,
258
+ indexerId UNINDEXED,
259
+ fields UNINDEXED,
260
+ tokenize = 'porter unicode61'
261
+ );
262
+ CREATE TABLE IF NOT EXISTS kw_meta (
263
+ doc_id TEXT PRIMARY KEY,
264
+ rid INTEGER NOT NULL,
265
+ nodeId TEXT NOT NULL,
266
+ indexerId TEXT NOT NULL
267
+ );
268
+ CREATE INDEX IF NOT EXISTS idx_kw_meta_node ON kw_meta(nodeId);
269
+ CREATE INDEX IF NOT EXISTS idx_kw_meta_indexer ON kw_meta(indexerId);
270
+ CREATE INDEX IF NOT EXISTS idx_kw_meta_node_indexer ON kw_meta(nodeId, indexerId);
271
+ `);
272
+ // An index written before kw_meta existed has rows the sidecar doesn't know about.
273
+ // Backfill it once rather than silently leaking those rows forever — they would
274
+ // never be deleted, so a re-index would double-count every document.
275
+ const orphaned = await this.db.get(
276
+ 'SELECT COUNT(*) AS n FROM kw_docs WHERE rowid NOT IN (SELECT rid FROM kw_meta)',
277
+ );
278
+ if (orphaned?.n) await this.#adoptExistingRows();
279
+ }
280
+
281
+ /** Bring pre-sidecar rows under management (one-time, on upgrade). */
282
+ async #adoptExistingRows() {
283
+ const rows = await this.db.all(
284
+ 'SELECT rowid AS rid, doc_id, nodeId, indexerId FROM kw_docs WHERE rowid NOT IN (SELECT rid FROM kw_meta)',
285
+ );
286
+ if (!rows.length) return;
287
+ await this.db.batch(rows.map((r) => ({
288
+ sql: 'INSERT INTO kw_meta(doc_id, rid, nodeId, indexerId) VALUES (?,?,?,?) ON CONFLICT(doc_id) DO UPDATE SET rid = excluded.rid',
289
+ params: [r.doc_id, r.rid, r.nodeId, r.indexerId],
290
+ })));
291
+ console.warn(`[trove] adopted ${rows.length} pre-existing keyword rows into the delete index`);
292
+ }
293
+
294
+ async add(docs) {
295
+ if (!docs.length) return;
296
+ // Resolve the rowids of anything being replaced first — an indexed lookup, not a
297
+ // scan of the term index.
298
+ const ids = docs.map((d) => d.id);
299
+ const existing = await this.db.all(
300
+ `SELECT doc_id, rid FROM kw_meta WHERE doc_id IN (${ids.map(() => '?').join(',')})`,
301
+ ...ids,
302
+ );
303
+ if (existing.length) {
304
+ await this.db.batch(existing.map((row) => ({ sql: 'DELETE FROM kw_docs WHERE rowid = ?', params: [row.rid] })));
305
+ }
306
+ for (const d of docs) {
307
+ const body = d.text || '';
308
+ // `content` is what gets INDEXED — the text plus the field values, so a search
309
+ // for a filename finds the chunk. `body` is what gets SHOWN, kept separate so a
310
+ // snippet is the document's prose and not prose with metadata glued on.
311
+ const content = [body, ...Object.values(d.fields || {})].join(' ');
312
+ // SQLite assigns the rowid; we read it back. An in-process counter would be
313
+ // faster to write in a batch, and was — until two Trove processes opened the same
314
+ // database, both started counting at 1, and every insert failed on the primary
315
+ // key. There is no coordination between processes to have, so the only correct
316
+ // answer is to let the database allocate. The read-back is a same-connection call
317
+ // and costs nothing measurable next to the FTS insert itself.
318
+ await this.db.run(
319
+ 'INSERT INTO kw_docs(content, body, doc_id, nodeId, indexerId, fields) VALUES (?,?,?,?,?,?)',
320
+ content, body, d.id, d.nodeId, d.indexerId, JSON.stringify(d.fields || {}),
321
+ );
322
+ const rid = (await this.db.get('SELECT last_insert_rowid() AS rid'))?.rid;
323
+ await this.db.run(
324
+ 'INSERT INTO kw_meta(doc_id, rid, nodeId, indexerId) VALUES (?,?,?,?) '
325
+ + 'ON CONFLICT(doc_id) DO UPDATE SET rid = excluded.rid, nodeId = excluded.nodeId, indexerId = excluded.indexerId',
326
+ d.id, rid, d.nodeId, d.indexerId,
327
+ );
328
+ }
329
+ }
330
+
331
+ async removeByNode(nodeId) {
332
+ await this.#removeWhere('nodeId = ?', [nodeId]);
333
+ }
334
+ async removeByIndexer(indexerId) {
335
+ await this.#removeWhere('indexerId = ?', [indexerId]);
336
+ }
337
+ async removeByNodeIndexer(nodeId, indexerId) {
338
+ await this.#removeWhere('nodeId = ? AND indexerId = ?', [nodeId, indexerId]);
339
+ }
340
+ /** Indexed lookup in the sidecar, then delete by rowid — the one predicate FTS5 answers. */
341
+ async #removeWhere(where, params) {
342
+ const rows = await this.db.all(`SELECT rid FROM kw_meta WHERE ${where}`, ...params);
343
+ if (!rows.length) return;
344
+ await this.db.batch([
345
+ ...rows.map((r) => ({ sql: 'DELETE FROM kw_docs WHERE rowid = ?', params: [r.rid] })),
346
+ { sql: `DELETE FROM kw_meta WHERE ${where}`, params },
347
+ ]);
348
+ }
349
+
350
+ async search(query, opts = {}) {
351
+ const match = toMatchQuery(query);
352
+ if (!match) return [];
353
+ const params = [match];
354
+ let where = 'kw_docs MATCH ?';
355
+ if (opts.indexers?.length) {
356
+ where += ` AND indexerId IN (${opts.indexers.map(() => '?').join(',')})`;
357
+ params.push(...opts.indexers);
358
+ }
359
+ params.push(opts.limit ?? 40);
360
+ const rows = await this.db.all(
361
+ `SELECT doc_id AS docId, nodeId, indexerId, fields, bm25(kw_docs) AS rank
362
+ FROM kw_docs WHERE ${where} ORDER BY rank LIMIT ?`,
363
+ ...params,
364
+ );
365
+ // bm25() is negative, better = more negative. The rest of the system works in
366
+ // 0..1-where-higher-is-better, so map into that shape rather than leaking one
367
+ // store's convention into the blend in SearchService.
368
+ return rows.map((r) => ({
369
+ docId: r.docId, nodeId: r.nodeId, indexerId: r.indexerId,
370
+ score: scoreFromBm25(r.rank), fields: parseJson(r.fields),
371
+ }));
372
+ }
373
+
374
+ /**
375
+ * A window of the document's prose around the first matching term.
376
+ *
377
+ * Built by hand rather than with FTS5's `snippet()`: that only works on INDEXED
378
+ * columns, and the indexed column deliberately has the field values appended, so it
379
+ * would show "…the spice melange dune.txt". The reader wants the prose. This also
380
+ * matches MemoryKeywordStore's excerpt exactly, so swapping stores doesn't change
381
+ * what a result looks like.
382
+ */
383
+ async snippet(docId, query) {
384
+ // Via the sidecar, for the same reason deletes go through it: `doc_id` is an
385
+ // UNINDEXED FTS5 column, so looking a document up by it scans the whole index. One
386
+ // snippet per result row means a page of results was doing forty full scans — the
387
+ // single largest cost in a search, and it grew with the drive.
388
+ const meta = await this.db.get('SELECT rid FROM kw_meta WHERE doc_id = ?', docId);
389
+ if (!meta) return null;
390
+ const row = await this.db.get('SELECT body FROM kw_docs WHERE rowid = ?', meta.rid);
391
+ const text = row?.body;
392
+ if (!text) return null;
393
+ const lower = text.toLowerCase();
394
+ let at = -1;
395
+ for (const t of tokenize(query)) {
396
+ const i = lower.indexOf(t);
397
+ if (i >= 0) { at = i; break; }
398
+ }
399
+ if (at < 0) return text.slice(0, 160).trim();
400
+ const start = Math.max(0, at - 60);
401
+ return (start > 0 ? '…' : '') + text.slice(start, start + 200).trim() + '…';
402
+ }
403
+
404
+ async count() {
405
+ // Counts the REAL index, not the sidecar. The sidecar is a delete index; it is not
406
+ // authoritative about what the index contains, and the two can drift — a restore
407
+ // that lost the FTS table, or someone clearing it by hand. Since the only caller is
408
+ // the startup "was the index lost?" check, an honest answer is worth more than the
409
+ // cheaper one: reading the sidecar would report a full index sitting on top of no
410
+ // content, and the rebuild that exists for exactly that case would never fire.
411
+ return (await this.db.get('SELECT COUNT(*) AS n FROM kw_docs'))?.n ?? 0;
412
+ }
413
+ }
414
+
415
+ // --- helpers -----------------------------------------------------------------
416
+
417
+ function toF32(v) {
418
+ return v instanceof Float32Array ? v : Float32Array.from(v);
419
+ }
420
+ function bytesOf(f32) {
421
+ return new Uint8Array(f32.buffer, f32.byteOffset, f32.byteLength);
422
+ }
423
+ function parseJson(s) {
424
+ try { return s ? JSON.parse(s) : {}; } catch { return {}; }
425
+ }
426
+
427
+ /**
428
+ * Turn a user's raw words into an FTS5 MATCH expression.
429
+ *
430
+ * The input is untrusted and FTS5's query syntax is full of operators (`"`, `*`, `^`,
431
+ * `NEAR`, `OR`), so a raw string can be a syntax error — which would surface as a
432
+ * failed search rather than no results. Every token is quoted as a literal and joined
433
+ * with OR, matching the in-memory store's "score by term coverage" behaviour.
434
+ */
435
+ function toMatchQuery(query) {
436
+ const terms = tokenize(query);
437
+ if (!terms.length) return null;
438
+ return terms.map((t) => `"${t.replace(/"/g, '')}"`).join(' OR ');
439
+ }
440
+
441
+ /**
442
+ * Map BM25 (negative, lower is better) onto 0..1 with higher better, FLOORED at 0.5.
443
+ *
444
+ * The floor matters. BM25's IDF term is zero when a term appears in every document, so
445
+ * on a small drive — exactly the case for a new self-host — every match scores 0 and
446
+ * contributes nothing to the hybrid blend, making keyword search silently useless
447
+ * until the corpus grows. A match is evidence regardless of how common the term is, so
448
+ * matching at all is worth 0.5 and BM25 ranks within the matches. This also keeps the
449
+ * band comparable to MemoryKeywordStore's term-coverage score, so swapping stores
450
+ * doesn't shift the semantic/keyword balance.
451
+ */
452
+ function scoreFromBm25(rank) {
453
+ const r = typeof rank === 'number' && rank < 0 ? -rank : 0;
454
+ return 0.5 + 0.5 * (r / (r + 1)); // saturating: one huge score can't dominate
455
+ }
@@ -0,0 +1,59 @@
1
+ // One tag matcher, for everybody.
2
+ //
3
+ // There were four: this logic in `transformer.js` (post-filtering semantic results), a
4
+ // near-copy in the memory metadata store, a third in the browser (used by the offline
5
+ // path), and SQL in the sqlite store. Each pair "mirrored" the others in a comment and
6
+ // diverged in fact:
7
+ //
8
+ // - the client lowercased both operands for `<`/`<=`/`>`/`>=`; the server did not, so
9
+ // `#author:>Bob` against `author: "alice"` was false online and true offline;
10
+ // - the sqlite store bound `String(value)` against `json_extract`, which preserves JSON
11
+ // types, so a numeric tag `pages: 120` matched `#pages:!=120` and not `#pages:120`;
12
+ // - the sqlite store treated `present` as `IS NOT NULL`, so a tag explicitly set to
13
+ // `false` counted as present — everywhere else it does not;
14
+ // - the sqlite store never consulted `meta`, which the interface documents as part of
15
+ // what a filter matches.
16
+ //
17
+ // So the semantics live here, once, and the SQL in `metadata/sqlite.js` is written to
18
+ // agree with THIS file rather than with its own history.
19
+ //
20
+ // The rules:
21
+ // present the tag exists and is not `false` and not `''`
22
+ // = / != numeric compare when BOTH sides are numbers, else case-insensitive text
23
+ // < <= > >= same, and text comparison is case-insensitive (it was not on the server,
24
+ // which is the one difference a user could actually observe between an
25
+ // online and an offline search)
26
+
27
+ /** A node's filterable properties: `meta`, overlaid by its merged tags. */
28
+ export function tagProps(node, mergedTags = null) {
29
+ return { ...(node?.meta || {}), ...(mergedTags || node?.tags || {}) };
30
+ }
31
+
32
+ /** Does one value satisfy one parsed filter? */
33
+ export function matchesFilter(value, filter) {
34
+ if (filter.present) return value != null && value !== false && value !== '';
35
+ if (value == null) return false;
36
+ const na = Number(value);
37
+ const nb = Number(filter.value);
38
+ // `Number('')` is 0, which would make an empty string compare as a number — hence
39
+ // the explicit emptiness checks rather than `!Number.isNaN` alone.
40
+ const numeric = value !== '' && value !== true && value !== false
41
+ && filter.value !== '' && filter.value != null
42
+ && !Number.isNaN(na) && !Number.isNaN(nb);
43
+ const x = numeric ? na : String(value).toLowerCase();
44
+ const y = numeric ? nb : String(filter.value).toLowerCase();
45
+ switch (filter.op) {
46
+ case '!=': return x !== y;
47
+ case '<': return x < y;
48
+ case '<=': return x <= y;
49
+ case '>': return x > y;
50
+ case '>=': return x >= y;
51
+ default: return x === y;
52
+ }
53
+ }
54
+
55
+ /** Does a node satisfy every filter? `mergedTags` when the caller already computed them. */
56
+ export function matchTagFilters(node, filters, mergedTags = null) {
57
+ const props = tagProps(node, mergedTags);
58
+ return (filters || []).every((f) => matchesFilter(props[f.key], f));
59
+ }