@bendyline/gezel-retrieval 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +44 -2
- package/dist/index.d.ts +254 -0
- package/dist/index.js +849 -0
- package/dist/sql-BFQtPDha.d.ts +24 -0
- package/dist/sqlite-wasm.d.ts +69 -0
- package/dist/sqlite-wasm.js +68 -0
- package/package.json +68 -3
package/dist/index.js
ADDED
|
@@ -0,0 +1,849 @@
|
|
|
1
|
+
// src/chunks.ts
|
|
2
|
+
import {
|
|
3
|
+
MARKDOWN_CHUNKS_2,
|
|
4
|
+
chunkMarkdownProfile
|
|
5
|
+
} from "@bendyline/gezel-knowledge/browser";
|
|
6
|
+
function markdownChunksByCharacters() {
|
|
7
|
+
return { id: "markdown-chunks@1", options: { unit: "chars", target: 4e3, overlap: 400 } };
|
|
8
|
+
}
|
|
9
|
+
function markdownChunksByTokens(countTokens, tokenizer) {
|
|
10
|
+
return {
|
|
11
|
+
id: `${MARKDOWN_CHUNKS_2.id}/${tokenizer}`,
|
|
12
|
+
options: {
|
|
13
|
+
unit: "tokens",
|
|
14
|
+
target: MARKDOWN_CHUNKS_2.target,
|
|
15
|
+
overlap: MARKDOWN_CHUNKS_2.overlap,
|
|
16
|
+
countTokens
|
|
17
|
+
}
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
var MATCH_OPEN = "";
|
|
21
|
+
var MATCH_CLOSE = "";
|
|
22
|
+
function normalizeBody(body) {
|
|
23
|
+
const lf = body.replace(/\r\n?/g, "\n").replaceAll(MATCH_OPEN, "").replaceAll(MATCH_CLOSE, "");
|
|
24
|
+
return lf.toWellFormed?.() ?? lf;
|
|
25
|
+
}
|
|
26
|
+
function codePointLength(text, from = 0, to = text.length) {
|
|
27
|
+
let count = 0;
|
|
28
|
+
for (let index = from; index < to; index++) {
|
|
29
|
+
const unit = text.charCodeAt(index);
|
|
30
|
+
if (unit >= 55296 && unit <= 56319) index++;
|
|
31
|
+
count++;
|
|
32
|
+
}
|
|
33
|
+
return count;
|
|
34
|
+
}
|
|
35
|
+
function placeChunks(body, chunking) {
|
|
36
|
+
const chunks = chunkMarkdownProfile(body, chunking.options);
|
|
37
|
+
const lineStarts = [0];
|
|
38
|
+
for (let index = body.indexOf("\n"); index !== -1; index = body.indexOf("\n", index + 1)) {
|
|
39
|
+
lineStarts.push(index + 1);
|
|
40
|
+
}
|
|
41
|
+
const astral = /[\uD800-\uDBFF]/.test(body);
|
|
42
|
+
const pointsAtLine = [];
|
|
43
|
+
if (astral) {
|
|
44
|
+
let points = 0;
|
|
45
|
+
let previous = 0;
|
|
46
|
+
for (const start of lineStarts) {
|
|
47
|
+
points += codePointLength(body, previous, start);
|
|
48
|
+
pointsAtLine.push(points);
|
|
49
|
+
previous = start;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return chunks.map((chunk) => {
|
|
53
|
+
const line = chunk.lineStart - 1;
|
|
54
|
+
const from = lineStarts[line] ?? 0;
|
|
55
|
+
const until = lineStarts[chunk.lineEnd] ?? body.length;
|
|
56
|
+
const at = body.indexOf(chunk.text, from);
|
|
57
|
+
const sliced = at !== -1 && at + chunk.text.length <= until;
|
|
58
|
+
let start = null;
|
|
59
|
+
let length = null;
|
|
60
|
+
if (sliced && chunk.text.length > 0) {
|
|
61
|
+
start = astral ? (pointsAtLine[line] ?? 0) + codePointLength(body, from, at) : at;
|
|
62
|
+
length = astral ? codePointLength(chunk.text) : chunk.text.length;
|
|
63
|
+
}
|
|
64
|
+
return {
|
|
65
|
+
kind: chunk.kind,
|
|
66
|
+
headingPath: chunk.headingPath,
|
|
67
|
+
lineStart: chunk.lineStart,
|
|
68
|
+
lineEnd: chunk.lineEnd,
|
|
69
|
+
text: chunk.text,
|
|
70
|
+
start,
|
|
71
|
+
length
|
|
72
|
+
};
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// src/live-index.ts
|
|
77
|
+
import { asymmetricTopK, selectiveFtsQuery } from "@bendyline/gezel-knowledge/browser";
|
|
78
|
+
import { sha256Hex } from "@bendyline/gezk";
|
|
79
|
+
|
|
80
|
+
// src/schema.ts
|
|
81
|
+
var LIVE_INDEX_SCHEMA = 2;
|
|
82
|
+
var LIVE_CHUNK_FTS_WEIGHTS = { title: 5, heading: 2, text: 1 };
|
|
83
|
+
var DDL = `
|
|
84
|
+
CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;
|
|
85
|
+
|
|
86
|
+
CREATE TABLE documents(
|
|
87
|
+
id INTEGER PRIMARY KEY,
|
|
88
|
+
path TEXT NOT NULL UNIQUE,
|
|
89
|
+
version TEXT NOT NULL,
|
|
90
|
+
title TEXT NOT NULL,
|
|
91
|
+
body TEXT NOT NULL,
|
|
92
|
+
payload BLOB,
|
|
93
|
+
chunked_with TEXT NOT NULL,
|
|
94
|
+
updated_at INTEGER NOT NULL
|
|
95
|
+
);
|
|
96
|
+
|
|
97
|
+
CREATE TABLE chunks(
|
|
98
|
+
id INTEGER PRIMARY KEY,
|
|
99
|
+
document_id INTEGER NOT NULL,
|
|
100
|
+
ordinal INTEGER NOT NULL,
|
|
101
|
+
kind TEXT NOT NULL,
|
|
102
|
+
heading_path TEXT NOT NULL,
|
|
103
|
+
heading_text TEXT NOT NULL,
|
|
104
|
+
line_start INTEGER NOT NULL,
|
|
105
|
+
line_end INTEGER NOT NULL,
|
|
106
|
+
text_start INTEGER,
|
|
107
|
+
text_length INTEGER,
|
|
108
|
+
text_inline TEXT,
|
|
109
|
+
embed_hash TEXT
|
|
110
|
+
);
|
|
111
|
+
CREATE INDEX chunks_document ON chunks(document_id, ordinal);
|
|
112
|
+
CREATE INDEX chunks_embed_hash ON chunks(embed_hash);
|
|
113
|
+
|
|
114
|
+
CREATE TABLE embeddings(
|
|
115
|
+
id INTEGER PRIMARY KEY,
|
|
116
|
+
hash TEXT NOT NULL UNIQUE,
|
|
117
|
+
bits BLOB NOT NULL,
|
|
118
|
+
int8 BLOB NOT NULL
|
|
119
|
+
);
|
|
120
|
+
|
|
121
|
+
CREATE VIEW chunk_content AS
|
|
122
|
+
SELECT c.id AS id, c.document_id AS document_id, d.title AS title,
|
|
123
|
+
c.heading_text AS heading_text,
|
|
124
|
+
COALESCE(c.text_inline, substr(d.body, c.text_start + 1, c.text_length)) AS text
|
|
125
|
+
FROM chunks c JOIN documents d ON d.id = c.document_id;
|
|
126
|
+
|
|
127
|
+
CREATE VIRTUAL TABLE fts_chunks USING fts5(
|
|
128
|
+
title, heading_text, text,
|
|
129
|
+
content = 'chunk_content', content_rowid = 'id',
|
|
130
|
+
tokenize = 'unicode61 remove_diacritics 2',
|
|
131
|
+
prefix = '2 3'
|
|
132
|
+
);
|
|
133
|
+
|
|
134
|
+
CREATE VIRTUAL TABLE fts_chunks_vocab USING fts5vocab(fts_chunks, row);
|
|
135
|
+
`;
|
|
136
|
+
function createLiveIndexSchema(db) {
|
|
137
|
+
db.exec(DDL);
|
|
138
|
+
}
|
|
139
|
+
function dropLiveIndexSchema(db) {
|
|
140
|
+
db.exec(`
|
|
141
|
+
DROP TABLE IF EXISTS fts_chunks_vocab;
|
|
142
|
+
DROP TABLE IF EXISTS fts_chunks;
|
|
143
|
+
DROP VIEW IF EXISTS chunk_content;
|
|
144
|
+
DROP TABLE IF EXISTS embeddings;
|
|
145
|
+
DROP TABLE IF EXISTS chunks;
|
|
146
|
+
DROP TABLE IF EXISTS documents;
|
|
147
|
+
DROP TABLE IF EXISTS meta;
|
|
148
|
+
`);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// src/vectors.ts
|
|
152
|
+
import {
|
|
153
|
+
KNOWLEDGE_VECTOR_FLOORS,
|
|
154
|
+
MARKDOWN_CHUNKS_2 as MARKDOWN_CHUNKS_22,
|
|
155
|
+
passageEmbedInput
|
|
156
|
+
} from "@bendyline/gezel-knowledge/browser";
|
|
157
|
+
import {
|
|
158
|
+
centerVector,
|
|
159
|
+
embeddingProfileCenter,
|
|
160
|
+
profileUnitVector,
|
|
161
|
+
quantizeBinaryForProfile,
|
|
162
|
+
quantizeInt8,
|
|
163
|
+
rerankScore
|
|
164
|
+
} from "@bendyline/gezk";
|
|
165
|
+
function liveIndexEmbedding(profile, countTokens) {
|
|
166
|
+
return {
|
|
167
|
+
chunking: markdownChunksByTokens(countTokens, profile.id),
|
|
168
|
+
embedding: { profile, countTokens }
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
function embedInputFor(embedding, title, headingPath, text) {
|
|
172
|
+
return passageEmbedInput({
|
|
173
|
+
passageInstruction: embedding.profile.passageInstruction,
|
|
174
|
+
title,
|
|
175
|
+
headingPath,
|
|
176
|
+
text,
|
|
177
|
+
headerMaxTokens: MARKDOWN_CHUNKS_22.contextHeader.max,
|
|
178
|
+
countTokens: embedding.countTokens
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
function quantizeEmbedding(profile, vector) {
|
|
182
|
+
const unit = profileUnitVector(profile, vector);
|
|
183
|
+
const int8 = quantizeInt8(unit);
|
|
184
|
+
return {
|
|
185
|
+
bits: quantizeBinaryForProfile(profile, unit),
|
|
186
|
+
int8: new Uint8Array(int8.buffer, int8.byteOffset, int8.byteLength)
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
function prepareQuery(profile, vector) {
|
|
190
|
+
const unit = profileUnitVector(profile, vector);
|
|
191
|
+
const center = embeddingProfileCenter(profile);
|
|
192
|
+
return { unit, scan: center ? centerVector(unit, center) : unit };
|
|
193
|
+
}
|
|
194
|
+
function similarity(unit, int8) {
|
|
195
|
+
return rerankScore(unit, new Int8Array(int8.buffer, int8.byteOffset, int8.byteLength));
|
|
196
|
+
}
|
|
197
|
+
function measuredFloor(profile) {
|
|
198
|
+
return KNOWLEDGE_VECTOR_FLOORS[profile.id] ?? null;
|
|
199
|
+
}
|
|
200
|
+
function excerpt(text, max = 240) {
|
|
201
|
+
const flat = text.replace(/\s+/g, " ").trim();
|
|
202
|
+
if (flat.length <= max) return flat;
|
|
203
|
+
const cut = flat.lastIndexOf(" ", max);
|
|
204
|
+
return `${flat.slice(0, cut > max / 2 ? cut : max)}\u2026`;
|
|
205
|
+
}
|
|
206
|
+
var LIVE_RRF_K = 60;
|
|
207
|
+
var LIVE_HYBRID_WEIGHTS = { lexical: 1, vector: 1 };
|
|
208
|
+
|
|
209
|
+
// src/live-index.ts
|
|
210
|
+
var SEARCH_LIMIT_MAX = 200;
|
|
211
|
+
var VECTOR_CANDIDATES_PER_HIT = 16;
|
|
212
|
+
var VECTOR_CANDIDATES_MIN = 256;
|
|
213
|
+
var SNIPPET_TOKENS = 24;
|
|
214
|
+
var TERM_CACHE_MAX = 512;
|
|
215
|
+
var BM25 = `bm25(fts_chunks, ${LIVE_CHUNK_FTS_WEIGHTS.title}.0, ${LIVE_CHUNK_FTS_WEIGHTS.heading}.0, ${LIVE_CHUNK_FTS_WEIGHTS.text}.0)`;
|
|
216
|
+
function clamp(value, fallback, max) {
|
|
217
|
+
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
218
|
+
return Math.min(max, Math.max(1, Math.floor(value)));
|
|
219
|
+
}
|
|
220
|
+
function requireText(value, name, allowEmpty) {
|
|
221
|
+
if (typeof value !== "string" || !allowEmpty && value.length === 0) {
|
|
222
|
+
throw new TypeError(`${name} must be a ${allowEmpty ? "" : "non-empty "}string`);
|
|
223
|
+
}
|
|
224
|
+
return value;
|
|
225
|
+
}
|
|
226
|
+
function indexedTerm(phrase) {
|
|
227
|
+
return phrase.slice(1, -1).replace(/""/g, '"').normalize("NFKD").replace(/\p{M}/gu, "").toLowerCase();
|
|
228
|
+
}
|
|
229
|
+
function withPrefix(query, match) {
|
|
230
|
+
const last = /[\p{L}\p{N}_]+$/u.exec(query.normalize("NFKC"))?.[0];
|
|
231
|
+
if (!last) return match;
|
|
232
|
+
const quoted = `"${last.replace(/"/g, '""')}"`;
|
|
233
|
+
return match.split(" OR ").map((phrase) => phrase === quoted ? `${quoted}*` : phrase).join(" OR ");
|
|
234
|
+
}
|
|
235
|
+
function parseSnippet(marked) {
|
|
236
|
+
let snippet = "";
|
|
237
|
+
const ranges = [];
|
|
238
|
+
let open = -1;
|
|
239
|
+
for (const character of marked) {
|
|
240
|
+
if (character === MATCH_OPEN) open = snippet.length;
|
|
241
|
+
else if (character === MATCH_CLOSE) {
|
|
242
|
+
if (open !== -1 && snippet.length > open) ranges.push([open, snippet.length]);
|
|
243
|
+
open = -1;
|
|
244
|
+
} else snippet += character;
|
|
245
|
+
}
|
|
246
|
+
return { snippet, ranges };
|
|
247
|
+
}
|
|
248
|
+
var LiveIndex = class _LiveIndex {
|
|
249
|
+
reset;
|
|
250
|
+
#db;
|
|
251
|
+
#chunking;
|
|
252
|
+
#embedding;
|
|
253
|
+
/** What `documents.chunked_with` records: the chunking, and the profile hashes are for. */
|
|
254
|
+
#identity;
|
|
255
|
+
#now;
|
|
256
|
+
/** Every stored vector's sign bits in one block, and the hash of each row. */
|
|
257
|
+
#bitIndex = null;
|
|
258
|
+
#statements = /* @__PURE__ */ new Map();
|
|
259
|
+
#termRows = /* @__PURE__ */ new Map();
|
|
260
|
+
#chunkCount = null;
|
|
261
|
+
#savepoints = 0;
|
|
262
|
+
constructor(db, options, reset) {
|
|
263
|
+
this.#db = db;
|
|
264
|
+
this.#chunking = options.chunking;
|
|
265
|
+
this.#embedding = options.embedding ?? null;
|
|
266
|
+
this.#identity = identityOf(options);
|
|
267
|
+
this.#now = options.now ?? Date.now;
|
|
268
|
+
this.reset = reset;
|
|
269
|
+
}
|
|
270
|
+
/** Open (or create) the index in `db`, emptying it when its schema or the app's format changed. */
|
|
271
|
+
static open(db, options) {
|
|
272
|
+
requireText(options.appVersion, "appVersion", false);
|
|
273
|
+
requireText(options.chunking?.id, "chunking.id", false);
|
|
274
|
+
const hasMeta = db.prepare("SELECT 1 AS present FROM sqlite_master WHERE type = 'table' AND name = 'meta'").get() !== void 0;
|
|
275
|
+
const meta = (key) => hasMeta ? db.prepare("SELECT value FROM meta WHERE key = ?").get(key)?.value : void 0;
|
|
276
|
+
let reset = null;
|
|
277
|
+
if (!hasMeta) reset = "created";
|
|
278
|
+
else if (meta("schema") !== String(LIVE_INDEX_SCHEMA)) reset = "schema";
|
|
279
|
+
else if (meta("app") !== options.appVersion) reset = "app";
|
|
280
|
+
db.exec("BEGIN");
|
|
281
|
+
try {
|
|
282
|
+
if (reset === "created" || reset === "schema") {
|
|
283
|
+
dropLiveIndexSchema(db);
|
|
284
|
+
createLiveIndexSchema(db);
|
|
285
|
+
} else if (reset === "app") {
|
|
286
|
+
db.exec(
|
|
287
|
+
"INSERT INTO fts_chunks(fts_chunks) VALUES('delete-all'); DELETE FROM chunks; DELETE FROM documents;"
|
|
288
|
+
);
|
|
289
|
+
}
|
|
290
|
+
const profile = options.embedding?.profile.id ?? "";
|
|
291
|
+
if (reset === null && meta("embedding") !== profile) db.exec("DELETE FROM embeddings");
|
|
292
|
+
const write = db.prepare("INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)");
|
|
293
|
+
write.run("schema", String(LIVE_INDEX_SCHEMA));
|
|
294
|
+
write.run("app", options.appVersion);
|
|
295
|
+
write.run("chunking", identityOf(options));
|
|
296
|
+
write.run("embedding", profile);
|
|
297
|
+
db.exec("COMMIT");
|
|
298
|
+
} catch (error) {
|
|
299
|
+
db.exec("ROLLBACK");
|
|
300
|
+
throw error;
|
|
301
|
+
}
|
|
302
|
+
return new _LiveIndex(db, options, reset);
|
|
303
|
+
}
|
|
304
|
+
/** Run `fn` atomically; writes inside it are undone if it throws. Nests. */
|
|
305
|
+
transaction(fn) {
|
|
306
|
+
const name = `live_${++this.#savepoints}`;
|
|
307
|
+
this.#db.exec(`SAVEPOINT ${name}`);
|
|
308
|
+
try {
|
|
309
|
+
const result = fn();
|
|
310
|
+
this.#db.exec(`RELEASE ${name}`);
|
|
311
|
+
return result;
|
|
312
|
+
} catch (error) {
|
|
313
|
+
this.#db.exec(`ROLLBACK TO ${name}`);
|
|
314
|
+
this.#db.exec(`RELEASE ${name}`);
|
|
315
|
+
this.#changed();
|
|
316
|
+
throw error;
|
|
317
|
+
} finally {
|
|
318
|
+
this.#savepoints--;
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
/** Add or replace a document. Returns how many chunks it was indexed as. */
|
|
322
|
+
upsert(input) {
|
|
323
|
+
const path = requireText(input.path, "path", false);
|
|
324
|
+
const version = requireText(input.version, "version", true);
|
|
325
|
+
const title = requireText(input.title, "title", true);
|
|
326
|
+
const body = normalizeBody(requireText(input.body, "body", true));
|
|
327
|
+
const payload = input.payload ?? null;
|
|
328
|
+
if (payload !== null && !(payload instanceof Uint8Array)) {
|
|
329
|
+
throw new TypeError("payload must be a Uint8Array or null");
|
|
330
|
+
}
|
|
331
|
+
const placed = placeChunks(body, this.#chunking);
|
|
332
|
+
return this.transaction(() => {
|
|
333
|
+
const existing = this.#statement("SELECT id FROM documents WHERE path = ?").get(path);
|
|
334
|
+
let id;
|
|
335
|
+
if (existing) {
|
|
336
|
+
id = Number(existing.id);
|
|
337
|
+
this.#unindex(id);
|
|
338
|
+
this.#statement(
|
|
339
|
+
`UPDATE documents SET version = ?, title = ?, body = ?, payload = ?, chunked_with = ?,
|
|
340
|
+
updated_at = ? WHERE id = ?`
|
|
341
|
+
).run(version, title, body, payload, this.#identity, this.#now(), id);
|
|
342
|
+
} else {
|
|
343
|
+
const row = this.#statement(
|
|
344
|
+
`INSERT INTO documents (path, version, title, body, payload, chunked_with, updated_at)
|
|
345
|
+
VALUES (?, ?, ?, ?, ?, ?, ?) RETURNING id`
|
|
346
|
+
).get(path, version, title, body, payload, this.#identity, this.#now());
|
|
347
|
+
id = Number(row?.id);
|
|
348
|
+
}
|
|
349
|
+
const chunks = this.#index(id, title, placed);
|
|
350
|
+
this.#changed();
|
|
351
|
+
return { chunks };
|
|
352
|
+
});
|
|
353
|
+
}
|
|
354
|
+
/** Remove a document; false when it was not indexed. */
|
|
355
|
+
remove(path) {
|
|
356
|
+
return this.transaction(() => {
|
|
357
|
+
const existing = this.#statement("SELECT id FROM documents WHERE path = ?").get(path);
|
|
358
|
+
if (!existing) return false;
|
|
359
|
+
const id = Number(existing.id);
|
|
360
|
+
this.#unindex(id);
|
|
361
|
+
this.#statement("DELETE FROM documents WHERE id = ?").run(id);
|
|
362
|
+
this.#changed();
|
|
363
|
+
return true;
|
|
364
|
+
});
|
|
365
|
+
}
|
|
366
|
+
/** Remove every document, and every stored vector. */
|
|
367
|
+
clear() {
|
|
368
|
+
this.transaction(() => {
|
|
369
|
+
this.#db.exec(
|
|
370
|
+
"INSERT INTO fts_chunks(fts_chunks) VALUES('delete-all'); DELETE FROM chunks; DELETE FROM documents; DELETE FROM embeddings;"
|
|
371
|
+
);
|
|
372
|
+
this.#changed();
|
|
373
|
+
this.#bitIndex = null;
|
|
374
|
+
});
|
|
375
|
+
}
|
|
376
|
+
/** Every document's path and version, for diffing against the app's own listing. */
|
|
377
|
+
manifest() {
|
|
378
|
+
return this.#statement("SELECT path, version FROM documents ORDER BY path").all().map((row) => ({ path: String(row.path), version: String(row.version) }));
|
|
379
|
+
}
|
|
380
|
+
get(path) {
|
|
381
|
+
const row = this.#statement(
|
|
382
|
+
"SELECT path, version, title, payload, updated_at FROM documents WHERE path = ?"
|
|
383
|
+
).get(path);
|
|
384
|
+
return row ? toDocument(row) : null;
|
|
385
|
+
}
|
|
386
|
+
/** The body as indexed (LF line endings), or null. */
|
|
387
|
+
body(path) {
|
|
388
|
+
const row = this.#statement("SELECT body FROM documents WHERE path = ?").get(path);
|
|
389
|
+
return row ? String(row.body) : null;
|
|
390
|
+
}
|
|
391
|
+
/** Every document in batches, without holding them all in memory. */
|
|
392
|
+
*documents(batchSize = 256) {
|
|
393
|
+
const size = clamp(batchSize, 256, 1e4);
|
|
394
|
+
let after = 0;
|
|
395
|
+
for (; ; ) {
|
|
396
|
+
const rows = this.#statement(
|
|
397
|
+
`SELECT id, path, version, title, payload, updated_at FROM documents
|
|
398
|
+
WHERE id > ? ORDER BY id LIMIT ?`
|
|
399
|
+
).all(after, size);
|
|
400
|
+
for (const row of rows) yield toDocument(row);
|
|
401
|
+
if (rows.length < size) return;
|
|
402
|
+
after = Number(rows[rows.length - 1]?.id);
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
stats() {
|
|
406
|
+
const row = this.#statement(
|
|
407
|
+
`SELECT (SELECT COUNT(*) FROM documents) AS documents,
|
|
408
|
+
(SELECT COUNT(*) FROM chunks) AS chunks,
|
|
409
|
+
(SELECT COUNT(*) FROM documents WHERE chunked_with != ?) AS stale`
|
|
410
|
+
).get(this.#identity);
|
|
411
|
+
return {
|
|
412
|
+
documents: Number(row?.documents ?? 0),
|
|
413
|
+
chunks: Number(row?.chunks ?? 0),
|
|
414
|
+
stale: Number(row?.stale ?? 0)
|
|
415
|
+
};
|
|
416
|
+
}
|
|
417
|
+
/**
|
|
418
|
+
* Re-chunk up to `limit` documents indexed under an earlier chunking, from
|
|
419
|
+
* their stored bodies. Returns how many it did; 0 when none are left.
|
|
420
|
+
* They stay searchable by their old chunks until then.
|
|
421
|
+
*/
|
|
422
|
+
rechunkStale(limit = 100) {
|
|
423
|
+
const rows = this.#statement(
|
|
424
|
+
"SELECT id, title, body FROM documents WHERE chunked_with != ? LIMIT ?"
|
|
425
|
+
).all(this.#identity, clamp(limit, 100, 1e5));
|
|
426
|
+
if (rows.length === 0) return 0;
|
|
427
|
+
this.transaction(() => {
|
|
428
|
+
for (const row of rows) {
|
|
429
|
+
const id = Number(row.id);
|
|
430
|
+
this.#unindex(id);
|
|
431
|
+
this.#index(id, String(row.title), placeChunks(String(row.body), this.#chunking));
|
|
432
|
+
this.#statement("UPDATE documents SET chunked_with = ? WHERE id = ?").run(
|
|
433
|
+
this.#identity,
|
|
434
|
+
id
|
|
435
|
+
);
|
|
436
|
+
}
|
|
437
|
+
this.#changed();
|
|
438
|
+
});
|
|
439
|
+
return rows.length;
|
|
440
|
+
}
|
|
441
|
+
/**
|
|
442
|
+
* Best-matching passages for a query, by BM25 over titles, headings and
|
|
443
|
+
* text. The query is plain words, not FTS syntax; words in more than half
|
|
444
|
+
* the passages are dropped, since they cost a full ranking and cannot
|
|
445
|
+
* change its order.
|
|
446
|
+
*/
|
|
447
|
+
search(query, options = {}) {
|
|
448
|
+
const limit = clamp(options.limit, 20, SEARCH_LIMIT_MAX);
|
|
449
|
+
const perDocument = clamp(options.perDocument, 3, SEARCH_LIMIT_MAX);
|
|
450
|
+
const rows = this.#rows();
|
|
451
|
+
if (rows === 0) return [];
|
|
452
|
+
let match = selectiveFtsQuery(
|
|
453
|
+
requireText(query, "query", true),
|
|
454
|
+
rows,
|
|
455
|
+
(phrase) => this.#termRowCount(phrase)
|
|
456
|
+
);
|
|
457
|
+
if (!match) return [];
|
|
458
|
+
if (options.prefix) match = withPrefix(query, match);
|
|
459
|
+
const candidates = this.#statement(
|
|
460
|
+
`SELECT f.rowid AS id, d.path AS path, ${BM25} AS score
|
|
461
|
+
FROM fts_chunks f JOIN chunks c ON c.id = f.rowid JOIN documents d ON d.id = c.document_id
|
|
462
|
+
WHERE fts_chunks MATCH ? ORDER BY score LIMIT ?`
|
|
463
|
+
).all(match, Math.max(200, limit * perDocument * 4));
|
|
464
|
+
const picked = [];
|
|
465
|
+
const perPath = /* @__PURE__ */ new Map();
|
|
466
|
+
for (const row of candidates) {
|
|
467
|
+
const path = String(row.path);
|
|
468
|
+
if (options.exclude?.has(path)) continue;
|
|
469
|
+
const taken = perPath.get(path) ?? 0;
|
|
470
|
+
if (taken >= perDocument) continue;
|
|
471
|
+
perPath.set(path, taken + 1);
|
|
472
|
+
picked.push({ id: Number(row.id), score: -Number(row.score) });
|
|
473
|
+
if (picked.length >= limit) break;
|
|
474
|
+
}
|
|
475
|
+
if (picked.length === 0) return [];
|
|
476
|
+
const ids = JSON.stringify(picked.map((hit) => hit.id));
|
|
477
|
+
const snippets = /* @__PURE__ */ new Map();
|
|
478
|
+
for (const row of this.#statement(
|
|
479
|
+
`SELECT f.rowid AS id, snippet(fts_chunks, 2, char(2), char(3), '\u2026', ${SNIPPET_TOKENS}) AS snippet
|
|
480
|
+
FROM fts_chunks f WHERE fts_chunks MATCH ? AND f.rowid IN (SELECT value FROM json_each(?))`
|
|
481
|
+
).all(match, ids)) {
|
|
482
|
+
snippets.set(Number(row.id), String(row.snippet ?? ""));
|
|
483
|
+
}
|
|
484
|
+
const details = /* @__PURE__ */ new Map();
|
|
485
|
+
for (const row of this.#statement(
|
|
486
|
+
`SELECT c.id AS id, d.path AS path, d.title AS title, c.heading_path AS heading_path,
|
|
487
|
+
c.line_start AS line_start, c.line_end AS line_end
|
|
488
|
+
FROM chunks c JOIN documents d ON d.id = c.document_id
|
|
489
|
+
WHERE c.id IN (SELECT value FROM json_each(?))`
|
|
490
|
+
).all(ids)) {
|
|
491
|
+
details.set(Number(row.id), row);
|
|
492
|
+
}
|
|
493
|
+
return picked.flatMap(({ id, score }) => {
|
|
494
|
+
const row = details.get(id);
|
|
495
|
+
if (!row) return [];
|
|
496
|
+
const { snippet, ranges } = parseSnippet(snippets.get(id) ?? "");
|
|
497
|
+
return [
|
|
498
|
+
{
|
|
499
|
+
path: String(row.path),
|
|
500
|
+
title: String(row.title),
|
|
501
|
+
headingPath: JSON.parse(String(row.heading_path)),
|
|
502
|
+
lineStart: Number(row.line_start),
|
|
503
|
+
lineEnd: Number(row.line_end),
|
|
504
|
+
snippet,
|
|
505
|
+
ranges,
|
|
506
|
+
score,
|
|
507
|
+
chunk: id
|
|
508
|
+
}
|
|
509
|
+
];
|
|
510
|
+
});
|
|
511
|
+
}
|
|
512
|
+
/**
|
|
513
|
+
* Passages still waiting for a vector, most recently changed documents
|
|
514
|
+
* first. Embed each `input` with the profile's passage model and hand the
|
|
515
|
+
* vectors back through `storeEmbeddings`. Empty without an embedding
|
|
516
|
+
* profile, or when every passage has one.
|
|
517
|
+
*/
|
|
518
|
+
/**
|
|
519
|
+
* The full text of these passages, by chunk id (a hit's `chunk`), for an
|
|
520
|
+
* app that quotes passages to a model. Ids that are gone are absent.
|
|
521
|
+
*/
|
|
522
|
+
passageTexts(chunks) {
|
|
523
|
+
const ids = [...new Set(chunks)].filter((id) => Number.isSafeInteger(id)).slice(0, 1e3);
|
|
524
|
+
const out = /* @__PURE__ */ new Map();
|
|
525
|
+
if (ids.length === 0) return out;
|
|
526
|
+
for (const row of this.#statement(
|
|
527
|
+
"SELECT id, text FROM chunk_content WHERE id IN (SELECT value FROM json_each(?))"
|
|
528
|
+
).all(JSON.stringify(ids))) {
|
|
529
|
+
out.set(Number(row.id), String(row.text));
|
|
530
|
+
}
|
|
531
|
+
return out;
|
|
532
|
+
}
|
|
533
|
+
pendingEmbeddings(limit = 64) {
|
|
534
|
+
const embedding = this.#embedding;
|
|
535
|
+
if (!embedding) return [];
|
|
536
|
+
const rows = this.#statement(
|
|
537
|
+
`SELECT c.embed_hash AS hash, MIN(c.id) AS id, MAX(d.updated_at) AS updated
|
|
538
|
+
FROM chunks c JOIN documents d ON d.id = c.document_id
|
|
539
|
+
LEFT JOIN embeddings e ON e.hash = c.embed_hash
|
|
540
|
+
WHERE c.embed_hash IS NOT NULL AND e.hash IS NULL
|
|
541
|
+
GROUP BY c.embed_hash ORDER BY updated DESC, id LIMIT ?`
|
|
542
|
+
).all(clamp(limit, 64, 4096));
|
|
543
|
+
if (rows.length === 0) return [];
|
|
544
|
+
const content = /* @__PURE__ */ new Map();
|
|
545
|
+
for (const row of this.#statement(
|
|
546
|
+
`SELECT v.id AS id, v.title AS title, c.heading_path AS heading_path, v.text AS text
|
|
547
|
+
FROM chunk_content v JOIN chunks c ON c.id = v.id
|
|
548
|
+
WHERE v.id IN (SELECT value FROM json_each(?))`
|
|
549
|
+
).all(JSON.stringify(rows.map((row2) => Number(row2.id))))) {
|
|
550
|
+
content.set(Number(row.id), row);
|
|
551
|
+
}
|
|
552
|
+
return rows.flatMap((row) => {
|
|
553
|
+
const chunk = content.get(Number(row.id));
|
|
554
|
+
if (!chunk) return [];
|
|
555
|
+
return [
|
|
556
|
+
{
|
|
557
|
+
hash: String(row.hash),
|
|
558
|
+
input: embedInputFor(
|
|
559
|
+
embedding,
|
|
560
|
+
String(chunk.title),
|
|
561
|
+
JSON.parse(String(chunk.heading_path)),
|
|
562
|
+
String(chunk.text)
|
|
563
|
+
)
|
|
564
|
+
}
|
|
565
|
+
];
|
|
566
|
+
});
|
|
567
|
+
}
|
|
568
|
+
/** Store passage vectors, raw from the model, against the hashes `pendingEmbeddings` gave. */
|
|
569
|
+
storeEmbeddings(entries) {
|
|
570
|
+
const embedding = this.#embedding;
|
|
571
|
+
if (!embedding) throw new Error("This live index has no embedding profile");
|
|
572
|
+
const quantized = entries.map(({ hash, vector }) => {
|
|
573
|
+
if (typeof hash !== "string" || !/^[0-9a-f]{64}$/.test(hash)) {
|
|
574
|
+
throw new TypeError("embedding hash must be a SHA-256 hex string");
|
|
575
|
+
}
|
|
576
|
+
return { hash, ...quantizeEmbedding(embedding.profile, vector) };
|
|
577
|
+
});
|
|
578
|
+
this.transaction(() => {
|
|
579
|
+
const insert = this.#statement(
|
|
580
|
+
"INSERT OR REPLACE INTO embeddings(hash, bits, int8) VALUES (?, ?, ?)"
|
|
581
|
+
);
|
|
582
|
+
for (const { hash, bits, int8 } of quantized) insert.run(hash, bits, int8);
|
|
583
|
+
this.#bitIndex = null;
|
|
584
|
+
});
|
|
585
|
+
}
|
|
586
|
+
embeddingStats() {
|
|
587
|
+
const row = this.#statement(
|
|
588
|
+
`SELECT (SELECT COUNT(DISTINCT embed_hash) FROM chunks WHERE embed_hash IS NOT NULL) AS total,
|
|
589
|
+
(SELECT COUNT(DISTINCT c.embed_hash) FROM chunks c
|
|
590
|
+
JOIN embeddings e ON e.hash = c.embed_hash) AS embedded`
|
|
591
|
+
).get();
|
|
592
|
+
return { total: Number(row?.total ?? 0), embedded: Number(row?.embedded ?? 0) };
|
|
593
|
+
}
|
|
594
|
+
/** Drop vectors whose text no passage holds any more; returns how many. */
|
|
595
|
+
pruneEmbeddings() {
|
|
596
|
+
return this.transaction(() => {
|
|
597
|
+
const result = this.#statement(
|
|
598
|
+
"DELETE FROM embeddings WHERE hash NOT IN (SELECT embed_hash FROM chunks WHERE embed_hash IS NOT NULL)"
|
|
599
|
+
).run();
|
|
600
|
+
this.#bitIndex = null;
|
|
601
|
+
return Number(result.changes);
|
|
602
|
+
});
|
|
603
|
+
}
|
|
604
|
+
/**
|
|
605
|
+
* Passages nearest a query embedding (raw from the profile's query model,
|
|
606
|
+
* query instruction applied): the catalogs' two-stage scan, sign bits then
|
|
607
|
+
* int8 rescoring, kept only above the profile's measured cosine floor.
|
|
608
|
+
*/
|
|
609
|
+
semanticSearch(vector, options = {}) {
|
|
610
|
+
const embedding = this.#embedding;
|
|
611
|
+
if (!embedding) return [];
|
|
612
|
+
const limit = clamp(options.limit, 20, SEARCH_LIMIT_MAX);
|
|
613
|
+
const perDocument = clamp(options.perDocument, 3, SEARCH_LIMIT_MAX);
|
|
614
|
+
const floor = options.floor === void 0 ? measuredFloor(embedding.profile) : options.floor;
|
|
615
|
+
const index = this.#bits();
|
|
616
|
+
if (index.rows === 0) return [];
|
|
617
|
+
const { unit, scan } = prepareQuery(embedding.profile, vector);
|
|
618
|
+
const candidates = asymmetricTopK(
|
|
619
|
+
index,
|
|
620
|
+
scan,
|
|
621
|
+
Math.max(VECTOR_CANDIDATES_MIN, limit * perDocument * VECTOR_CANDIDATES_PER_HIT)
|
|
622
|
+
);
|
|
623
|
+
const hashes = candidates.map((candidate) => index.hashes[candidate.chunkId - 1]);
|
|
624
|
+
const scored = /* @__PURE__ */ new Map();
|
|
625
|
+
for (const row of this.#statement(
|
|
626
|
+
"SELECT hash, int8 FROM embeddings WHERE hash IN (SELECT value FROM json_each(?))"
|
|
627
|
+
).all(JSON.stringify(hashes))) {
|
|
628
|
+
const score = similarity(unit, row.int8);
|
|
629
|
+
if (floor === null || score >= floor) scored.set(String(row.hash), score);
|
|
630
|
+
}
|
|
631
|
+
if (scored.size === 0) return [];
|
|
632
|
+
const chunks = this.#statement(
|
|
633
|
+
`SELECT c.id AS id, c.embed_hash AS hash, d.path AS path, d.title AS title,
|
|
634
|
+
c.heading_path AS heading_path, c.line_start AS line_start, c.line_end AS line_end,
|
|
635
|
+
v.text AS text
|
|
636
|
+
FROM chunks c JOIN documents d ON d.id = c.document_id JOIN chunk_content v ON v.id = c.id
|
|
637
|
+
WHERE c.embed_hash IN (SELECT value FROM json_each(?))`
|
|
638
|
+
).all(JSON.stringify([...scored.keys()]));
|
|
639
|
+
chunks.sort(
|
|
640
|
+
(left, right) => (scored.get(String(right.hash)) ?? 0) - (scored.get(String(left.hash)) ?? 0) || Number(left.id) - Number(right.id)
|
|
641
|
+
);
|
|
642
|
+
const hits = [];
|
|
643
|
+
const perPath = /* @__PURE__ */ new Map();
|
|
644
|
+
for (const row of chunks) {
|
|
645
|
+
const path = String(row.path);
|
|
646
|
+
if (options.exclude?.has(path)) continue;
|
|
647
|
+
const taken = perPath.get(path) ?? 0;
|
|
648
|
+
if (taken >= perDocument) continue;
|
|
649
|
+
perPath.set(path, taken + 1);
|
|
650
|
+
const score = scored.get(String(row.hash)) ?? 0;
|
|
651
|
+
hits.push({
|
|
652
|
+
path,
|
|
653
|
+
title: String(row.title),
|
|
654
|
+
headingPath: JSON.parse(String(row.heading_path)),
|
|
655
|
+
lineStart: Number(row.line_start),
|
|
656
|
+
lineEnd: Number(row.line_end),
|
|
657
|
+
snippet: excerpt(String(row.text)),
|
|
658
|
+
ranges: [],
|
|
659
|
+
score,
|
|
660
|
+
chunk: Number(row.id),
|
|
661
|
+
similarity: score
|
|
662
|
+
});
|
|
663
|
+
if (hits.length >= limit) break;
|
|
664
|
+
}
|
|
665
|
+
return hits;
|
|
666
|
+
}
|
|
667
|
+
/**
|
|
668
|
+
* Lexical and semantic results fused by reciprocal rank: the passages that
|
|
669
|
+
* match the words, and those that mean the same thing, in one list. Without
|
|
670
|
+
* a query vector (no model yet) it is the lexical search.
|
|
671
|
+
*/
|
|
672
|
+
hybridSearch(query, vector, options = {}) {
|
|
673
|
+
const limit = clamp(options.limit, 20, SEARCH_LIMIT_MAX);
|
|
674
|
+
const perDocument = clamp(options.perDocument, 3, SEARCH_LIMIT_MAX);
|
|
675
|
+
const wide = {
|
|
676
|
+
...options,
|
|
677
|
+
limit: Math.min(SEARCH_LIMIT_MAX, limit * 3),
|
|
678
|
+
perDocument: perDocument * 2
|
|
679
|
+
};
|
|
680
|
+
const lexical = this.search(query, wide);
|
|
681
|
+
const semantic = vector ? this.semanticSearch(vector, wide) : [];
|
|
682
|
+
if (semantic.length === 0) return lexical.slice(0, limit);
|
|
683
|
+
const fused = /* @__PURE__ */ new Map();
|
|
684
|
+
lexical.forEach((hit, index) => {
|
|
685
|
+
fused.set(hit.chunk, {
|
|
686
|
+
hit,
|
|
687
|
+
score: LIVE_HYBRID_WEIGHTS.lexical / (LIVE_RRF_K + index + 1),
|
|
688
|
+
lexical: index + 1
|
|
689
|
+
});
|
|
690
|
+
});
|
|
691
|
+
semantic.forEach((hit, index) => {
|
|
692
|
+
const share = LIVE_HYBRID_WEIGHTS.vector / (LIVE_RRF_K + index + 1);
|
|
693
|
+
const existing = fused.get(hit.chunk);
|
|
694
|
+
if (existing) {
|
|
695
|
+
existing.score += share;
|
|
696
|
+
existing.vector = index + 1;
|
|
697
|
+
existing.hit = { ...existing.hit, similarity: hit.similarity };
|
|
698
|
+
} else {
|
|
699
|
+
fused.set(hit.chunk, { hit, score: share, vector: index + 1 });
|
|
700
|
+
}
|
|
701
|
+
});
|
|
702
|
+
const ordered = [...fused.values()].sort(
|
|
703
|
+
(left, right) => right.score - left.score || left.hit.chunk - right.hit.chunk
|
|
704
|
+
);
|
|
705
|
+
const hits = [];
|
|
706
|
+
const perPath = /* @__PURE__ */ new Map();
|
|
707
|
+
for (const entry of ordered) {
|
|
708
|
+
const taken = perPath.get(entry.hit.path) ?? 0;
|
|
709
|
+
if (taken >= perDocument) continue;
|
|
710
|
+
perPath.set(entry.hit.path, taken + 1);
|
|
711
|
+
hits.push({
|
|
712
|
+
...entry.hit,
|
|
713
|
+
score: entry.score,
|
|
714
|
+
arms: {
|
|
715
|
+
...entry.lexical ? { lexical: entry.lexical } : {},
|
|
716
|
+
...entry.vector ? { vector: entry.vector } : {}
|
|
717
|
+
}
|
|
718
|
+
});
|
|
719
|
+
if (hits.length >= limit) break;
|
|
720
|
+
}
|
|
721
|
+
return hits;
|
|
722
|
+
}
|
|
723
|
+
#statement(sql) {
|
|
724
|
+
let statement = this.#statements.get(sql);
|
|
725
|
+
if (!statement) {
|
|
726
|
+
statement = this.#db.prepare(sql);
|
|
727
|
+
this.#statements.set(sql, statement);
|
|
728
|
+
}
|
|
729
|
+
return statement;
|
|
730
|
+
}
|
|
731
|
+
/** Drop a document's chunks from FTS and the chunk table; its row must be unchanged. */
|
|
732
|
+
#unindex(documentId) {
|
|
733
|
+
this.#statement(
|
|
734
|
+
`INSERT INTO fts_chunks(fts_chunks, rowid, title, heading_text, text)
|
|
735
|
+
SELECT 'delete', id, title, heading_text, text FROM chunk_content WHERE document_id = ?`
|
|
736
|
+
).run(documentId);
|
|
737
|
+
this.#statement("DELETE FROM chunks WHERE document_id = ?").run(documentId);
|
|
738
|
+
}
|
|
739
|
+
#index(documentId, title, placed) {
|
|
740
|
+
const chunks = placed.length > 0 ? placed : [
|
|
741
|
+
{
|
|
742
|
+
kind: "doc",
|
|
743
|
+
headingPath: [],
|
|
744
|
+
lineStart: 1,
|
|
745
|
+
lineEnd: 1,
|
|
746
|
+
text: "",
|
|
747
|
+
start: 0,
|
|
748
|
+
length: 0
|
|
749
|
+
}
|
|
750
|
+
];
|
|
751
|
+
const insert = this.#statement(
|
|
752
|
+
`INSERT INTO chunks (document_id, ordinal, kind, heading_path, heading_text, line_start,
|
|
753
|
+
line_end, text_start, text_length, text_inline, embed_hash)
|
|
754
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
755
|
+
);
|
|
756
|
+
const embedding = this.#embedding;
|
|
757
|
+
chunks.forEach((chunk, ordinal) => {
|
|
758
|
+
const inline = chunk.start === null || chunk.length === null;
|
|
759
|
+
const embedHash = embedding && chunk.text.length > 0 ? sha256Hex(embedInputFor(embedding, title, chunk.headingPath, chunk.text)) : null;
|
|
760
|
+
insert.run(
|
|
761
|
+
documentId,
|
|
762
|
+
ordinal,
|
|
763
|
+
chunk.kind,
|
|
764
|
+
JSON.stringify(chunk.headingPath),
|
|
765
|
+
chunk.headingPath.join(" / "),
|
|
766
|
+
chunk.lineStart,
|
|
767
|
+
chunk.lineEnd,
|
|
768
|
+
inline ? null : chunk.start,
|
|
769
|
+
inline ? null : chunk.length,
|
|
770
|
+
inline ? chunk.text : null,
|
|
771
|
+
embedHash
|
|
772
|
+
);
|
|
773
|
+
});
|
|
774
|
+
this.#statement(
|
|
775
|
+
`INSERT INTO fts_chunks(rowid, title, heading_text, text)
|
|
776
|
+
SELECT id, title, heading_text, text FROM chunk_content WHERE document_id = ?`
|
|
777
|
+
).run(documentId);
|
|
778
|
+
return chunks.length;
|
|
779
|
+
}
|
|
780
|
+
/** The sign-bit block for the first-stage scan, rebuilt after vectors change. */
|
|
781
|
+
#bits() {
|
|
782
|
+
if (this.#bitIndex) return this.#bitIndex;
|
|
783
|
+
const rows = this.#statement("SELECT hash, bits FROM embeddings ORDER BY id").all();
|
|
784
|
+
const bytesPerRow = Math.ceil((this.#embedding?.profile.dimensions ?? 0) / 8);
|
|
785
|
+
const bits = new Uint8Array(rows.length * bytesPerRow);
|
|
786
|
+
const hashes = [];
|
|
787
|
+
rows.forEach((row, index) => {
|
|
788
|
+
bits.set(row.bits, index * bytesPerRow);
|
|
789
|
+
hashes.push(String(row.hash));
|
|
790
|
+
});
|
|
791
|
+
this.#bitIndex = { bits, bytesPerRow, rows: rows.length, hashes };
|
|
792
|
+
return this.#bitIndex;
|
|
793
|
+
}
|
|
794
|
+
#rows() {
|
|
795
|
+
if (this.#chunkCount === null) {
|
|
796
|
+
this.#chunkCount = Number(this.#statement("SELECT COUNT(*) AS n FROM chunks").get()?.n ?? 0);
|
|
797
|
+
}
|
|
798
|
+
return this.#chunkCount;
|
|
799
|
+
}
|
|
800
|
+
#termRowCount(phrase) {
|
|
801
|
+
let rows = this.#termRows.get(phrase);
|
|
802
|
+
if (rows === void 0) {
|
|
803
|
+
if (this.#termRows.size >= TERM_CACHE_MAX) this.#termRows.clear();
|
|
804
|
+
rows = Number(
|
|
805
|
+
this.#statement("SELECT doc FROM fts_chunks_vocab WHERE term = ?").get(indexedTerm(phrase))?.doc ?? 0
|
|
806
|
+
);
|
|
807
|
+
this.#termRows.set(phrase, rows);
|
|
808
|
+
}
|
|
809
|
+
return rows;
|
|
810
|
+
}
|
|
811
|
+
#changed() {
|
|
812
|
+
this.#chunkCount = null;
|
|
813
|
+
this.#termRows.clear();
|
|
814
|
+
}
|
|
815
|
+
};
|
|
816
|
+
function identityOf(options) {
|
|
817
|
+
const profile = options.embedding?.profile.id;
|
|
818
|
+
return profile ? `${options.chunking.id}|embed:${profile}` : options.chunking.id;
|
|
819
|
+
}
|
|
820
|
+
function toDocument(row) {
|
|
821
|
+
const payload = row.payload;
|
|
822
|
+
return {
|
|
823
|
+
path: String(row.path),
|
|
824
|
+
version: String(row.version),
|
|
825
|
+
title: String(row.title),
|
|
826
|
+
payload: payload instanceof Uint8Array ? payload : null,
|
|
827
|
+
updatedAt: Number(row.updated_at)
|
|
828
|
+
};
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
// src/index.ts
|
|
832
|
+
import {
|
|
833
|
+
BGE_SMALL_EN_V15_1,
|
|
834
|
+
KNOWLEDGE_EMBEDDING_PROFILES,
|
|
835
|
+
MULTILINGUAL_E5_SMALL_2
|
|
836
|
+
} from "@bendyline/gezel-knowledge/browser";
|
|
837
|
+
export {
|
|
838
|
+
BGE_SMALL_EN_V15_1,
|
|
839
|
+
KNOWLEDGE_EMBEDDING_PROFILES,
|
|
840
|
+
LIVE_CHUNK_FTS_WEIGHTS,
|
|
841
|
+
LIVE_HYBRID_WEIGHTS,
|
|
842
|
+
LIVE_INDEX_SCHEMA,
|
|
843
|
+
LIVE_RRF_K,
|
|
844
|
+
LiveIndex,
|
|
845
|
+
MULTILINGUAL_E5_SMALL_2,
|
|
846
|
+
liveIndexEmbedding,
|
|
847
|
+
markdownChunksByCharacters,
|
|
848
|
+
markdownChunksByTokens
|
|
849
|
+
};
|