@bendyline/gezel-retrieval 0.0.0-stage → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Bendyline
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md CHANGED
@@ -1,3 +1,45 @@
1
- # Temporary Holding Version
1
+ # @bendyline/gezel-retrieval
2
2
 
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
3
+ A mutable retrieval index for documents an app owns. Each document's Markdown is chunked by heading, indexed with SQLite FTS5, and searchable by BM25 with snippets and line spans. Documents are replaced or removed one at a time, so an app keeps the index current as its files change and reopens it across launches instead of rebuilding it.
4
+
5
+ It is the mutable counterpart to Gezel's `.gezk` knowledge catalogs and shares their semantics: the `markdown-chunks` profiles, the FTS tokenizer, and the query rule that drops words too common to rank by. It is browser-safe and runs in a Web Worker or a mobile WebView as well as in Node, over whatever synchronous SQLite handle the host supplies.
6
+
7
+ ```ts
8
+ import { LiveIndex, markdownChunksByCharacters } from '@bendyline/gezel-retrieval';
9
+
10
+ const index = LiveIndex.open(db, {
11
+ chunking: markdownChunksByCharacters(),
12
+ appVersion: 'my-app-index@1',
13
+ });
14
+ if (index.reset) {
15
+ // New, incompatible, or written by another app format: ingest everything.
16
+ }
17
+
18
+ index.upsert({ path: 'notes/garden.md', version: etag, title: 'Garden', body: markdown });
19
+ index.remove('notes/old.md');
20
+
21
+ // Diff against the app's own listing to find what changed while it was closed.
22
+ const known = new Map(index.manifest().map(({ path, version }) => [path, version]));
23
+
24
+ const hits = index.search('aphid spray', { limit: 20, perDocument: 3, prefix: true });
25
+ // → [{ path, title, headingPath, lineStart, lineEnd, snippet, ranges, score }]
26
+ ```
27
+
28
+ ## SQLite handles
29
+
30
+ - **Node**: `new DatabaseSync(path)` from `node:sqlite` satisfies `SqlDatabase` as is.
31
+ - **Browser worker / WebView**: `@bendyline/gezel-retrieval/sqlite-wasm` adapts the official sqlite-wasm build (`@sqlite.org/sqlite-wasm`, which the app bundles and serves `sqlite3.wasm` for). `openPoolDatabase` keeps the index in origin-private storage (the `opfs-sahpool` VFS, dedicated Worker only); `openMemoryDatabase` is for a context that cannot own the pool, such as a second window.
32
+
33
+ Only one context may hold a pool at a time; serialize owners with a Web Lock.
34
+
35
+ ## What it stores
36
+
37
+ - **Documents**: the app's key (`path`), its change token (`version`), the title, the body that was chunked, and an opaque `payload` the app may use for anything.
38
+ - **Chunks**: heading path, line span and the chunk's location in the body. Chunk text is not copied; `fts_chunks` is an external-content FTS5 table reading through a view.
39
+ - **Meta**: the schema version, the app's format (`appVersion`) and the chunking id.
40
+
41
+ When the schema or `appVersion` changes, `open` empties the index and reports why in `reset`. When only the chunking changes, documents are re-chunked from their stored bodies with `rechunkStale()`, with no file reads; they stay searchable in the meantime.
42
+
43
+ ## Not yet
44
+
45
+ Vectors (embedding profiles, quantized storage, hybrid fusion with Gezel's measured floors) arrive in a later release.
@@ -0,0 +1,254 @@
1
+ import { ChunkerOptions } from '@bendyline/gezel-knowledge/browser';
2
+ export { BGE_SMALL_EN_V15_1, KNOWLEDGE_EMBEDDING_PROFILES, MULTILINGUAL_E5_SMALL_2 } from '@bendyline/gezel-knowledge/browser';
3
+ import { S as SqlDatabase } from './sql-BFQtPDha.js';
4
+ export { a as SqlRow, b as SqlStatement, c as SqlValue } from './sql-BFQtPDha.js';
5
+ import { KnowledgeEmbeddingProfile } from '@bendyline/gezk';
6
+ export { KnowledgeEmbeddingProfile } from '@bendyline/gezk';
7
+
8
+ /** How a live index splits documents. A different `id` re-chunks every document. */
9
+ interface LiveChunking {
10
+ readonly id: string;
11
+ readonly options: ChunkerOptions;
12
+ }
13
+ /**
14
+ * `markdown-chunks@1` (gezk spec §7): 4,000-character sections with 400 of
15
+ * overlap. Needs no tokenizer, so an app can index before any model exists.
16
+ */
17
+ declare function markdownChunksByCharacters(): LiveChunking;
18
+ /**
19
+ * `markdown-chunks@2`, counted with an embedding profile's tokenizer: the
20
+ * chunking knowledge catalogs use. `tokenizer` names that tokenizer (an
21
+ * embedding profile id, say) so switching it re-chunks.
22
+ */
23
+ declare function markdownChunksByTokens(countTokens: (text: string) => number, tokenizer: string): LiveChunking;
24
+
25
+ /**
26
+ * The embedding profile a live index stores vectors for, and its tokenizer.
27
+ * Passages embed exactly as they would in a knowledge catalog built with the
28
+ * same profile, so the profile's measured floor applies to both.
29
+ */
30
+ interface LiveEmbedding {
31
+ readonly profile: KnowledgeEmbeddingProfile;
32
+ /** The profile's tokenizer: context headers are budgeted with it. */
33
+ readonly countTokens: (text: string) => number;
34
+ }
35
+ /**
36
+ * The chunking and embedding a live index needs for `profile`: the catalogs'
37
+ * `markdown-chunks@2`, counted with the profile's own tokenizer.
38
+ */
39
+ declare function liveIndexEmbedding(profile: KnowledgeEmbeddingProfile, countTokens: (text: string) => number): {
40
+ readonly chunking: LiveChunking;
41
+ readonly embedding: LiveEmbedding;
42
+ };
43
+ /** Reciprocal-rank fusion constant, as the catalogs' unified search uses. */
44
+ declare const LIVE_RRF_K = 60;
45
+ /**
46
+ * Arm weights for hybrid search. Equal: in a person's own notes the exact
47
+ * word they remember is as telling as a passage about the same thing.
48
+ */
49
+ declare const LIVE_HYBRID_WEIGHTS: {
50
+ readonly lexical: 1;
51
+ readonly vector: 1;
52
+ };
53
+
54
+ interface LiveIndexOptions {
55
+ readonly chunking: LiveChunking;
56
+ /**
57
+ * The app's format for `version` and `payload`. When it changes, stored
58
+ * rows no longer mean what the app expects, so the index starts empty.
59
+ */
60
+ readonly appVersion: string;
61
+ /** Milliseconds since the epoch, for `updatedAt`. */
62
+ readonly now?: () => number;
63
+ /**
64
+ * Store vectors for this embedding profile (see `liveIndexEmbedding`).
65
+ * Without it the index is lexical only. Changing it marks every document
66
+ * stale for `rechunkStale()` and drops vectors from the old profile.
67
+ */
68
+ readonly embedding?: LiveEmbedding;
69
+ }
70
+ /** Why the index opened empty: a new database, an incompatible one, or a new app format. */
71
+ type LiveIndexReset = 'created' | 'schema' | 'app' | null;
72
+ interface LiveDocumentInput {
73
+ /** The app's key for the document, unique in the index (a workspace path, say). */
74
+ readonly path: string;
75
+ /** The app's change token: the index stores it and hands it back in `manifest()`. */
76
+ readonly version: string;
77
+ readonly title: string;
78
+ /** The Markdown to chunk and index. Line numbers in results refer to it. */
79
+ readonly body: string;
80
+ /** Opaque app data stored beside the document, never interpreted. */
81
+ readonly payload?: Uint8Array | null;
82
+ }
83
+ interface LiveDocument {
84
+ readonly path: string;
85
+ readonly version: string;
86
+ readonly title: string;
87
+ readonly payload: Uint8Array | null;
88
+ readonly updatedAt: number;
89
+ }
90
+ interface LiveManifestEntry {
91
+ readonly path: string;
92
+ readonly version: string;
93
+ }
94
+ interface LiveSearchOptions {
95
+ /** Most hits returned; default 20, at most 200. */
96
+ readonly limit?: number;
97
+ /** Most hits from one document; default 3. */
98
+ readonly perDocument?: number;
99
+ /** Match the query's last word as a prefix, for search as you type. */
100
+ readonly prefix?: boolean;
101
+ /** Documents to leave out (the one being edited, say). */
102
+ readonly exclude?: ReadonlySet<string>;
103
+ }
104
+ interface LiveSearchHit {
105
+ readonly path: string;
106
+ readonly title: string;
107
+ /** Headings above the passage, outermost first. */
108
+ readonly headingPath: readonly string[];
109
+ /** 1-based inclusive line span of the passage in the document body. */
110
+ readonly lineStart: number;
111
+ readonly lineEnd: number;
112
+ /** An excerpt of the passage around its matches. */
113
+ readonly snippet: string;
114
+ /** Match ranges within `snippet`, as [start, end) UTF-16 offsets. */
115
+ readonly ranges: ReadonlyArray<readonly [number, number]>;
116
+ /** Higher is better; comparable within one search. */
117
+ readonly score: number;
118
+ /** The passage's chunk, stable until its document changes. */
119
+ readonly chunk: number;
120
+ /** Cosine similarity to the query, for a hit the vector arm found. */
121
+ readonly similarity?: number;
122
+ /** Rank in each arm that found the hit (1 is best), for a hybrid search. */
123
+ readonly arms?: {
124
+ readonly lexical?: number;
125
+ readonly vector?: number;
126
+ };
127
+ }
128
+ interface LiveSemanticOptions {
129
+ readonly limit?: number;
130
+ readonly perDocument?: number;
131
+ readonly exclude?: ReadonlySet<string>;
132
+ /**
133
+ * Cosine below which a hit is no evidence. Default: the profile's measured
134
+ * floor; `null` keeps every candidate (calibration, tests).
135
+ */
136
+ readonly floor?: number | null;
137
+ }
138
+ /** A passage awaiting its vector: embed `input` and hand the result back with `hash`. */
139
+ interface LivePendingEmbedding {
140
+ readonly hash: string;
141
+ readonly input: string;
142
+ }
143
+ interface LiveEmbeddingStats {
144
+ /** Distinct passages that need a vector. */
145
+ readonly total: number;
146
+ /** Of those, how many have one. */
147
+ readonly embedded: number;
148
+ }
149
+ interface LiveIndexStats {
150
+ readonly documents: number;
151
+ readonly chunks: number;
152
+ /** Documents chunked by an earlier chunking, awaiting `rechunkStale()`. */
153
+ readonly stale: number;
154
+ }
155
+ /**
156
+ * A mutable retrieval index over documents an app owns: each document's
157
+ * Markdown is chunked by heading, indexed with SQLite FTS5, and searchable
158
+ * by BM25 with snippets and line spans. Documents are replaced or removed
159
+ * one at a time, so an app keeps it current as files change and reopens it
160
+ * across launches instead of rebuilding.
161
+ *
162
+ * Runs over any synchronous SQLite handle: `node:sqlite` in Node, or
163
+ * sqlite-wasm on origin-private storage in a Web Worker (`./sqlite-wasm`).
164
+ * The index owns its tables; give it a database of its own.
165
+ */
166
+ declare class LiveIndex {
167
+ #private;
168
+ readonly reset: LiveIndexReset;
169
+ private constructor();
170
+ /** Open (or create) the index in `db`, emptying it when its schema or the app's format changed. */
171
+ static open(db: SqlDatabase, options: LiveIndexOptions): LiveIndex;
172
+ /** Run `fn` atomically; writes inside it are undone if it throws. Nests. */
173
+ transaction<T>(fn: () => T): T;
174
+ /** Add or replace a document. Returns how many chunks it was indexed as. */
175
+ upsert(input: LiveDocumentInput): {
176
+ readonly chunks: number;
177
+ };
178
+ /** Remove a document; false when it was not indexed. */
179
+ remove(path: string): boolean;
180
+ /** Remove every document, and every stored vector. */
181
+ clear(): void;
182
+ /** Every document's path and version, for diffing against the app's own listing. */
183
+ manifest(): LiveManifestEntry[];
184
+ get(path: string): LiveDocument | null;
185
+ /** The body as indexed (LF line endings), or null. */
186
+ body(path: string): string | null;
187
+ /** Every document in batches, without holding them all in memory. */
188
+ documents(batchSize?: number): Generator<LiveDocument>;
189
+ stats(): LiveIndexStats;
190
+ /**
191
+ * Re-chunk up to `limit` documents indexed under an earlier chunking, from
192
+ * their stored bodies. Returns how many it did; 0 when none are left.
193
+ * They stay searchable by their old chunks until then.
194
+ */
195
+ rechunkStale(limit?: number): number;
196
+ /**
197
+ * Best-matching passages for a query, by BM25 over titles, headings and
198
+ * text. The query is plain words, not FTS syntax; words in more than half
199
+ * the passages are dropped, since they cost a full ranking and cannot
200
+ * change its order.
201
+ */
202
+ search(query: string, options?: LiveSearchOptions): LiveSearchHit[];
203
+ /**
204
+ * Passages still waiting for a vector, most recently changed documents
205
+ * first. Embed each `input` with the profile's passage model and hand the
206
+ * vectors back through `storeEmbeddings`. Empty without an embedding
207
+ * profile, or when every passage has one.
208
+ */
209
+ /**
210
+ * The full text of these passages, by chunk id (a hit's `chunk`), for an
211
+ * app that quotes passages to a model. Ids that are gone are absent.
212
+ */
213
+ passageTexts(chunks: readonly number[]): Map<number, string>;
214
+ pendingEmbeddings(limit?: number): LivePendingEmbedding[];
215
+ /** Store passage vectors, raw from the model, against the hashes `pendingEmbeddings` gave. */
216
+ storeEmbeddings(entries: ReadonlyArray<{
217
+ readonly hash: string;
218
+ readonly vector: ArrayLike<number>;
219
+ }>): void;
220
+ embeddingStats(): LiveEmbeddingStats;
221
+ /** Drop vectors whose text no passage holds any more; returns how many. */
222
+ pruneEmbeddings(): number;
223
+ /**
224
+ * Passages nearest a query embedding (raw from the profile's query model,
225
+ * query instruction applied): the catalogs' two-stage scan, sign bits then
226
+ * int8 rescoring, kept only above the profile's measured cosine floor.
227
+ */
228
+ semanticSearch(vector: ArrayLike<number>, options?: LiveSemanticOptions): LiveSearchHit[];
229
+ /**
230
+ * Lexical and semantic results fused by reciprocal rank: the passages that
231
+ * match the words, and those that mean the same thing, in one list. Without
232
+ * a query vector (no model yet) it is the lexical search.
233
+ */
234
+ hybridSearch(query: string, vector: ArrayLike<number> | null, options?: LiveSearchOptions & Pick<LiveSemanticOptions, 'floor'>): LiveSearchHit[];
235
+ }
236
+
237
+ /**
238
+ * Bumped for a change an existing database cannot be migrated across; the
239
+ * index is then dropped and rebuilt, and the app re-ingests its documents.
240
+ */
241
+ declare const LIVE_INDEX_SCHEMA = 2;
242
+ /**
243
+ * bm25() column weights for `fts_chunks`, in column order. A document's own
244
+ * name outranks a heading, which outranks a passing mention: the shape of a
245
+ * person searching their own notes, where the note titled "Visa renewal" is
246
+ * the answer to "visa renewal" far more often than a note that mentions it.
247
+ */
248
+ declare const LIVE_CHUNK_FTS_WEIGHTS: {
249
+ readonly title: 5;
250
+ readonly heading: 2;
251
+ readonly text: 1;
252
+ };
253
+
254
+ export { LIVE_CHUNK_FTS_WEIGHTS, LIVE_HYBRID_WEIGHTS, LIVE_INDEX_SCHEMA, LIVE_RRF_K, type LiveChunking, type LiveDocument, type LiveDocumentInput, type LiveEmbedding, type LiveEmbeddingStats, LiveIndex, type LiveIndexOptions, type LiveIndexReset, type LiveIndexStats, type LiveManifestEntry, type LivePendingEmbedding, type LiveSearchHit, type LiveSearchOptions, type LiveSemanticOptions, SqlDatabase, liveIndexEmbedding, markdownChunksByCharacters, markdownChunksByTokens };