@4fu/arbor-notes-sqlite 2026.1106.1 → 2026.1107.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -1
- package/index.d.ts +3 -1
- package/index.js +3 -1
- package/notes.d.ts +61 -2
- package/notes.js +228 -16
- package/package.json +17 -6
- package/schema.d.ts +8 -3
- package/schema.js +60 -10
- package/search.d.ts +49 -16
- package/search.js +109 -32
- package/semantic.d.ts +29 -0
- package/semantic.js +57 -0
package/README.md
CHANGED
|
@@ -56,7 +56,9 @@ npm install @4fu/arbor-core @4fu/arbor-journal-sqlite @4fu/arbor-notes-sqlite \
|
|
|
56
56
|
```
|
|
57
57
|
|
|
58
58
|
`@4fu/arbor-plugin-http` is optional; it is the example plugin below.
|
|
59
|
-
`@4fu/arbor-segmenter-jieba` is an optional peer of the
|
|
59
|
+
`@4fu/arbor-segmenter-jieba` is an optional peer of the lexical package, and
|
|
60
|
+
`@4fu/arbor-search-zvec` an optional peer of the journal: absent either one, the
|
|
61
|
+
journal searches lexically and nothing else changes.
|
|
60
62
|
|
|
61
63
|
## A minimal host
|
|
62
64
|
|
|
@@ -120,6 +122,8 @@ window gets, and how a continuation keeps working in it, are in
|
|
|
120
122
|
| [`@4fu/arbor-journal-sqlite`](packages/journal-sqlite) | the default `Journal` and `BlobStore` on SQLite |
|
|
121
123
|
| [`@4fu/arbor-notes-sqlite`](packages/notes-sqlite) | the default `NoteStore` on SQLite, beside the journal |
|
|
122
124
|
| [`@4fu/arbor-segmenter-jieba`](packages/segmenter-jieba) | the default `Segmenter`: jieba's dictionary segmentation for the word index, optional |
|
|
125
|
+
| [`@4fu/arbor-lexical`](packages/lexical) | the lexical search the journal and notes share: word segmentation, the FTS5 helpers and the merge order |
|
|
126
|
+
| [`@4fu/arbor-search-zvec`](packages/search-zvec) | the optional semantic channel: event text embedded and searched by meaning with zvec |
|
|
123
127
|
| [`@4fu/arbor-workspace-dir`](packages/workspace-dir) | the fallback `Workspace`: one directory per window, with a continuation working in its predecessor's directory in place |
|
|
124
128
|
| [`@4fu/arbor-workspace-ovnfs`](packages/workspace-ovnfs) | the default `Workspace` when ovnfs is installed: one shared folder, per-window copy-on-write checkouts the model opens and applies back |
|
|
125
129
|
| [`@4fu/arbor-environment-dir`](packages/environment-dir) | the default `Environment`: one `/env` directory, a revision per item, validation and deltas |
|
|
@@ -171,6 +175,7 @@ tarball would contain ([docs/release.md](docs/release.md)).
|
|
|
171
175
|
| `packages/errors` | the error shape every refusal uses: the rule and the next action |
|
|
172
176
|
| `packages/journal-sqlite` | the default `Journal` and `BlobStore` on SQLite |
|
|
173
177
|
| `packages/segmenter-jieba` | the default `Segmenter`: jieba's dictionary segmentation for the word index, optional |
|
|
178
|
+
| `packages/lexical` | the lexical search the journal and notes share: word segmentation, the FTS5 helpers and the merge order |
|
|
174
179
|
| `packages/notes-sqlite` | the default `NoteStore` on SQLite, beside the journal |
|
|
175
180
|
| `packages/judge-jev` | the default `Judge`: Jev, TypeSafe AI's System One classifier, answering the dispatch decisions |
|
|
176
181
|
| `packages/workspace-dir` | the default `Workspace`: one directory per window, with a continuation working in its predecessor's directory in place, and the lookup of another window's directory it contributes to the catalog |
|
package/index.d.ts
CHANGED
|
@@ -4,10 +4,12 @@
|
|
|
4
4
|
export { SqliteNoteStore, SUMMARY_LIMIT, DEFAULT_BODY_LIMIT, MAX_BODY_LIMIT } from "./notes.ts";
|
|
5
5
|
export { DEFAULT_MIN_INTERVAL_MS, DEFAULT_ACTIVE_BUDGET, DEFAULT_JUDGE_SHARE, DEFAULT_HALF_LIFE_MS, deriveSummary, } from "./notes.ts";
|
|
6
6
|
export type { SqliteNoteStoreOptions } from "./notes.ts";
|
|
7
|
-
export { SCHEMA_VERSION, FTS_TABLE, createSchema } from "./schema.ts";
|
|
7
|
+
export { SCHEMA_VERSION, FTS_TABLE, WORD_TABLE, createSchema } from "./schema.ts";
|
|
8
8
|
export { noteIdOf, noteRefOf, parseNoteId, parseNoteRef, noteRefForms, storedWindowRef, storedEventRef, } from "./refs.ts";
|
|
9
|
+
export { fallbackSegmenter, resolveSegmenter, indexText } from "@4fu/arbor-lexical";
|
|
9
10
|
export { USE_WEIGHTS, MAX_CREDIT_HOPS, decay, estimateTokens, linkedCredit, loadCredit, loadEdges, pickByBudget, applyJudgeShare, scoreNotes, } from "./score.ts";
|
|
10
11
|
export type { ScoredNote } from "./score.ts";
|
|
11
12
|
export { MIN_TRIGRAM_CHARS, substringClause, holdingIds, escapeLike, likePattern, quoteFts, } from "./search.ts";
|
|
12
13
|
export type { SubstringClause } from "./search.ts";
|
|
13
14
|
export { bodyDiff } from "./diff.ts";
|
|
15
|
+
export { resolveSemanticIndex } from "./semantic.ts";
|
package/index.js
CHANGED
|
@@ -3,8 +3,10 @@
|
|
|
3
3
|
*/
|
|
4
4
|
export { SqliteNoteStore, SUMMARY_LIMIT, DEFAULT_BODY_LIMIT, MAX_BODY_LIMIT } from "./notes.js";
|
|
5
5
|
export { DEFAULT_MIN_INTERVAL_MS, DEFAULT_ACTIVE_BUDGET, DEFAULT_JUDGE_SHARE, DEFAULT_HALF_LIFE_MS, deriveSummary, } from "./notes.js";
|
|
6
|
-
export { SCHEMA_VERSION, FTS_TABLE, createSchema } from "./schema.js";
|
|
6
|
+
export { SCHEMA_VERSION, FTS_TABLE, WORD_TABLE, createSchema } from "./schema.js";
|
|
7
7
|
export { noteIdOf, noteRefOf, parseNoteId, parseNoteRef, noteRefForms, storedWindowRef, storedEventRef, } from "./refs.js";
|
|
8
|
+
export { fallbackSegmenter, resolveSegmenter, indexText } from "@4fu/arbor-lexical";
|
|
8
9
|
export { USE_WEIGHTS, MAX_CREDIT_HOPS, decay, estimateTokens, linkedCredit, loadCredit, loadEdges, pickByBudget, applyJudgeShare, scoreNotes, } from "./score.js";
|
|
9
10
|
export { MIN_TRIGRAM_CHARS, substringClause, holdingIds, escapeLike, likePattern, quoteFts, } from "./search.js";
|
|
10
11
|
export { bodyDiff } from "./diff.js";
|
|
12
|
+
export { resolveSemanticIndex } from "./semantic.js";
|
package/notes.d.ts
CHANGED
|
@@ -15,7 +15,11 @@
|
|
|
15
15
|
* count and token budget, and frozen with the epoch's version, so every
|
|
16
16
|
* window opened in one epoch has the same prefix and later windows see the
|
|
17
17
|
* changes as a delta;
|
|
18
|
-
* - every write recorded in the journal as a `note_revision` event
|
|
18
|
+
* - every write recorded in the journal as a `note_revision` event;
|
|
19
|
+
* - lexical search: a word index with BM25, a trigram index for exact
|
|
20
|
+
* substrings, and a `LIKE` scan for one- and two-character queries, merged
|
|
21
|
+
* with exact substring hits first - and a semantic channel, note text embedded
|
|
22
|
+
* and searched by meaning, when `@4fu/arbor-search-zvec` is installed.
|
|
19
23
|
*
|
|
20
24
|
* It is synchronous inside: one `DatabaseSync`, one transaction per write, so
|
|
21
25
|
* the revision a write checks and the revision it writes are one atomic step.
|
|
@@ -23,7 +27,7 @@
|
|
|
23
27
|
* database and the journal. The interface is asynchronous because a `NoteStore`
|
|
24
28
|
* may live behind a process boundary.
|
|
25
29
|
*/
|
|
26
|
-
import type { ActiveCandidate, ActiveSetBudget, Journal, Note, NoteBody, NoteContext, NoteDelta, NoteQuery, NoteRef, NoteRevision, NoteSummary, NoteUseInput, NoteWriteInput, NotesSnapshot, NoteStore, Page, SnapshotOptions } from "@4fu/arbor-interfaces";
|
|
30
|
+
import type { ActiveCandidate, ActiveSetBudget, Journal, Note, NoteBody, NoteContext, NoteDelta, NoteQuery, NoteRef, NoteRevision, NoteSummary, NoteUseInput, NoteWriteInput, NotesSnapshot, NoteStore, Page, Segmenter, SemanticIndex, SnapshotOptions } from "@4fu/arbor-interfaces";
|
|
27
31
|
/** A summary is at most this many characters (docs/memory.md). */
|
|
28
32
|
export declare const SUMMARY_LIMIT = 200;
|
|
29
33
|
/** How many characters of a body one read returns. */
|
|
@@ -46,6 +50,16 @@ export interface SqliteNoteStoreOptions {
|
|
|
46
50
|
now?: () => number;
|
|
47
51
|
/** Usage loses half of its weight this often. 24 hours. */
|
|
48
52
|
usageHalfLifeMs?: number;
|
|
53
|
+
/** Word segmentation for the word index. Defaults to jieba, or the approximation. */
|
|
54
|
+
segmenter?: Segmenter;
|
|
55
|
+
/**
|
|
56
|
+
* The semantic channel: note text embedded and searched by meaning
|
|
57
|
+
* (docs/memory.md#search). Omitted, the zvec channel is used when
|
|
58
|
+
* `@4fu/arbor-search-zvec` is installed and its store opens, and there is no
|
|
59
|
+
* channel otherwise. `null` forces lexical search even when the package is
|
|
60
|
+
* installed.
|
|
61
|
+
*/
|
|
62
|
+
semantic?: SemanticIndex | null;
|
|
49
63
|
/**
|
|
50
64
|
* One fixed string from the deployment's text table, so this implementation
|
|
51
65
|
* writes none of its own (docs/prompting.md#language-and-the-text-table).
|
|
@@ -60,11 +74,17 @@ export declare class SqliteNoteStore implements NoteStore {
|
|
|
60
74
|
private readonly db;
|
|
61
75
|
private readonly journal;
|
|
62
76
|
private readonly clock;
|
|
77
|
+
private readonly segmenter;
|
|
63
78
|
private readonly halfLifeMs;
|
|
79
|
+
private readonly semantic;
|
|
80
|
+
/** The temporary directory an in-memory store's vector collection lives in. */
|
|
81
|
+
private readonly ownedVectorDirectory;
|
|
64
82
|
/** The table every refusal this store raises is written from. */
|
|
65
83
|
private readonly text;
|
|
66
84
|
/** One write at a time: a write spans the notes database and the journal. */
|
|
67
85
|
private writeQueue;
|
|
86
|
+
private semanticReady;
|
|
87
|
+
private semanticCaughtUp;
|
|
68
88
|
private closed;
|
|
69
89
|
constructor(options: SqliteNoteStoreOptions);
|
|
70
90
|
write(input: NoteWriteInput): Promise<Note>;
|
|
@@ -100,7 +120,44 @@ export declare class SqliteNoteStore implements NoteStore {
|
|
|
100
120
|
}): Promise<Page<ActiveCandidate>>;
|
|
101
121
|
recordUse(input: NoteUseInput): Promise<void>;
|
|
102
122
|
close(): Promise<void>;
|
|
123
|
+
/**
|
|
124
|
+
* Index the notes the semantic channel does not hold yet, and say how many.
|
|
125
|
+
*
|
|
126
|
+
* A restart over an existing database, a channel that was not ready when the
|
|
127
|
+
* notes were written, and a write whose indexing failed all leave notes to
|
|
128
|
+
* catch up on. A channel that is not ready yet holds what it was given and
|
|
129
|
+
* indexes that itself, so this answers 0 until it is.
|
|
130
|
+
*/
|
|
131
|
+
catchUpSemantic(): Promise<number>;
|
|
103
132
|
private ensureOpen;
|
|
133
|
+
/**
|
|
134
|
+
* Hand one note's text to the semantic channel, when there is one.
|
|
135
|
+
*
|
|
136
|
+
* The note is embedded as one text: its title, its summary and its body, in
|
|
137
|
+
* the order the lexical index weights them. The vector channel holds one
|
|
138
|
+
* vector per note and cannot weight a note's parts against each other the way
|
|
139
|
+
* BM25 weights the three columns, so the title - the note's name - leads the
|
|
140
|
+
* text and the body follows it, and a query that means either reaches the
|
|
141
|
+
* note (docs/memory.md#search).
|
|
142
|
+
*/
|
|
143
|
+
private addToSemantic;
|
|
144
|
+
/** Take one note's text back out of the semantic channel, as a delete does. */
|
|
145
|
+
private removeFromSemantic;
|
|
146
|
+
/**
|
|
147
|
+
* The semantic channel's candidates, as ranks in the order it gave them.
|
|
148
|
+
*
|
|
149
|
+
* An absent channel, one that is not ready yet, and one that refuses all
|
|
150
|
+
* answer with nothing, and the search is exactly the lexical one. The note
|
|
151
|
+
* read's own filters decide which candidates count, so a channel that filters
|
|
152
|
+
* less than the store does still answers the read it was asked for.
|
|
153
|
+
*/
|
|
154
|
+
private semanticRanks;
|
|
155
|
+
/**
|
|
156
|
+
* The notes among `ids` the read's own filters allow, in the order given. A
|
|
157
|
+
* deleted note is not among them, so a vector that outlived its note never
|
|
158
|
+
* reaches a page.
|
|
159
|
+
*/
|
|
160
|
+
private allowedNoteIds;
|
|
104
161
|
private noteRow;
|
|
105
162
|
private revisionRow;
|
|
106
163
|
private latestRevisionRow;
|
|
@@ -108,6 +165,8 @@ export declare class SqliteNoteStore implements NoteStore {
|
|
|
108
165
|
/** The current summary and revision of several notes, in one read. */
|
|
109
166
|
private noteStatesFor;
|
|
110
167
|
private summaryPageBySql;
|
|
168
|
+
/** One page of an ordered id list, in the order the list gives. */
|
|
169
|
+
private summaryPage;
|
|
111
170
|
private summaryPageByIds;
|
|
112
171
|
/**
|
|
113
172
|
* The notes about a target, or linked to a ref: the notes the events on that
|
package/notes.js
CHANGED
|
@@ -15,7 +15,11 @@
|
|
|
15
15
|
* count and token budget, and frozen with the epoch's version, so every
|
|
16
16
|
* window opened in one epoch has the same prefix and later windows see the
|
|
17
17
|
* changes as a delta;
|
|
18
|
-
* - every write recorded in the journal as a `note_revision` event
|
|
18
|
+
* - every write recorded in the journal as a `note_revision` event;
|
|
19
|
+
* - lexical search: a word index with BM25, a trigram index for exact
|
|
20
|
+
* substrings, and a `LIKE` scan for one- and two-character queries, merged
|
|
21
|
+
* with exact substring hits first - and a semantic channel, note text embedded
|
|
22
|
+
* and searched by meaning, when `@4fu/arbor-search-zvec` is installed.
|
|
19
23
|
*
|
|
20
24
|
* It is synchronous inside: one `DatabaseSync`, one transaction per write, so
|
|
21
25
|
* the revision a write checks and the revision it writes are one atomic step.
|
|
@@ -24,13 +28,19 @@
|
|
|
24
28
|
* may live behind a process boundary.
|
|
25
29
|
*/
|
|
26
30
|
import { DatabaseSync } from "node:sqlite";
|
|
31
|
+
import { randomUUID } from "node:crypto";
|
|
32
|
+
import { rm } from "node:fs/promises";
|
|
33
|
+
import { tmpdir } from "node:os";
|
|
34
|
+
import { join } from "node:path";
|
|
27
35
|
import { arborError, isArborError } from "@4fu/arbor-errors";
|
|
28
36
|
import { DEFAULT_PAGE_LIMIT, MAX_PAGE_LIMIT } from "@4fu/arbor-interfaces";
|
|
37
|
+
import { indexText, resolveSegmenter } from "@4fu/arbor-lexical";
|
|
29
38
|
import { bodyDiff } from "./diff.js";
|
|
30
39
|
import { noteIdOf, noteRefOf, noteRefForms, parseNoteId, parseNoteRef, storedEventRef, storedWindowRef, } from "./refs.js";
|
|
31
|
-
import { FTS_TABLE, createSchema } from "./schema.js";
|
|
40
|
+
import { FTS_TABLE, WORD_TABLE, createSchema } from "./schema.js";
|
|
32
41
|
import { USE_WEIGHTS, applyJudgeShare, pickByBudget, scoreNotes } from "./score.js";
|
|
33
|
-
import { holdingIds,
|
|
42
|
+
import { holdingIds, matchedIds } from "./search.js";
|
|
43
|
+
import { resolveSemanticIndex } from "./semantic.js";
|
|
34
44
|
import { textOf, zhTextTable } from "@4fu/arbor-prompt";
|
|
35
45
|
/** A summary is at most this many characters (docs/memory.md). */
|
|
36
46
|
export const SUMMARY_LIMIT = 200;
|
|
@@ -47,6 +57,13 @@ export const DEFAULT_JUDGE_SHARE = 0.15;
|
|
|
47
57
|
export const DEFAULT_HALF_LIFE_MS = 86_400_000;
|
|
48
58
|
/** How many events one `about` lookup reads. */
|
|
49
59
|
const ABOUT_EVENT_LIMIT = 200;
|
|
60
|
+
/**
|
|
61
|
+
* The fewest candidates the semantic channel is asked for. A page of one still
|
|
62
|
+
* fuses the channels over a handful of candidates, so a hit the word index
|
|
63
|
+
* ranked first is not lost to a semantic hit the channel would have ranked
|
|
64
|
+
* second.
|
|
65
|
+
*/
|
|
66
|
+
const MIN_SEMANTIC_CANDIDATES = 10;
|
|
50
67
|
function toNumber(value) {
|
|
51
68
|
if (typeof value === "number")
|
|
52
69
|
return value;
|
|
@@ -164,20 +181,38 @@ export class SqliteNoteStore {
|
|
|
164
181
|
db;
|
|
165
182
|
journal;
|
|
166
183
|
clock;
|
|
184
|
+
segmenter;
|
|
167
185
|
halfLifeMs;
|
|
186
|
+
semantic;
|
|
187
|
+
/** The temporary directory an in-memory store's vector collection lives in. */
|
|
188
|
+
ownedVectorDirectory;
|
|
168
189
|
/** The table every refusal this store raises is written from. */
|
|
169
190
|
text;
|
|
170
191
|
/** One write at a time: a write spans the notes database and the journal. */
|
|
171
192
|
writeQueue = Promise.resolve();
|
|
193
|
+
semanticReady = false;
|
|
194
|
+
semanticCaughtUp = false;
|
|
172
195
|
closed = false;
|
|
173
196
|
constructor(options) {
|
|
174
197
|
this.path = options.path ?? ":memory:";
|
|
175
198
|
this.db = new DatabaseSync(this.path);
|
|
176
|
-
|
|
199
|
+
this.segmenter = options.segmenter ?? resolveSegmenter();
|
|
200
|
+
// The segmenter is resolved first: a version-1 database migrates by
|
|
201
|
+
// segmenting the notes it already holds into the word index.
|
|
202
|
+
createSchema(this.db, this.segmenter);
|
|
177
203
|
this.journal = options.journal;
|
|
178
204
|
this.clock = options.now ?? (() => Date.now());
|
|
179
205
|
this.halfLifeMs = options.usageHalfLifeMs ?? DEFAULT_HALF_LIFE_MS;
|
|
180
206
|
this.text = options.text ?? ((key, args) => textOf(zhTextTable, key, args));
|
|
207
|
+
// An in-memory store has no directory of its own; give it a temporary one
|
|
208
|
+
// and take it away on close. The semantic channel's store lives under it,
|
|
209
|
+
// beside where the notes database would be.
|
|
210
|
+
const root = this.path === ":memory:" ? join(tmpdir(), `arbor-notes-${randomUUID()}`) : null;
|
|
211
|
+
this.semantic =
|
|
212
|
+
options.semantic === undefined
|
|
213
|
+
? resolveSemanticIndex(root === null ? `${this.path}.vectors` : join(root, "vectors"))
|
|
214
|
+
: options.semantic;
|
|
215
|
+
this.ownedVectorDirectory = root;
|
|
181
216
|
}
|
|
182
217
|
// --- writes -------------------------------------------------------------
|
|
183
218
|
async write(input) {
|
|
@@ -233,6 +268,17 @@ export class SqliteNoteStore {
|
|
|
233
268
|
.run(event.ref, written.id, written.revision);
|
|
234
269
|
this.db.prepare("UPDATE notes SET event = ? WHERE id = ?").run(event.ref, written.id);
|
|
235
270
|
});
|
|
271
|
+
// The semantic channel is asynchronous, so the note is indexed once its
|
|
272
|
+
// revision is committed: a write never waits on a model, and a channel that
|
|
273
|
+
// is not ready yet holds the text until it is. With no channel there is
|
|
274
|
+
// nothing to wait for, and the write takes the path it always did. A delete
|
|
275
|
+
// takes the note's vector back out, as a redaction does in the journal.
|
|
276
|
+
if (this.semantic !== null) {
|
|
277
|
+
if (written.deleted)
|
|
278
|
+
await this.removeFromSemantic(written.id);
|
|
279
|
+
else
|
|
280
|
+
await this.addToSemantic(written.id, written.title, written.summary, written.body);
|
|
281
|
+
}
|
|
236
282
|
return toNote(this.noteRow(written.id));
|
|
237
283
|
}
|
|
238
284
|
/** The rules a write has to meet before it touches the one copy. */
|
|
@@ -375,12 +421,16 @@ export class SqliteNoteStore {
|
|
|
375
421
|
.prepare(`INSERT INTO note_revisions (note, revision, title, summary, body, deleted, time, window, event, anchor, refs)
|
|
376
422
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?)`)
|
|
377
423
|
.run(id, revision, input.title, summary, body, deleted ? 1 : 0, time, window, anchorJson, refs);
|
|
378
|
-
// The search
|
|
424
|
+
// The search indexes hold the current state; a deleted note is not found.
|
|
379
425
|
this.db.prepare(`DELETE FROM ${FTS_TABLE} WHERE rowid = ?`).run(id);
|
|
426
|
+
this.db.prepare(`DELETE FROM ${WORD_TABLE} WHERE rowid = ?`).run(id);
|
|
380
427
|
if (!deleted) {
|
|
381
428
|
this.db
|
|
382
429
|
.prepare(`INSERT INTO ${FTS_TABLE} (rowid, title, summary, body) VALUES (?, ?, ?, ?)`)
|
|
383
430
|
.run(id, input.title, summary, body);
|
|
431
|
+
this.db
|
|
432
|
+
.prepare(`INSERT INTO ${WORD_TABLE} (rowid, title, summary, body) VALUES (?, ?, ?, ?)`)
|
|
433
|
+
.run(id, indexText(input.title, this.segmenter), indexText(summary, this.segmenter), indexText(body, this.segmenter));
|
|
384
434
|
}
|
|
385
435
|
// The refs the revision names: what it cites and what it holds as
|
|
386
436
|
// evidence. The evidence ones between notes are what the credit walk
|
|
@@ -403,6 +453,8 @@ export class SqliteNoteStore {
|
|
|
403
453
|
time,
|
|
404
454
|
title: input.title,
|
|
405
455
|
summary,
|
|
456
|
+
body,
|
|
457
|
+
deleted,
|
|
406
458
|
window,
|
|
407
459
|
anchor,
|
|
408
460
|
};
|
|
@@ -417,6 +469,7 @@ export class SqliteNoteStore {
|
|
|
417
469
|
.prepare("DELETE FROM note_refs WHERE note = ? AND revision = ?")
|
|
418
470
|
.run(written.id, written.revision);
|
|
419
471
|
this.db.prepare(`DELETE FROM ${FTS_TABLE} WHERE rowid = ?`).run(written.id);
|
|
472
|
+
this.db.prepare(`DELETE FROM ${WORD_TABLE} WHERE rowid = ?`).run(written.id);
|
|
420
473
|
if (written.created) {
|
|
421
474
|
this.db.prepare("DELETE FROM notes WHERE id = ?").run(written.id);
|
|
422
475
|
return;
|
|
@@ -431,6 +484,9 @@ export class SqliteNoteStore {
|
|
|
431
484
|
this.db
|
|
432
485
|
.prepare(`INSERT INTO ${FTS_TABLE} (rowid, title, summary, body) VALUES (?, ?, ?, ?)`)
|
|
433
486
|
.run(written.id, previous.title, previous.summary, previous.body);
|
|
487
|
+
this.db
|
|
488
|
+
.prepare(`INSERT INTO ${WORD_TABLE} (rowid, title, summary, body) VALUES (?, ?, ?, ?)`)
|
|
489
|
+
.run(written.id, indexText(previous.title, this.segmenter), indexText(previous.summary, this.segmenter), indexText(previous.body, this.segmenter));
|
|
434
490
|
}
|
|
435
491
|
});
|
|
436
492
|
}
|
|
@@ -511,16 +567,25 @@ export class SqliteNoteStore {
|
|
|
511
567
|
const offset = Math.max(0, Math.floor(query.offset ?? 0));
|
|
512
568
|
const text = (query.query ?? "").trim();
|
|
513
569
|
// `about` and `linked` bound the result to the notes the journal resolved,
|
|
514
|
-
// in weight order; a text match narrows that set.
|
|
570
|
+
// in weight order; a text match narrows that set. The semantic channel does
|
|
571
|
+
// not widen a bound the journal resolved, so it is not asked at all here.
|
|
515
572
|
if (query.about !== undefined || query.linked !== undefined) {
|
|
516
573
|
const ordered = await this.linkedNoteIds(query.about, query.linked);
|
|
517
|
-
const holding = text === "" ? null : new Set(holdingIds(this.db, text, ordered));
|
|
574
|
+
const holding = text === "" ? null : new Set(holdingIds(this.db, text, ordered, this.segmenter));
|
|
518
575
|
const chosen = holding === null ? ordered : ordered.filter((id) => holding.has(id));
|
|
519
576
|
const page = this.summaryPageByIds(chosen, query, limit, offset);
|
|
520
577
|
return { ...page, nextCall: searchCall(query, offset, limit) };
|
|
521
578
|
}
|
|
522
|
-
|
|
523
|
-
|
|
579
|
+
// No query lists every note by id; a query is answered by the lexical
|
|
580
|
+
// channels and the semantic one, merged: exact substring hits first, then
|
|
581
|
+
// word hits, then the hits only the semantic channel found.
|
|
582
|
+
const semantic = text === "" ? [] : await this.semanticRanks(text, limit, query.status);
|
|
583
|
+
const matched = text === ""
|
|
584
|
+
? null
|
|
585
|
+
: matchedIds(this.db, text, this.segmenter, query.status, semantic);
|
|
586
|
+
const page = matched === null
|
|
587
|
+
? this.summaryPageBySql(query, limit, offset)
|
|
588
|
+
: this.summaryPage(matched, limit, offset);
|
|
524
589
|
return { ...page, nextCall: searchCall(query, offset, limit) };
|
|
525
590
|
}
|
|
526
591
|
async active() {
|
|
@@ -686,6 +751,39 @@ export class SqliteNoteStore {
|
|
|
686
751
|
this.closed = true;
|
|
687
752
|
await this.writeQueue;
|
|
688
753
|
this.db.close();
|
|
754
|
+
// The channel's own store is closed with the notes database, and the
|
|
755
|
+
// temporary directory an in-memory store gave it goes with it.
|
|
756
|
+
if (this.semantic !== null)
|
|
757
|
+
await this.semantic.close();
|
|
758
|
+
if (this.ownedVectorDirectory !== null) {
|
|
759
|
+
await rm(this.ownedVectorDirectory, { recursive: true, force: true });
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
/**
|
|
763
|
+
* Index the notes the semantic channel does not hold yet, and say how many.
|
|
764
|
+
*
|
|
765
|
+
* A restart over an existing database, a channel that was not ready when the
|
|
766
|
+
* notes were written, and a write whose indexing failed all leave notes to
|
|
767
|
+
* catch up on. A channel that is not ready yet holds what it was given and
|
|
768
|
+
* indexes that itself, so this answers 0 until it is.
|
|
769
|
+
*/
|
|
770
|
+
async catchUpSemantic() {
|
|
771
|
+
this.ensureOpen();
|
|
772
|
+
if (this.semantic === null || !(await this.semantic.ready()))
|
|
773
|
+
return 0;
|
|
774
|
+
const held = await this.semantic.held();
|
|
775
|
+
const rows = this.db
|
|
776
|
+
.prepare(`SELECT id, title, summary, body FROM notes WHERE status <> 'deleted' ORDER BY id`)
|
|
777
|
+
.all();
|
|
778
|
+
let indexed = 0;
|
|
779
|
+
for (const row of rows) {
|
|
780
|
+
const id = toNumber(row.id);
|
|
781
|
+
if (held.has(noteIdOf(id)))
|
|
782
|
+
continue;
|
|
783
|
+
await this.addToSemantic(id, row.title, row.summary, row.body);
|
|
784
|
+
indexed += 1;
|
|
785
|
+
}
|
|
786
|
+
return indexed;
|
|
689
787
|
}
|
|
690
788
|
// --- internals ---------------------------------------------------------
|
|
691
789
|
ensureOpen() {
|
|
@@ -698,6 +796,104 @@ export class SqliteNoteStore {
|
|
|
698
796
|
});
|
|
699
797
|
}
|
|
700
798
|
}
|
|
799
|
+
/**
|
|
800
|
+
* Hand one note's text to the semantic channel, when there is one.
|
|
801
|
+
*
|
|
802
|
+
* The note is embedded as one text: its title, its summary and its body, in
|
|
803
|
+
* the order the lexical index weights them. The vector channel holds one
|
|
804
|
+
* vector per note and cannot weight a note's parts against each other the way
|
|
805
|
+
* BM25 weights the three columns, so the title - the note's name - leads the
|
|
806
|
+
* text and the body follows it, and a query that means either reaches the
|
|
807
|
+
* note (docs/memory.md#search).
|
|
808
|
+
*/
|
|
809
|
+
async addToSemantic(id, title, summary, body) {
|
|
810
|
+
if (this.semantic === null)
|
|
811
|
+
return;
|
|
812
|
+
try {
|
|
813
|
+
await this.semantic.add(noteIdOf(id), [title, summary, body].join("\n"));
|
|
814
|
+
}
|
|
815
|
+
catch {
|
|
816
|
+
// An optional channel that refuses leaves the note searchable lexically;
|
|
817
|
+
// a catch-up indexes it once the channel answers again.
|
|
818
|
+
}
|
|
819
|
+
}
|
|
820
|
+
/** Take one note's text back out of the semantic channel, as a delete does. */
|
|
821
|
+
async removeFromSemantic(id) {
|
|
822
|
+
if (this.semantic === null)
|
|
823
|
+
return;
|
|
824
|
+
try {
|
|
825
|
+
await this.semantic.remove(noteIdOf(id));
|
|
826
|
+
}
|
|
827
|
+
catch {
|
|
828
|
+
// As with a write: the channel is optional, and the note is gone from
|
|
829
|
+
// every index the store owns.
|
|
830
|
+
}
|
|
831
|
+
}
|
|
832
|
+
/**
|
|
833
|
+
* The semantic channel's candidates, as ranks in the order it gave them.
|
|
834
|
+
*
|
|
835
|
+
* An absent channel, one that is not ready yet, and one that refuses all
|
|
836
|
+
* answer with nothing, and the search is exactly the lexical one. The note
|
|
837
|
+
* read's own filters decide which candidates count, so a channel that filters
|
|
838
|
+
* less than the store does still answers the read it was asked for.
|
|
839
|
+
*/
|
|
840
|
+
async semanticRanks(query, limit, status) {
|
|
841
|
+
if (this.semantic === null)
|
|
842
|
+
return [];
|
|
843
|
+
const ready = await this.semantic.ready();
|
|
844
|
+
if (ready !== this.semanticReady) {
|
|
845
|
+
// The channel just became usable, or stopped being: index what it missed
|
|
846
|
+
// while it was not.
|
|
847
|
+
this.semanticReady = ready;
|
|
848
|
+
this.semanticCaughtUp = false;
|
|
849
|
+
}
|
|
850
|
+
if (!this.semanticCaughtUp) {
|
|
851
|
+
this.semanticCaughtUp = true;
|
|
852
|
+
await this.catchUpSemantic();
|
|
853
|
+
}
|
|
854
|
+
if (!ready)
|
|
855
|
+
return [];
|
|
856
|
+
// The filters a note read applies are the store's own, and none of them is
|
|
857
|
+
// a field of the channel: a note's status changes at every snapshot epoch
|
|
858
|
+
// without a revision, and `about` and `linked` are bounds the journal
|
|
859
|
+
// resolved. So the channel is asked for the notes nearest the query and
|
|
860
|
+
// nothing else, and the store applies its own filters to what comes back -
|
|
861
|
+
// exactly as the journal does with the targets it filters on itself
|
|
862
|
+
// (docs/memory.md#search).
|
|
863
|
+
const asked = {};
|
|
864
|
+
const refs = await this.semantic.search(query, Math.max(limit, MIN_SEMANTIC_CANDIDATES), asked);
|
|
865
|
+
const ids = [];
|
|
866
|
+
for (const ref of refs) {
|
|
867
|
+
const id = parseNoteId(ref);
|
|
868
|
+
if (id !== null)
|
|
869
|
+
ids.push(id);
|
|
870
|
+
}
|
|
871
|
+
const allowed = this.allowedNoteIds(ids, status);
|
|
872
|
+
// The channel's order, as ranks the shared merge reads: the nearest hit
|
|
873
|
+
// carries the most negative rank, so it normalizes to the best score.
|
|
874
|
+
return allowed.map((id, index) => ({ rowid: id, rank: index - allowed.length }));
|
|
875
|
+
}
|
|
876
|
+
/**
|
|
877
|
+
* The notes among `ids` the read's own filters allow, in the order given. A
|
|
878
|
+
* deleted note is not among them, so a vector that outlived its note never
|
|
879
|
+
* reaches a page.
|
|
880
|
+
*/
|
|
881
|
+
allowedNoteIds(ids, status) {
|
|
882
|
+
if (ids.length === 0)
|
|
883
|
+
return [];
|
|
884
|
+
const clauses = ["status <> 'deleted'"];
|
|
885
|
+
const params = [...ids];
|
|
886
|
+
if (status !== undefined) {
|
|
887
|
+
clauses.push("status = ?");
|
|
888
|
+
params.push(status);
|
|
889
|
+
}
|
|
890
|
+
const rows = this.db
|
|
891
|
+
.prepare(`SELECT id FROM notes
|
|
892
|
+
WHERE id IN (${ids.map(() => "?").join(", ")}) AND ${clauses.join(" AND ")}`)
|
|
893
|
+
.all(...params);
|
|
894
|
+
const allowed = new Set(rows.map((row) => toNumber(row.id)));
|
|
895
|
+
return ids.filter((id) => allowed.has(id));
|
|
896
|
+
}
|
|
701
897
|
noteRow(id) {
|
|
702
898
|
const row = this.db
|
|
703
899
|
.prepare(`SELECT id, created, updated, revision, title, summary, body, status, window, anchor, event
|
|
@@ -737,13 +933,11 @@ export class SqliteNoteStore {
|
|
|
737
933
|
}
|
|
738
934
|
return states;
|
|
739
935
|
}
|
|
740
|
-
summaryPageBySql(query, limit, offset
|
|
741
|
-
|
|
936
|
+
summaryPageBySql(query, limit, offset) {
|
|
937
|
+
// A delete leaves the row and marks it, and this reads the notes table
|
|
938
|
+
// itself, so it leaves the deleted ones out on its own (docs/memory.md).
|
|
939
|
+
const clauses = ["n.status <> 'deleted'"];
|
|
742
940
|
const params = [];
|
|
743
|
-
if (match !== null) {
|
|
744
|
-
clauses.push(match.sql);
|
|
745
|
-
params.push(...match.params);
|
|
746
|
-
}
|
|
747
941
|
if (query.status !== undefined) {
|
|
748
942
|
clauses.push("n.status = ?");
|
|
749
943
|
params.push(query.status);
|
|
@@ -759,10 +953,28 @@ export class SqliteNoteStore {
|
|
|
759
953
|
.get(...params).n);
|
|
760
954
|
return { items: rows.map(toSummary), total, limit };
|
|
761
955
|
}
|
|
956
|
+
/** One page of an ordered id list, in the order the list gives. */
|
|
957
|
+
summaryPage(ordered, limit, offset) {
|
|
958
|
+
const page = ordered.slice(offset, offset + limit);
|
|
959
|
+
if (page.length === 0)
|
|
960
|
+
return { items: [], total: ordered.length, limit };
|
|
961
|
+
const rows = this.db
|
|
962
|
+
.prepare(`SELECT id, revision, title, summary, status, updated FROM notes
|
|
963
|
+
WHERE id IN (${page.map(() => "?").join(", ")})`)
|
|
964
|
+
.all(...page);
|
|
965
|
+
const byId = new Map(rows.map((row) => [toNumber(row.id), row]));
|
|
966
|
+
const items = page
|
|
967
|
+
.map((id) => byId.get(id))
|
|
968
|
+
.filter((row) => row !== undefined)
|
|
969
|
+
.map(toSummary);
|
|
970
|
+
return { items, total: ordered.length, limit };
|
|
971
|
+
}
|
|
762
972
|
summaryPageByIds(ordered, query, limit, offset) {
|
|
763
973
|
if (ordered.length === 0)
|
|
764
974
|
return { items: [], total: 0, limit };
|
|
765
|
-
|
|
975
|
+
// A delete leaves the row and marks it, and no search returns the note
|
|
976
|
+
// again, whichever lookup ordered it (docs/memory.md).
|
|
977
|
+
const clauses = [`status <> 'deleted'`, `id IN (${ordered.map(() => "?").join(", ")})`];
|
|
766
978
|
const params = [...ordered];
|
|
767
979
|
if (query.status !== undefined) {
|
|
768
980
|
clauses.push("status = ?");
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@4fu/arbor-notes-sqlite",
|
|
3
|
-
"version": "2026.
|
|
3
|
+
"version": "2026.1107.0",
|
|
4
4
|
"type": "module",
|
|
5
|
-
"description": "The default NoteStore: one copy of every note, checked writes, revisions, anchors
|
|
5
|
+
"description": "The default NoteStore: one copy of every note, checked writes, revisions, anchors, the active set and search on SQLite beside the journal: lexical, and semantic when the optional channel is installed.",
|
|
6
6
|
"exports": {
|
|
7
7
|
".": {
|
|
8
8
|
"types": "./index.d.ts",
|
|
@@ -25,7 +25,9 @@
|
|
|
25
25
|
"score.d.ts",
|
|
26
26
|
"score.js",
|
|
27
27
|
"search.d.ts",
|
|
28
|
-
"search.js"
|
|
28
|
+
"search.js",
|
|
29
|
+
"semantic.d.ts",
|
|
30
|
+
"semantic.js"
|
|
29
31
|
],
|
|
30
32
|
"engines": {
|
|
31
33
|
"node": ">=24"
|
|
@@ -40,8 +42,17 @@
|
|
|
40
42
|
"access": "public"
|
|
41
43
|
},
|
|
42
44
|
"dependencies": {
|
|
43
|
-
"@4fu/arbor-errors": "2026.
|
|
44
|
-
"@4fu/arbor-interfaces": "2026.
|
|
45
|
-
"@4fu/arbor-
|
|
45
|
+
"@4fu/arbor-errors": "2026.1107.0",
|
|
46
|
+
"@4fu/arbor-interfaces": "2026.1107.0",
|
|
47
|
+
"@4fu/arbor-lexical": "2026.1107.0",
|
|
48
|
+
"@4fu/arbor-prompt": "2026.1107.0"
|
|
49
|
+
},
|
|
50
|
+
"peerDependencies": {
|
|
51
|
+
"@4fu/arbor-search-zvec": "2026.1107.0"
|
|
52
|
+
},
|
|
53
|
+
"peerDependenciesMeta": {
|
|
54
|
+
"@4fu/arbor-search-zvec": {
|
|
55
|
+
"optional": true
|
|
56
|
+
}
|
|
46
57
|
}
|
|
47
58
|
}
|
package/schema.d.ts
CHANGED
|
@@ -23,12 +23,17 @@
|
|
|
23
23
|
* it froze them at, so a window's prefix never changes under it.
|
|
24
24
|
* - `notes_fts` - trigram index for exact substrings over title, summary and
|
|
25
25
|
* body.
|
|
26
|
+
* - `notes_words` - word index for BM25 over the same three columns, segmented
|
|
27
|
+
* (docs/journal.md#search).
|
|
26
28
|
*/
|
|
27
29
|
import type { DatabaseSync } from "node:sqlite";
|
|
30
|
+
import type { Segmenter } from "@4fu/arbor-interfaces";
|
|
28
31
|
/** Bumped when a change makes an existing database unusable. */
|
|
29
|
-
export declare const SCHEMA_VERSION =
|
|
32
|
+
export declare const SCHEMA_VERSION = 2;
|
|
30
33
|
/** The FTS5 table note search reads. */
|
|
31
34
|
export declare const FTS_TABLE = "notes_fts";
|
|
32
|
-
/**
|
|
33
|
-
export declare
|
|
35
|
+
/** The word index note search reads. */
|
|
36
|
+
export declare const WORD_TABLE = "notes_words";
|
|
37
|
+
/** Create the schema when it is missing, migrate an older one, and record the version. */
|
|
38
|
+
export declare function createSchema(db: DatabaseSync, segmenter?: Segmenter): void;
|
|
34
39
|
export declare function readMeta(db: DatabaseSync, key: string): string | null;
|
package/schema.js
CHANGED
|
@@ -23,11 +23,25 @@
|
|
|
23
23
|
* it froze them at, so a window's prefix never changes under it.
|
|
24
24
|
* - `notes_fts` - trigram index for exact substrings over title, summary and
|
|
25
25
|
* body.
|
|
26
|
+
* - `notes_words` - word index for BM25 over the same three columns, segmented
|
|
27
|
+
* (docs/journal.md#search).
|
|
26
28
|
*/
|
|
29
|
+
import { indexText, resolveSegmenter } from "@4fu/arbor-lexical";
|
|
27
30
|
/** Bumped when a change makes an existing database unusable. */
|
|
28
|
-
export const SCHEMA_VERSION =
|
|
31
|
+
export const SCHEMA_VERSION = 2;
|
|
29
32
|
/** The FTS5 table note search reads. */
|
|
30
33
|
export const FTS_TABLE = "notes_fts";
|
|
34
|
+
/** The word index note search reads. */
|
|
35
|
+
export const WORD_TABLE = "notes_words";
|
|
36
|
+
/** The trigram index: exact substrings, for Chinese and mixed text. */
|
|
37
|
+
const TRIGRAM_INDEX = `CREATE VIRTUAL TABLE IF NOT EXISTS ${FTS_TABLE}
|
|
38
|
+
USING fts5(title, summary, body, tokenize = 'trigram')`;
|
|
39
|
+
/**
|
|
40
|
+
* The word index: one column per searched field, each holding that field's
|
|
41
|
+
* segments, one per line. The columns are what the BM25 weights rank by.
|
|
42
|
+
*/
|
|
43
|
+
const WORD_INDEX = `CREATE VIRTUAL TABLE IF NOT EXISTS ${WORD_TABLE}
|
|
44
|
+
USING fts5(title, summary, body, tokenize = 'unicode61')`;
|
|
31
45
|
const STATEMENTS = [
|
|
32
46
|
`PRAGMA journal_mode = WAL`,
|
|
33
47
|
`PRAGMA foreign_keys = ON`,
|
|
@@ -114,25 +128,61 @@ const STATEMENTS = [
|
|
|
114
128
|
time INTEGER NOT NULL,
|
|
115
129
|
PRIMARY KEY (epoch, note)
|
|
116
130
|
)`,
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
USING fts5(title, summary, body, tokenize = 'trigram')`,
|
|
131
|
+
TRIGRAM_INDEX,
|
|
132
|
+
WORD_INDEX,
|
|
120
133
|
];
|
|
121
|
-
/**
|
|
122
|
-
|
|
134
|
+
/**
|
|
135
|
+
* Version 2 added the word index (docs/journal.md#search): a segmented word
|
|
136
|
+
* index with BM25 beside the trigram index, which is what lets a multi-word
|
|
137
|
+
* query find a note that holds the words apart from each other. Every note,
|
|
138
|
+
* every ref and the trigram index stay, so a version-1 database opens where it
|
|
139
|
+
* left off: the notes it already holds are segmented into the new index in the
|
|
140
|
+
* same step that records the version, and a delete leaves a note out of it.
|
|
141
|
+
*/
|
|
142
|
+
function migrateToVersion2(db, segmenter) {
|
|
143
|
+
// One transaction: the index and the version that says it is there land
|
|
144
|
+
// together, so a database that stops half way through is still the version it
|
|
145
|
+
// was and migrates again, whole, instead of half-indexing twice.
|
|
146
|
+
db.exec("BEGIN IMMEDIATE");
|
|
147
|
+
try {
|
|
148
|
+
db.exec(WORD_INDEX);
|
|
149
|
+
const insert = db.prepare(`INSERT INTO ${WORD_TABLE} (rowid, title, summary, body) VALUES (?, ?, ?, ?)`);
|
|
150
|
+
const rows = db
|
|
151
|
+
.prepare("SELECT id, title, summary, body FROM notes WHERE status <> 'deleted'")
|
|
152
|
+
.all();
|
|
153
|
+
for (const row of rows) {
|
|
154
|
+
insert.run(Number(row.id), indexText(row.title, segmenter), indexText(row.summary, segmenter), indexText(row.body, segmenter));
|
|
155
|
+
}
|
|
156
|
+
db.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES ('schema_version', ?)").run(String(SCHEMA_VERSION));
|
|
157
|
+
db.exec("COMMIT");
|
|
158
|
+
}
|
|
159
|
+
catch (error) {
|
|
160
|
+
try {
|
|
161
|
+
db.exec("ROLLBACK");
|
|
162
|
+
}
|
|
163
|
+
catch {
|
|
164
|
+
// The transaction is already gone; the original error is what matters.
|
|
165
|
+
}
|
|
166
|
+
throw error;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
/** Create the schema when it is missing, migrate an older one, and record the version. */
|
|
170
|
+
export function createSchema(db, segmenter = resolveSegmenter()) {
|
|
123
171
|
// The version lives in the one table that has to exist before anything else.
|
|
124
172
|
db.exec(`CREATE TABLE IF NOT EXISTS meta (
|
|
125
173
|
key TEXT PRIMARY KEY,
|
|
126
174
|
value TEXT NOT NULL
|
|
127
175
|
)`);
|
|
128
176
|
const version = readMeta(db, "schema_version");
|
|
129
|
-
if (version !== null && Number(version)
|
|
130
|
-
throw new Error(`notes schema ${version} is
|
|
177
|
+
if (version !== null && Number(version) > SCHEMA_VERSION) {
|
|
178
|
+
throw new Error(`notes schema ${version} is newer than this build (${SCHEMA_VERSION}); upgrade arbor, or migrate the database`);
|
|
131
179
|
}
|
|
180
|
+
if (Number(version) === SCHEMA_VERSION - 1)
|
|
181
|
+
migrateToVersion2(db, segmenter);
|
|
132
182
|
for (const statement of STATEMENTS)
|
|
133
183
|
db.exec(statement);
|
|
134
|
-
if (version === null) {
|
|
135
|
-
db.prepare("INSERT INTO meta (key, value) VALUES ('schema_version', ?)").run(String(SCHEMA_VERSION));
|
|
184
|
+
if (version === null || Number(version) !== SCHEMA_VERSION) {
|
|
185
|
+
db.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES ('schema_version', ?)").run(String(SCHEMA_VERSION));
|
|
136
186
|
}
|
|
137
187
|
}
|
|
138
188
|
export function readMeta(db, key) {
|
package/search.d.ts
CHANGED
|
@@ -1,27 +1,60 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Lexical search over notes (docs/journal.md#search, docs/memory.md).
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
4
|
+
* The channels the journal has, merged the same way, and one more when a
|
|
5
|
+
* deployment installed it:
|
|
6
|
+
*
|
|
7
|
+
* - a word index, segmented and ranked with BM25 over title, summary and body,
|
|
8
|
+
* the title and the summary weighted above the body;
|
|
9
|
+
* - a trigram index for exact substrings, which is what makes a Chinese
|
|
10
|
+
* substring findable;
|
|
11
|
+
* - a `LIKE` scan, for one- and two-character queries, which have no trigram of
|
|
12
|
+
* their own;
|
|
13
|
+
* - a semantic channel, note text embedded and searched by meaning, when
|
|
14
|
+
* `@4fu/arbor-search-zvec` is installed.
|
|
15
|
+
*
|
|
16
|
+
* Exact substring hits come first, then word hits that are not already there,
|
|
17
|
+
* then the hits only the semantic channel found, each channel by its own
|
|
18
|
+
* relevance. The `LIKE` scan has no relevance of its own, so it answers in id
|
|
19
|
+
* order and a page is byte-stable however it is paged.
|
|
10
20
|
*/
|
|
11
21
|
import type { DatabaseSync, SQLInputValue } from "node:sqlite";
|
|
12
|
-
|
|
13
|
-
|
|
22
|
+
import type { NoteStatus, Segmenter } from "@4fu/arbor-interfaces";
|
|
23
|
+
import { MIN_TRIGRAM_CHARS, escapeLike, likePattern, quoteFts } from "@4fu/arbor-lexical";
|
|
24
|
+
import type { Ranked } from "@4fu/arbor-lexical";
|
|
25
|
+
export { MIN_TRIGRAM_CHARS, escapeLike, likePattern, quoteFts };
|
|
26
|
+
/**
|
|
27
|
+
* The BM25 column weights both note indexes are ranked with, in the order the
|
|
28
|
+
* FTS5 tables carry their columns: title, summary, body. The title names the
|
|
29
|
+
* note and the summary is its one line, so a hit in either counts for more
|
|
30
|
+
* than one in the body: ten times as much for the title, four times for the
|
|
31
|
+
* summary.
|
|
32
|
+
*/
|
|
33
|
+
export declare const COLUMN_WEIGHTS = "10.0, 4.0, 1.0";
|
|
14
34
|
/** One condition over `notes n`, and the values it is bound to. */
|
|
15
35
|
export interface SubstringClause {
|
|
16
36
|
sql: string;
|
|
17
37
|
params: readonly SQLInputValue[];
|
|
18
38
|
}
|
|
19
|
-
/** The notes holding the query, as a condition over `notes n`. */
|
|
39
|
+
/** The notes holding the query as an exact substring, as a condition over `notes n`. */
|
|
20
40
|
export declare function substringClause(query: string): SubstringClause;
|
|
21
|
-
/**
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
41
|
+
/**
|
|
42
|
+
* The notes a text query matches, in the order search returns them: exact
|
|
43
|
+
* substring hits first, then word hits that are not already there, each channel
|
|
44
|
+
* by its own relevance, then the hits only the semantic channel found. A
|
|
45
|
+
* one- or two-character query has no trigram and no word match of its own, so
|
|
46
|
+
* the fallback scan answers it in id order, and the semantic channel's hits
|
|
47
|
+
* follow it.
|
|
48
|
+
*
|
|
49
|
+
* `semantic` is the channel's own candidates as ranks, in the order it gave
|
|
50
|
+
* them: the note store asks it for the notes nearest the query and applies its
|
|
51
|
+
* own filters to what comes back, so a channel that filters less than the store
|
|
52
|
+
* does still answers the read it was asked (docs/memory.md#search).
|
|
53
|
+
*/
|
|
54
|
+
export declare function matchedIds(db: DatabaseSync, query: string, segmenter: Segmenter, status?: NoteStatus, semantic?: readonly Ranked[]): number[];
|
|
55
|
+
/**
|
|
56
|
+
* The ids of several notes that hold the query: an exact substring or the
|
|
57
|
+
* query's words. `about` and `linked` bound a lookup to the notes the journal
|
|
58
|
+
* resolved, in their own weight order, so this answers a set and not an order.
|
|
59
|
+
*/
|
|
60
|
+
export declare function holdingIds(db: DatabaseSync, query: string, ids: readonly number[], segmenter: Segmenter): number[];
|
package/search.js
CHANGED
|
@@ -1,50 +1,127 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Lexical search over notes (docs/journal.md#search, docs/memory.md).
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
4
|
+
* The channels the journal has, merged the same way, and one more when a
|
|
5
|
+
* deployment installed it:
|
|
6
|
+
*
|
|
7
|
+
* - a word index, segmented and ranked with BM25 over title, summary and body,
|
|
8
|
+
* the title and the summary weighted above the body;
|
|
9
|
+
* - a trigram index for exact substrings, which is what makes a Chinese
|
|
10
|
+
* substring findable;
|
|
11
|
+
* - a `LIKE` scan, for one- and two-character queries, which have no trigram of
|
|
12
|
+
* their own;
|
|
13
|
+
* - a semantic channel, note text embedded and searched by meaning, when
|
|
14
|
+
* `@4fu/arbor-search-zvec` is installed.
|
|
15
|
+
*
|
|
16
|
+
* Exact substring hits come first, then word hits that are not already there,
|
|
17
|
+
* then the hits only the semantic channel found, each channel by its own
|
|
18
|
+
* relevance. The `LIKE` scan has no relevance of its own, so it answers in id
|
|
19
|
+
* order and a page is byte-stable however it is paged.
|
|
10
20
|
*/
|
|
11
|
-
import {
|
|
12
|
-
|
|
13
|
-
export
|
|
14
|
-
/**
|
|
21
|
+
import { MIN_TRIGRAM_CHARS, escapeLike, likePattern, mergeHits, quoteFts, trigramMatch, wordMatch, } from "@4fu/arbor-lexical";
|
|
22
|
+
import { FTS_TABLE, WORD_TABLE } from "./schema.js";
|
|
23
|
+
export { MIN_TRIGRAM_CHARS, escapeLike, likePattern, quoteFts };
|
|
24
|
+
/**
|
|
25
|
+
* The BM25 column weights both note indexes are ranked with, in the order the
|
|
26
|
+
* FTS5 tables carry their columns: title, summary, body. The title names the
|
|
27
|
+
* note and the summary is its one line, so a hit in either counts for more
|
|
28
|
+
* than one in the body: ten times as much for the title, four times for the
|
|
29
|
+
* summary.
|
|
30
|
+
*/
|
|
31
|
+
export const COLUMN_WEIGHTS = "10.0, 4.0, 1.0";
|
|
32
|
+
/** The `LIKE` scan's condition, for a query the trigram index cannot answer. */
|
|
33
|
+
function likeClause(query) {
|
|
34
|
+
const pattern = likePattern(query);
|
|
35
|
+
return {
|
|
36
|
+
sql: `(n.title LIKE ? ESCAPE '\\' OR n.summary LIKE ? ESCAPE '\\' OR n.body LIKE ? ESCAPE '\\')`,
|
|
37
|
+
params: [pattern, pattern, pattern],
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
/** The word index's condition: every one of the query's segments. */
|
|
41
|
+
function wordClause(tokens) {
|
|
42
|
+
return {
|
|
43
|
+
sql: `n.id IN (SELECT rowid FROM ${WORD_TABLE} WHERE ${WORD_TABLE} MATCH ?)`,
|
|
44
|
+
params: [wordMatch(tokens)],
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
/** The notes holding the query as an exact substring, as a condition over `notes n`. */
|
|
15
48
|
export function substringClause(query) {
|
|
16
49
|
if (Array.from(query).length >= MIN_TRIGRAM_CHARS) {
|
|
17
50
|
return {
|
|
18
51
|
sql: `n.id IN (SELECT rowid FROM ${FTS_TABLE} WHERE ${FTS_TABLE} MATCH ?)`,
|
|
19
|
-
params: [
|
|
52
|
+
params: [trigramMatch(query)],
|
|
20
53
|
};
|
|
21
54
|
}
|
|
22
|
-
|
|
23
|
-
return {
|
|
24
|
-
sql: `(n.title LIKE ? ESCAPE '\\' OR n.summary LIKE ? ESCAPE '\\' OR n.body LIKE ? ESCAPE '\\')`,
|
|
25
|
-
params: [pattern, pattern, pattern],
|
|
26
|
-
};
|
|
55
|
+
return likeClause(query);
|
|
27
56
|
}
|
|
28
|
-
/**
|
|
29
|
-
|
|
57
|
+
/** Every match from one index, by relevance, with the status filter applied. */
|
|
58
|
+
function ranked(db, table, match, status) {
|
|
59
|
+
const filtered = status === undefined ? "" : " AND n.status = ?";
|
|
60
|
+
const params = status === undefined ? [match] : [match, status];
|
|
61
|
+
const rows = db
|
|
62
|
+
.prepare(`SELECT f.rowid AS rowid, bm25(${table}, ${COLUMN_WEIGHTS}) AS rank
|
|
63
|
+
FROM ${table} f JOIN notes n ON n.id = f.rowid
|
|
64
|
+
WHERE ${table} MATCH ?${filtered}
|
|
65
|
+
ORDER BY rank, f.rowid`)
|
|
66
|
+
.all(...params);
|
|
67
|
+
return rows.map((row) => ({ rowid: Number(row.rowid), rank: Number(row.rank) }));
|
|
68
|
+
}
|
|
69
|
+
/** The notes a one- or two-character query holds, in id order. */
|
|
70
|
+
function scanIds(db, query, status) {
|
|
71
|
+
const clause = likeClause(query);
|
|
72
|
+
// A delete leaves the row and marks it, and the scan reads the notes table
|
|
73
|
+
// itself, so it leaves the deleted ones out on its own (docs/memory.md).
|
|
74
|
+
const filtered = status === undefined ? " AND n.status <> 'deleted'" : " AND n.status = ?";
|
|
75
|
+
const rows = db
|
|
76
|
+
.prepare(`SELECT n.id AS id FROM notes n WHERE ${clause.sql}${filtered} ORDER BY n.id`)
|
|
77
|
+
.all(...clause.params, ...(status === undefined ? [] : [status]));
|
|
78
|
+
return rows.map((row) => Number(row.id));
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* The notes a text query matches, in the order search returns them: exact
|
|
82
|
+
* substring hits first, then word hits that are not already there, each channel
|
|
83
|
+
* by its own relevance, then the hits only the semantic channel found. A
|
|
84
|
+
* one- or two-character query has no trigram and no word match of its own, so
|
|
85
|
+
* the fallback scan answers it in id order, and the semantic channel's hits
|
|
86
|
+
* follow it.
|
|
87
|
+
*
|
|
88
|
+
* `semantic` is the channel's own candidates as ranks, in the order it gave
|
|
89
|
+
* them: the note store asks it for the notes nearest the query and applies its
|
|
90
|
+
* own filters to what comes back, so a channel that filters less than the store
|
|
91
|
+
* does still answers the read it was asked (docs/memory.md#search).
|
|
92
|
+
*/
|
|
93
|
+
export function matchedIds(db, query, segmenter, status, semantic = []) {
|
|
94
|
+
if (Array.from(query).length < MIN_TRIGRAM_CHARS) {
|
|
95
|
+
// A one- or two-character query has no trigram and no word match of its
|
|
96
|
+
// own: the scan answers it in id order, and the semantic channel's hits
|
|
97
|
+
// follow it.
|
|
98
|
+
const scanned = scanIds(db, query, status).map((rowid) => ({ rowid, rank: 0 }));
|
|
99
|
+
return mergeHits({ channel: "substring", hits: scanned }, { channel: "semantic", hits: semantic }).map((hit) => hit.rowid);
|
|
100
|
+
}
|
|
101
|
+
const substrings = ranked(db, FTS_TABLE, trigramMatch(query), status);
|
|
102
|
+
const match = wordMatch(segmenter.segment(query));
|
|
103
|
+
const words = match === "" ? [] : ranked(db, WORD_TABLE, match, status);
|
|
104
|
+
return mergeHits({ channel: "substring", hits: substrings }, { channel: "word", hits: words }, { channel: "semantic", hits: semantic }).map((hit) => hit.rowid);
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* The ids of several notes that hold the query: an exact substring or the
|
|
108
|
+
* query's words. `about` and `linked` bound a lookup to the notes the journal
|
|
109
|
+
* resolved, in their own weight order, so this answers a set and not an order.
|
|
110
|
+
*/
|
|
111
|
+
export function holdingIds(db, query, ids, segmenter) {
|
|
30
112
|
if (ids.length === 0)
|
|
31
113
|
return [];
|
|
32
|
-
const
|
|
114
|
+
const clauses = [substringClause(query)];
|
|
115
|
+
if (Array.from(query).length >= MIN_TRIGRAM_CHARS) {
|
|
116
|
+
const tokens = segmenter.segment(query);
|
|
117
|
+
if (wordMatch(tokens) !== "")
|
|
118
|
+
clauses.push(wordClause(tokens));
|
|
119
|
+
}
|
|
33
120
|
const rows = db
|
|
34
121
|
.prepare(`SELECT n.id AS id FROM notes n
|
|
35
|
-
WHERE
|
|
122
|
+
WHERE (${clauses.map((clause) => clause.sql).join(" OR ")})
|
|
123
|
+
AND n.id IN (${ids.map(() => "?").join(", ")})
|
|
36
124
|
ORDER BY n.id`)
|
|
37
|
-
.all(...clause.params, ...ids);
|
|
125
|
+
.all(...clauses.flatMap((clause) => clause.params), ...ids);
|
|
38
126
|
return rows.map((row) => Number(row.id));
|
|
39
127
|
}
|
|
40
|
-
/** Escape a literal for `LIKE ... ESCAPE '\'`. */
|
|
41
|
-
export function escapeLike(value) {
|
|
42
|
-
return value.replace(/[\\%_]/g, (character) => `\\${character}`);
|
|
43
|
-
}
|
|
44
|
-
export function likePattern(query) {
|
|
45
|
-
return `%${escapeLike(query)}%`;
|
|
46
|
-
}
|
|
47
|
-
/** Quote one phrase for an FTS5 `MATCH`, so outside text is never syntax. */
|
|
48
|
-
export function quoteFts(value) {
|
|
49
|
-
return `"${value.replace(/"/g, '""')}"`;
|
|
50
|
-
}
|
package/semantic.d.ts
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The semantic channel the note store uses when the caller names none
|
|
3
|
+
* (docs/memory.md#search).
|
|
4
|
+
*
|
|
5
|
+
* The spec calls for note text to be embedded and searched by meaning beside the
|
|
6
|
+
* lexical indexes, and the default is zvec when `@4fu/arbor-search-zvec` is
|
|
7
|
+
* installed: `resolveSemanticIndex` loads it, and the store uses what it returns
|
|
8
|
+
* when the caller names no channel. The package is an optional peer dependency,
|
|
9
|
+
* so a deployment that leaves it out keeps working.
|
|
10
|
+
*
|
|
11
|
+
* What is left here is the absence that answers when the package is not there:
|
|
12
|
+
* no channel, and search is exactly lexical. That is also the answer when the
|
|
13
|
+
* package is installed but its vector store will not open - zvec's prebuilt
|
|
14
|
+
* binary does not load on this platform, or another host holds the collection -
|
|
15
|
+
* and when no embedding model is available, because the package says so by
|
|
16
|
+
* handing back nothing.
|
|
17
|
+
*/
|
|
18
|
+
import type { SemanticIndex } from "@4fu/arbor-interfaces";
|
|
19
|
+
/**
|
|
20
|
+
* The semantic channel a note store uses when the caller names none: the zvec
|
|
21
|
+
* one when the package is installed and its store opens, and none otherwise.
|
|
22
|
+
*
|
|
23
|
+
* `load` replaces how the zvec package is loaded. A host that resolves modules
|
|
24
|
+
* its own way passes its own; so does a test that has no zvec package to load. A
|
|
25
|
+
* load that fails falls back rather than stopping the store, because the package
|
|
26
|
+
* is optional: it may be missing, or its prebuilt binary may not load on this
|
|
27
|
+
* platform.
|
|
28
|
+
*/
|
|
29
|
+
export declare function resolveSemanticIndex(path: string, load?: (path: string) => SemanticIndex | null): SemanticIndex | null;
|
package/semantic.js
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The semantic channel the note store uses when the caller names none
|
|
3
|
+
* (docs/memory.md#search).
|
|
4
|
+
*
|
|
5
|
+
* The spec calls for note text to be embedded and searched by meaning beside the
|
|
6
|
+
* lexical indexes, and the default is zvec when `@4fu/arbor-search-zvec` is
|
|
7
|
+
* installed: `resolveSemanticIndex` loads it, and the store uses what it returns
|
|
8
|
+
* when the caller names no channel. The package is an optional peer dependency,
|
|
9
|
+
* so a deployment that leaves it out keeps working.
|
|
10
|
+
*
|
|
11
|
+
* What is left here is the absence that answers when the package is not there:
|
|
12
|
+
* no channel, and search is exactly lexical. That is also the answer when the
|
|
13
|
+
* package is installed but its vector store will not open - zvec's prebuilt
|
|
14
|
+
* binary does not load on this platform, or another host holds the collection -
|
|
15
|
+
* and when no embedding model is available, because the package says so by
|
|
16
|
+
* handing back nothing.
|
|
17
|
+
*/
|
|
18
|
+
import { createRequire } from "node:module";
|
|
19
|
+
/** The package that holds the zvec semantic index, when a deployment installed it. */
|
|
20
|
+
const ZVEC_PACKAGE = "@4fu/arbor-search-zvec";
|
|
21
|
+
/**
|
|
22
|
+
* Load the zvec semantic index over a store beside the notes database's own, or
|
|
23
|
+
* `null` when it is not there: the package is not installed, or it holds no
|
|
24
|
+
* index.
|
|
25
|
+
*/
|
|
26
|
+
function loadZvec(path) {
|
|
27
|
+
const require = createRequire(import.meta.url);
|
|
28
|
+
const loaded = require(ZVEC_PACKAGE);
|
|
29
|
+
const create = loaded?.createZvecIndex;
|
|
30
|
+
if (typeof create !== "function")
|
|
31
|
+
return null;
|
|
32
|
+
const index = create({ path });
|
|
33
|
+
if (index === null || typeof index !== "object")
|
|
34
|
+
return null;
|
|
35
|
+
const shape = index;
|
|
36
|
+
if (typeof shape.add !== "function" || typeof shape.search !== "function")
|
|
37
|
+
return null;
|
|
38
|
+
return index;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* The semantic channel a note store uses when the caller names none: the zvec
|
|
42
|
+
* one when the package is installed and its store opens, and none otherwise.
|
|
43
|
+
*
|
|
44
|
+
* `load` replaces how the zvec package is loaded. A host that resolves modules
|
|
45
|
+
* its own way passes its own; so does a test that has no zvec package to load. A
|
|
46
|
+
* load that fails falls back rather than stopping the store, because the package
|
|
47
|
+
* is optional: it may be missing, or its prebuilt binary may not load on this
|
|
48
|
+
* platform.
|
|
49
|
+
*/
|
|
50
|
+
export function resolveSemanticIndex(path, load = loadZvec) {
|
|
51
|
+
try {
|
|
52
|
+
return load(path);
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
return null;
|
|
56
|
+
}
|
|
57
|
+
}
|