@4fu/arbor-notes-sqlite 2026.1106.1 → 2026.1107.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -56,7 +56,9 @@ npm install @4fu/arbor-core @4fu/arbor-journal-sqlite @4fu/arbor-notes-sqlite \
56
56
  ```
57
57
 
58
58
  `@4fu/arbor-plugin-http` is optional; it is the example plugin below.
59
- `@4fu/arbor-segmenter-jieba` is an optional peer of the journal.
59
+ `@4fu/arbor-segmenter-jieba` is an optional peer of the lexical package, and
60
+ `@4fu/arbor-search-zvec` an optional peer of the journal: absent either one, the
61
+ journal searches lexically and nothing else changes.
60
62
 
61
63
  ## A minimal host
62
64
 
@@ -120,6 +122,8 @@ window gets, and how a continuation keeps working in it, are in
120
122
  | [`@4fu/arbor-journal-sqlite`](packages/journal-sqlite) | the default `Journal` and `BlobStore` on SQLite |
121
123
  | [`@4fu/arbor-notes-sqlite`](packages/notes-sqlite) | the default `NoteStore` on SQLite, beside the journal |
122
124
  | [`@4fu/arbor-segmenter-jieba`](packages/segmenter-jieba) | the default `Segmenter`: jieba's dictionary segmentation for the word index, optional |
125
+ | [`@4fu/arbor-lexical`](packages/lexical) | the lexical search the journal and notes share: word segmentation, the FTS5 helpers and the merge order |
126
+ | [`@4fu/arbor-search-zvec`](packages/search-zvec) | the optional semantic channel: event text embedded and searched by meaning with zvec |
123
127
  | [`@4fu/arbor-workspace-dir`](packages/workspace-dir) | the fallback `Workspace`: one directory per window, with a continuation working in its predecessor's directory in place |
124
128
  | [`@4fu/arbor-workspace-ovnfs`](packages/workspace-ovnfs) | the default `Workspace` when ovnfs is installed: one shared folder, per-window copy-on-write checkouts the model opens and applies back |
125
129
  | [`@4fu/arbor-environment-dir`](packages/environment-dir) | the default `Environment`: one `/env` directory, a revision per item, validation and deltas |
@@ -171,6 +175,7 @@ tarball would contain ([docs/release.md](docs/release.md)).
171
175
  | `packages/errors` | the error shape every refusal uses: the rule and the next action |
172
176
  | `packages/journal-sqlite` | the default `Journal` and `BlobStore` on SQLite |
173
177
  | `packages/segmenter-jieba` | the default `Segmenter`: jieba's dictionary segmentation for the word index, optional |
178
+ | `packages/lexical` | the lexical search the journal and notes share: word segmentation, the FTS5 helpers and the merge order |
174
179
  | `packages/notes-sqlite` | the default `NoteStore` on SQLite, beside the journal |
175
180
  | `packages/judge-jev` | the default `Judge`: Jev, TypeSafe AI's System One classifier, answering the dispatch decisions |
176
181
  | `packages/workspace-dir` | the default `Workspace`: one directory per window, with a continuation working in its predecessor's directory in place, and the lookup of another window's directory it contributes to the catalog |
package/index.d.ts CHANGED
@@ -4,10 +4,12 @@
4
4
  export { SqliteNoteStore, SUMMARY_LIMIT, DEFAULT_BODY_LIMIT, MAX_BODY_LIMIT } from "./notes.ts";
5
5
  export { DEFAULT_MIN_INTERVAL_MS, DEFAULT_ACTIVE_BUDGET, DEFAULT_JUDGE_SHARE, DEFAULT_HALF_LIFE_MS, deriveSummary, } from "./notes.ts";
6
6
  export type { SqliteNoteStoreOptions } from "./notes.ts";
7
- export { SCHEMA_VERSION, FTS_TABLE, createSchema } from "./schema.ts";
7
+ export { SCHEMA_VERSION, FTS_TABLE, WORD_TABLE, createSchema } from "./schema.ts";
8
8
  export { noteIdOf, noteRefOf, parseNoteId, parseNoteRef, noteRefForms, storedWindowRef, storedEventRef, } from "./refs.ts";
9
+ export { fallbackSegmenter, resolveSegmenter, indexText } from "@4fu/arbor-lexical";
9
10
  export { USE_WEIGHTS, MAX_CREDIT_HOPS, decay, estimateTokens, linkedCredit, loadCredit, loadEdges, pickByBudget, applyJudgeShare, scoreNotes, } from "./score.ts";
10
11
  export type { ScoredNote } from "./score.ts";
11
12
  export { MIN_TRIGRAM_CHARS, substringClause, holdingIds, escapeLike, likePattern, quoteFts, } from "./search.ts";
12
13
  export type { SubstringClause } from "./search.ts";
13
14
  export { bodyDiff } from "./diff.ts";
15
+ export { resolveSemanticIndex } from "./semantic.ts";
package/index.js CHANGED
@@ -3,8 +3,10 @@
3
3
  */
4
4
  export { SqliteNoteStore, SUMMARY_LIMIT, DEFAULT_BODY_LIMIT, MAX_BODY_LIMIT } from "./notes.js";
5
5
  export { DEFAULT_MIN_INTERVAL_MS, DEFAULT_ACTIVE_BUDGET, DEFAULT_JUDGE_SHARE, DEFAULT_HALF_LIFE_MS, deriveSummary, } from "./notes.js";
6
- export { SCHEMA_VERSION, FTS_TABLE, createSchema } from "./schema.js";
6
+ export { SCHEMA_VERSION, FTS_TABLE, WORD_TABLE, createSchema } from "./schema.js";
7
7
  export { noteIdOf, noteRefOf, parseNoteId, parseNoteRef, noteRefForms, storedWindowRef, storedEventRef, } from "./refs.js";
8
+ export { fallbackSegmenter, resolveSegmenter, indexText } from "@4fu/arbor-lexical";
8
9
  export { USE_WEIGHTS, MAX_CREDIT_HOPS, decay, estimateTokens, linkedCredit, loadCredit, loadEdges, pickByBudget, applyJudgeShare, scoreNotes, } from "./score.js";
9
10
  export { MIN_TRIGRAM_CHARS, substringClause, holdingIds, escapeLike, likePattern, quoteFts, } from "./search.js";
10
11
  export { bodyDiff } from "./diff.js";
12
+ export { resolveSemanticIndex } from "./semantic.js";
package/notes.d.ts CHANGED
@@ -15,7 +15,11 @@
15
15
  * count and token budget, and frozen with the epoch's version, so every
16
16
  * window opened in one epoch has the same prefix and later windows see the
17
17
  * changes as a delta;
18
- * - every write recorded in the journal as a `note_revision` event.
18
+ * - every write recorded in the journal as a `note_revision` event;
19
+ * - lexical search: a word index with BM25, a trigram index for exact
20
+ * substrings, and a `LIKE` scan for one- and two-character queries, merged
21
+ * with exact substring hits first - and a semantic channel, note text embedded
22
+ * and searched by meaning, when `@4fu/arbor-search-zvec` is installed.
19
23
  *
20
24
  * It is synchronous inside: one `DatabaseSync`, one transaction per write, so
21
25
  * the revision a write checks and the revision it writes are one atomic step.
@@ -23,7 +27,7 @@
23
27
  * database and the journal. The interface is asynchronous because a `NoteStore`
24
28
  * may live behind a process boundary.
25
29
  */
26
- import type { ActiveCandidate, ActiveSetBudget, Journal, Note, NoteBody, NoteContext, NoteDelta, NoteQuery, NoteRef, NoteRevision, NoteSummary, NoteUseInput, NoteWriteInput, NotesSnapshot, NoteStore, Page, SnapshotOptions } from "@4fu/arbor-interfaces";
30
+ import type { ActiveCandidate, ActiveSetBudget, Journal, Note, NoteBody, NoteContext, NoteDelta, NoteQuery, NoteRef, NoteRevision, NoteSummary, NoteUseInput, NoteWriteInput, NotesSnapshot, NoteStore, Page, Segmenter, SemanticIndex, SnapshotOptions } from "@4fu/arbor-interfaces";
27
31
  /** A summary is at most this many characters (docs/memory.md). */
28
32
  export declare const SUMMARY_LIMIT = 200;
29
33
  /** How many characters of a body one read returns. */
@@ -46,6 +50,16 @@ export interface SqliteNoteStoreOptions {
46
50
  now?: () => number;
47
51
  /** Usage loses half of its weight this often. 24 hours. */
48
52
  usageHalfLifeMs?: number;
53
+ /** Word segmentation for the word index. Defaults to jieba, or the approximation. */
54
+ segmenter?: Segmenter;
55
+ /**
56
+ * The semantic channel: note text embedded and searched by meaning
57
+ * (docs/memory.md#search). Omitted, the zvec channel is used when
58
+ * `@4fu/arbor-search-zvec` is installed and its store opens, and there is no
59
+ * channel otherwise. `null` forces lexical search even when the package is
60
+ * installed.
61
+ */
62
+ semantic?: SemanticIndex | null;
49
63
  /**
50
64
  * One fixed string from the deployment's text table, so this implementation
51
65
  * writes none of its own (docs/prompting.md#language-and-the-text-table).
@@ -60,11 +74,17 @@ export declare class SqliteNoteStore implements NoteStore {
60
74
  private readonly db;
61
75
  private readonly journal;
62
76
  private readonly clock;
77
+ private readonly segmenter;
63
78
  private readonly halfLifeMs;
79
+ private readonly semantic;
80
+ /** The temporary directory an in-memory store's vector collection lives in. */
81
+ private readonly ownedVectorDirectory;
64
82
  /** The table every refusal this store raises is written from. */
65
83
  private readonly text;
66
84
  /** One write at a time: a write spans the notes database and the journal. */
67
85
  private writeQueue;
86
+ private semanticReady;
87
+ private semanticCaughtUp;
68
88
  private closed;
69
89
  constructor(options: SqliteNoteStoreOptions);
70
90
  write(input: NoteWriteInput): Promise<Note>;
@@ -100,7 +120,44 @@ export declare class SqliteNoteStore implements NoteStore {
100
120
  }): Promise<Page<ActiveCandidate>>;
101
121
  recordUse(input: NoteUseInput): Promise<void>;
102
122
  close(): Promise<void>;
123
+ /**
124
+ * Index the notes the semantic channel does not hold yet, and say how many.
125
+ *
126
+ * A restart over an existing database, a channel that was not ready when the
127
+ * notes were written, and a write whose indexing failed all leave notes to
128
+ * catch up on. A channel that is not ready yet holds what it was given and
129
+ * indexes that itself, so this answers 0 until it is.
130
+ */
131
+ catchUpSemantic(): Promise<number>;
103
132
  private ensureOpen;
133
+ /**
134
+ * Hand one note's text to the semantic channel, when there is one.
135
+ *
136
+ * The note is embedded as one text: its title, its summary and its body, in
137
+ * the order the lexical index weights them. The vector channel holds one
138
+ * vector per note and cannot weight a note's parts against each other the way
139
+ * BM25 weights the three columns, so the title - the note's name - leads the
140
+ * text and the body follows it, and a query that means either reaches the
141
+ * note (docs/memory.md#search).
142
+ */
143
+ private addToSemantic;
144
+ /** Take one note's text back out of the semantic channel, as a delete does. */
145
+ private removeFromSemantic;
146
+ /**
147
+ * The semantic channel's candidates, as ranks in the order it gave them.
148
+ *
149
+ * An absent channel, one that is not ready yet, and one that refuses all
150
+ * answer with nothing, and the search is exactly the lexical one. The note
151
+ * read's own filters decide which candidates count, so a channel that filters
152
+ * less than the store does still answers the read it was asked for.
153
+ */
154
+ private semanticRanks;
155
+ /**
156
+ * The notes among `ids` the read's own filters allow, in the order given. A
157
+ * deleted note is not among them, so a vector that outlived its note never
158
+ * reaches a page.
159
+ */
160
+ private allowedNoteIds;
104
161
  private noteRow;
105
162
  private revisionRow;
106
163
  private latestRevisionRow;
@@ -108,6 +165,8 @@ export declare class SqliteNoteStore implements NoteStore {
108
165
  /** The current summary and revision of several notes, in one read. */
109
166
  private noteStatesFor;
110
167
  private summaryPageBySql;
168
+ /** One page of an ordered id list, in the order the list gives. */
169
+ private summaryPage;
111
170
  private summaryPageByIds;
112
171
  /**
113
172
  * The notes about a target, or linked to a ref: the notes the events on that
package/notes.js CHANGED
@@ -15,7 +15,11 @@
15
15
  * count and token budget, and frozen with the epoch's version, so every
16
16
  * window opened in one epoch has the same prefix and later windows see the
17
17
  * changes as a delta;
18
- * - every write recorded in the journal as a `note_revision` event.
18
+ * - every write recorded in the journal as a `note_revision` event;
19
+ * - lexical search: a word index with BM25, a trigram index for exact
20
+ * substrings, and a `LIKE` scan for one- and two-character queries, merged
21
+ * with exact substring hits first - and a semantic channel, note text embedded
22
+ * and searched by meaning, when `@4fu/arbor-search-zvec` is installed.
19
23
  *
20
24
  * It is synchronous inside: one `DatabaseSync`, one transaction per write, so
21
25
  * the revision a write checks and the revision it writes are one atomic step.
@@ -24,13 +28,19 @@
24
28
  * may live behind a process boundary.
25
29
  */
26
30
  import { DatabaseSync } from "node:sqlite";
31
+ import { randomUUID } from "node:crypto";
32
+ import { rm } from "node:fs/promises";
33
+ import { tmpdir } from "node:os";
34
+ import { join } from "node:path";
27
35
  import { arborError, isArborError } from "@4fu/arbor-errors";
28
36
  import { DEFAULT_PAGE_LIMIT, MAX_PAGE_LIMIT } from "@4fu/arbor-interfaces";
37
+ import { indexText, resolveSegmenter } from "@4fu/arbor-lexical";
29
38
  import { bodyDiff } from "./diff.js";
30
39
  import { noteIdOf, noteRefOf, noteRefForms, parseNoteId, parseNoteRef, storedEventRef, storedWindowRef, } from "./refs.js";
31
- import { FTS_TABLE, createSchema } from "./schema.js";
40
+ import { FTS_TABLE, WORD_TABLE, createSchema } from "./schema.js";
32
41
  import { USE_WEIGHTS, applyJudgeShare, pickByBudget, scoreNotes } from "./score.js";
33
- import { holdingIds, substringClause } from "./search.js";
42
+ import { holdingIds, matchedIds } from "./search.js";
43
+ import { resolveSemanticIndex } from "./semantic.js";
34
44
  import { textOf, zhTextTable } from "@4fu/arbor-prompt";
35
45
  /** A summary is at most this many characters (docs/memory.md). */
36
46
  export const SUMMARY_LIMIT = 200;
@@ -47,6 +57,13 @@ export const DEFAULT_JUDGE_SHARE = 0.15;
47
57
  export const DEFAULT_HALF_LIFE_MS = 86_400_000;
48
58
  /** How many events one `about` lookup reads. */
49
59
  const ABOUT_EVENT_LIMIT = 200;
60
+ /**
61
+ * The fewest candidates the semantic channel is asked for. A page of one still
62
+ * fuses the channels over a handful of candidates, so a hit the word index
63
+ * ranked first is not lost to a semantic hit the channel would have ranked
64
+ * second.
65
+ */
66
+ const MIN_SEMANTIC_CANDIDATES = 10;
50
67
  function toNumber(value) {
51
68
  if (typeof value === "number")
52
69
  return value;
@@ -164,20 +181,38 @@ export class SqliteNoteStore {
164
181
  db;
165
182
  journal;
166
183
  clock;
184
+ segmenter;
167
185
  halfLifeMs;
186
+ semantic;
187
+ /** The temporary directory an in-memory store's vector collection lives in. */
188
+ ownedVectorDirectory;
168
189
  /** The table every refusal this store raises is written from. */
169
190
  text;
170
191
  /** One write at a time: a write spans the notes database and the journal. */
171
192
  writeQueue = Promise.resolve();
193
+ semanticReady = false;
194
+ semanticCaughtUp = false;
172
195
  closed = false;
173
196
  constructor(options) {
174
197
  this.path = options.path ?? ":memory:";
175
198
  this.db = new DatabaseSync(this.path);
176
- createSchema(this.db);
199
+ this.segmenter = options.segmenter ?? resolveSegmenter();
200
+ // The segmenter is resolved first: a version-1 database migrates by
201
+ // segmenting the notes it already holds into the word index.
202
+ createSchema(this.db, this.segmenter);
177
203
  this.journal = options.journal;
178
204
  this.clock = options.now ?? (() => Date.now());
179
205
  this.halfLifeMs = options.usageHalfLifeMs ?? DEFAULT_HALF_LIFE_MS;
180
206
  this.text = options.text ?? ((key, args) => textOf(zhTextTable, key, args));
207
+ // An in-memory store has no directory of its own; give it a temporary one
208
+ // and take it away on close. The semantic channel's store lives under it,
209
+ // beside where the notes database would be.
210
+ const root = this.path === ":memory:" ? join(tmpdir(), `arbor-notes-${randomUUID()}`) : null;
211
+ this.semantic =
212
+ options.semantic === undefined
213
+ ? resolveSemanticIndex(root === null ? `${this.path}.vectors` : join(root, "vectors"))
214
+ : options.semantic;
215
+ this.ownedVectorDirectory = root;
181
216
  }
182
217
  // --- writes -------------------------------------------------------------
183
218
  async write(input) {
@@ -233,6 +268,17 @@ export class SqliteNoteStore {
233
268
  .run(event.ref, written.id, written.revision);
234
269
  this.db.prepare("UPDATE notes SET event = ? WHERE id = ?").run(event.ref, written.id);
235
270
  });
271
+ // The semantic channel is asynchronous, so the note is indexed once its
272
+ // revision is committed: a write never waits on a model, and a channel that
273
+ // is not ready yet holds the text until it is. With no channel there is
274
+ // nothing to wait for, and the write takes the path it always did. A delete
275
+ // takes the note's vector back out, as a redaction does in the journal.
276
+ if (this.semantic !== null) {
277
+ if (written.deleted)
278
+ await this.removeFromSemantic(written.id);
279
+ else
280
+ await this.addToSemantic(written.id, written.title, written.summary, written.body);
281
+ }
236
282
  return toNote(this.noteRow(written.id));
237
283
  }
238
284
  /** The rules a write has to meet before it touches the one copy. */
@@ -375,12 +421,16 @@ export class SqliteNoteStore {
375
421
  .prepare(`INSERT INTO note_revisions (note, revision, title, summary, body, deleted, time, window, event, anchor, refs)
376
422
  VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?)`)
377
423
  .run(id, revision, input.title, summary, body, deleted ? 1 : 0, time, window, anchorJson, refs);
378
- // The search index holds the current state; a deleted note is not found.
424
+ // The search indexes hold the current state; a deleted note is not found.
379
425
  this.db.prepare(`DELETE FROM ${FTS_TABLE} WHERE rowid = ?`).run(id);
426
+ this.db.prepare(`DELETE FROM ${WORD_TABLE} WHERE rowid = ?`).run(id);
380
427
  if (!deleted) {
381
428
  this.db
382
429
  .prepare(`INSERT INTO ${FTS_TABLE} (rowid, title, summary, body) VALUES (?, ?, ?, ?)`)
383
430
  .run(id, input.title, summary, body);
431
+ this.db
432
+ .prepare(`INSERT INTO ${WORD_TABLE} (rowid, title, summary, body) VALUES (?, ?, ?, ?)`)
433
+ .run(id, indexText(input.title, this.segmenter), indexText(summary, this.segmenter), indexText(body, this.segmenter));
384
434
  }
385
435
  // The refs the revision names: what it cites and what it holds as
386
436
  // evidence. The evidence ones between notes are what the credit walk
@@ -403,6 +453,8 @@ export class SqliteNoteStore {
403
453
  time,
404
454
  title: input.title,
405
455
  summary,
456
+ body,
457
+ deleted,
406
458
  window,
407
459
  anchor,
408
460
  };
@@ -417,6 +469,7 @@ export class SqliteNoteStore {
417
469
  .prepare("DELETE FROM note_refs WHERE note = ? AND revision = ?")
418
470
  .run(written.id, written.revision);
419
471
  this.db.prepare(`DELETE FROM ${FTS_TABLE} WHERE rowid = ?`).run(written.id);
472
+ this.db.prepare(`DELETE FROM ${WORD_TABLE} WHERE rowid = ?`).run(written.id);
420
473
  if (written.created) {
421
474
  this.db.prepare("DELETE FROM notes WHERE id = ?").run(written.id);
422
475
  return;
@@ -431,6 +484,9 @@ export class SqliteNoteStore {
431
484
  this.db
432
485
  .prepare(`INSERT INTO ${FTS_TABLE} (rowid, title, summary, body) VALUES (?, ?, ?, ?)`)
433
486
  .run(written.id, previous.title, previous.summary, previous.body);
487
+ this.db
488
+ .prepare(`INSERT INTO ${WORD_TABLE} (rowid, title, summary, body) VALUES (?, ?, ?, ?)`)
489
+ .run(written.id, indexText(previous.title, this.segmenter), indexText(previous.summary, this.segmenter), indexText(previous.body, this.segmenter));
434
490
  }
435
491
  });
436
492
  }
@@ -511,16 +567,25 @@ export class SqliteNoteStore {
511
567
  const offset = Math.max(0, Math.floor(query.offset ?? 0));
512
568
  const text = (query.query ?? "").trim();
513
569
  // `about` and `linked` bound the result to the notes the journal resolved,
514
- // in weight order; a text match narrows that set.
570
+ // in weight order; a text match narrows that set. The semantic channel does
571
+ // not widen a bound the journal resolved, so it is not asked at all here.
515
572
  if (query.about !== undefined || query.linked !== undefined) {
516
573
  const ordered = await this.linkedNoteIds(query.about, query.linked);
517
- const holding = text === "" ? null : new Set(holdingIds(this.db, text, ordered));
574
+ const holding = text === "" ? null : new Set(holdingIds(this.db, text, ordered, this.segmenter));
518
575
  const chosen = holding === null ? ordered : ordered.filter((id) => holding.has(id));
519
576
  const page = this.summaryPageByIds(chosen, query, limit, offset);
520
577
  return { ...page, nextCall: searchCall(query, offset, limit) };
521
578
  }
522
- const match = text === "" ? null : substringClause(text);
523
- const page = this.summaryPageBySql(query, limit, offset, match);
579
+ // No query lists every note by id; a query is answered by the lexical
580
+ // channels and the semantic one, merged: exact substring hits first, then
581
+ // word hits, then the hits only the semantic channel found.
582
+ const semantic = text === "" ? [] : await this.semanticRanks(text, limit, query.status);
583
+ const matched = text === ""
584
+ ? null
585
+ : matchedIds(this.db, text, this.segmenter, query.status, semantic);
586
+ const page = matched === null
587
+ ? this.summaryPageBySql(query, limit, offset)
588
+ : this.summaryPage(matched, limit, offset);
524
589
  return { ...page, nextCall: searchCall(query, offset, limit) };
525
590
  }
526
591
  async active() {
@@ -686,6 +751,39 @@ export class SqliteNoteStore {
686
751
  this.closed = true;
687
752
  await this.writeQueue;
688
753
  this.db.close();
754
+ // The channel's own store is closed with the notes database, and the
755
+ // temporary directory an in-memory store gave it goes with it.
756
+ if (this.semantic !== null)
757
+ await this.semantic.close();
758
+ if (this.ownedVectorDirectory !== null) {
759
+ await rm(this.ownedVectorDirectory, { recursive: true, force: true });
760
+ }
761
+ }
762
+ /**
763
+ * Index the notes the semantic channel does not hold yet, and say how many.
764
+ *
765
+ * A restart over an existing database, a channel that was not ready when the
766
+ * notes were written, and a write whose indexing failed all leave notes to
767
+ * catch up on. A channel that is not ready yet holds what it was given and
768
+ * indexes that itself, so this answers 0 until it is.
769
+ */
770
+ async catchUpSemantic() {
771
+ this.ensureOpen();
772
+ if (this.semantic === null || !(await this.semantic.ready()))
773
+ return 0;
774
+ const held = await this.semantic.held();
775
+ const rows = this.db
776
+ .prepare(`SELECT id, title, summary, body FROM notes WHERE status <> 'deleted' ORDER BY id`)
777
+ .all();
778
+ let indexed = 0;
779
+ for (const row of rows) {
780
+ const id = toNumber(row.id);
781
+ if (held.has(noteIdOf(id)))
782
+ continue;
783
+ await this.addToSemantic(id, row.title, row.summary, row.body);
784
+ indexed += 1;
785
+ }
786
+ return indexed;
689
787
  }
690
788
  // --- internals ---------------------------------------------------------
691
789
  ensureOpen() {
@@ -698,6 +796,104 @@ export class SqliteNoteStore {
698
796
  });
699
797
  }
700
798
  }
799
+ /**
800
+ * Hand one note's text to the semantic channel, when there is one.
801
+ *
802
+ * The note is embedded as one text: its title, its summary and its body, in
803
+ * the order the lexical index weights them. The vector channel holds one
804
+ * vector per note and cannot weight a note's parts against each other the way
805
+ * BM25 weights the three columns, so the title - the note's name - leads the
806
+ * text and the body follows it, and a query that means either reaches the
807
+ * note (docs/memory.md#search).
808
+ */
809
+ async addToSemantic(id, title, summary, body) {
810
+ if (this.semantic === null)
811
+ return;
812
+ try {
813
+ await this.semantic.add(noteIdOf(id), [title, summary, body].join("\n"));
814
+ }
815
+ catch {
816
+ // An optional channel that refuses leaves the note searchable lexically;
817
+ // a catch-up indexes it once the channel answers again.
818
+ }
819
+ }
820
+ /** Take one note's text back out of the semantic channel, as a delete does. */
821
+ async removeFromSemantic(id) {
822
+ if (this.semantic === null)
823
+ return;
824
+ try {
825
+ await this.semantic.remove(noteIdOf(id));
826
+ }
827
+ catch {
828
+ // As with a write: the channel is optional, and the note is gone from
829
+ // every index the store owns.
830
+ }
831
+ }
832
+ /**
833
+ * The semantic channel's candidates, as ranks in the order it gave them.
834
+ *
835
+ * An absent channel, one that is not ready yet, and one that refuses all
836
+ * answer with nothing, and the search is exactly the lexical one. The note
837
+ * read's own filters decide which candidates count, so a channel that filters
838
+ * less than the store does still answers the read it was asked for.
839
+ */
840
+ async semanticRanks(query, limit, status) {
841
+ if (this.semantic === null)
842
+ return [];
843
+ const ready = await this.semantic.ready();
844
+ if (ready !== this.semanticReady) {
845
+ // The channel just became usable, or stopped being: index what it missed
846
+ // while it was not.
847
+ this.semanticReady = ready;
848
+ this.semanticCaughtUp = false;
849
+ }
850
+ if (!this.semanticCaughtUp) {
851
+ this.semanticCaughtUp = true;
852
+ await this.catchUpSemantic();
853
+ }
854
+ if (!ready)
855
+ return [];
856
+ // The filters a note read applies are the store's own, and none of them is
857
+ // a field of the channel: a note's status changes at every snapshot epoch
858
+ // without a revision, and `about` and `linked` are bounds the journal
859
+ // resolved. So the channel is asked for the notes nearest the query and
860
+ // nothing else, and the store applies its own filters to what comes back -
861
+ // exactly as the journal does with the targets it filters on itself
862
+ // (docs/memory.md#search).
863
+ const asked = {};
864
+ const refs = await this.semantic.search(query, Math.max(limit, MIN_SEMANTIC_CANDIDATES), asked);
865
+ const ids = [];
866
+ for (const ref of refs) {
867
+ const id = parseNoteId(ref);
868
+ if (id !== null)
869
+ ids.push(id);
870
+ }
871
+ const allowed = this.allowedNoteIds(ids, status);
872
+ // The channel's order, as ranks the shared merge reads: the nearest hit
873
+ // carries the most negative rank, so it normalizes to the best score.
874
+ return allowed.map((id, index) => ({ rowid: id, rank: index - allowed.length }));
875
+ }
876
+ /**
877
+ * The notes among `ids` the read's own filters allow, in the order given. A
878
+ * deleted note is not among them, so a vector that outlived its note never
879
+ * reaches a page.
880
+ */
881
+ allowedNoteIds(ids, status) {
882
+ if (ids.length === 0)
883
+ return [];
884
+ const clauses = ["status <> 'deleted'"];
885
+ const params = [...ids];
886
+ if (status !== undefined) {
887
+ clauses.push("status = ?");
888
+ params.push(status);
889
+ }
890
+ const rows = this.db
891
+ .prepare(`SELECT id FROM notes
892
+ WHERE id IN (${ids.map(() => "?").join(", ")}) AND ${clauses.join(" AND ")}`)
893
+ .all(...params);
894
+ const allowed = new Set(rows.map((row) => toNumber(row.id)));
895
+ return ids.filter((id) => allowed.has(id));
896
+ }
701
897
  noteRow(id) {
702
898
  const row = this.db
703
899
  .prepare(`SELECT id, created, updated, revision, title, summary, body, status, window, anchor, event
@@ -737,13 +933,11 @@ export class SqliteNoteStore {
737
933
  }
738
934
  return states;
739
935
  }
740
- summaryPageBySql(query, limit, offset, match) {
741
- const clauses = [];
936
+ summaryPageBySql(query, limit, offset) {
937
+ // A delete leaves the row and marks it, and this reads the notes table
938
+ // itself, so it leaves the deleted ones out on its own (docs/memory.md).
939
+ const clauses = ["n.status <> 'deleted'"];
742
940
  const params = [];
743
- if (match !== null) {
744
- clauses.push(match.sql);
745
- params.push(...match.params);
746
- }
747
941
  if (query.status !== undefined) {
748
942
  clauses.push("n.status = ?");
749
943
  params.push(query.status);
@@ -759,10 +953,28 @@ export class SqliteNoteStore {
759
953
  .get(...params).n);
760
954
  return { items: rows.map(toSummary), total, limit };
761
955
  }
956
+ /** One page of an ordered id list, in the order the list gives. */
957
+ summaryPage(ordered, limit, offset) {
958
+ const page = ordered.slice(offset, offset + limit);
959
+ if (page.length === 0)
960
+ return { items: [], total: ordered.length, limit };
961
+ const rows = this.db
962
+ .prepare(`SELECT id, revision, title, summary, status, updated FROM notes
963
+ WHERE id IN (${page.map(() => "?").join(", ")})`)
964
+ .all(...page);
965
+ const byId = new Map(rows.map((row) => [toNumber(row.id), row]));
966
+ const items = page
967
+ .map((id) => byId.get(id))
968
+ .filter((row) => row !== undefined)
969
+ .map(toSummary);
970
+ return { items, total: ordered.length, limit };
971
+ }
762
972
  summaryPageByIds(ordered, query, limit, offset) {
763
973
  if (ordered.length === 0)
764
974
  return { items: [], total: 0, limit };
765
- const clauses = [`id IN (${ordered.map(() => "?").join(", ")})`];
975
+ // A delete leaves the row and marks it, and no search returns the note
976
+ // again, whichever lookup ordered it (docs/memory.md).
977
+ const clauses = [`status <> 'deleted'`, `id IN (${ordered.map(() => "?").join(", ")})`];
766
978
  const params = [...ordered];
767
979
  if (query.status !== undefined) {
768
980
  clauses.push("status = ?");
package/package.json CHANGED
@@ -1,8 +1,8 @@
1
1
  {
2
2
  "name": "@4fu/arbor-notes-sqlite",
3
- "version": "2026.1106.1",
3
+ "version": "2026.1107.0",
4
4
  "type": "module",
5
- "description": "The default NoteStore: one copy of every note, checked writes, revisions, anchors and the active set on SQLite beside the journal.",
5
+ "description": "The default NoteStore: one copy of every note, checked writes, revisions, anchors, the active set and search on SQLite beside the journal: lexical, and semantic when the optional channel is installed.",
6
6
  "exports": {
7
7
  ".": {
8
8
  "types": "./index.d.ts",
@@ -25,7 +25,9 @@
25
25
  "score.d.ts",
26
26
  "score.js",
27
27
  "search.d.ts",
28
- "search.js"
28
+ "search.js",
29
+ "semantic.d.ts",
30
+ "semantic.js"
29
31
  ],
30
32
  "engines": {
31
33
  "node": ">=24"
@@ -40,8 +42,17 @@
40
42
  "access": "public"
41
43
  },
42
44
  "dependencies": {
43
- "@4fu/arbor-errors": "2026.1106.1",
44
- "@4fu/arbor-interfaces": "2026.1106.1",
45
- "@4fu/arbor-prompt": "2026.1106.1"
45
+ "@4fu/arbor-errors": "2026.1107.0",
46
+ "@4fu/arbor-interfaces": "2026.1107.0",
47
+ "@4fu/arbor-lexical": "2026.1107.0",
48
+ "@4fu/arbor-prompt": "2026.1107.0"
49
+ },
50
+ "peerDependencies": {
51
+ "@4fu/arbor-search-zvec": "2026.1107.0"
52
+ },
53
+ "peerDependenciesMeta": {
54
+ "@4fu/arbor-search-zvec": {
55
+ "optional": true
56
+ }
46
57
  }
47
58
  }
package/schema.d.ts CHANGED
@@ -23,12 +23,17 @@
23
23
  * it froze them at, so a window's prefix never changes under it.
24
24
  * - `notes_fts` - trigram index for exact substrings over title, summary and
25
25
  * body.
26
+ * - `notes_words` - word index for BM25 over the same three columns, segmented
27
+ * (docs/journal.md#search).
26
28
  */
27
29
  import type { DatabaseSync } from "node:sqlite";
30
+ import type { Segmenter } from "@4fu/arbor-interfaces";
28
31
  /** Bumped when a change makes an existing database unusable. */
29
- export declare const SCHEMA_VERSION = 1;
32
+ export declare const SCHEMA_VERSION = 2;
30
33
  /** The FTS5 table note search reads. */
31
34
  export declare const FTS_TABLE = "notes_fts";
32
- /** Create the schema when it is missing, and check the version when it is not. */
33
- export declare function createSchema(db: DatabaseSync): void;
35
+ /** The word index note search reads. */
36
+ export declare const WORD_TABLE = "notes_words";
37
+ /** Create the schema when it is missing, migrate an older one, and record the version. */
38
+ export declare function createSchema(db: DatabaseSync, segmenter?: Segmenter): void;
34
39
  export declare function readMeta(db: DatabaseSync, key: string): string | null;
package/schema.js CHANGED
@@ -23,11 +23,25 @@
23
23
  * it froze them at, so a window's prefix never changes under it.
24
24
  * - `notes_fts` - trigram index for exact substrings over title, summary and
25
25
  * body.
26
+ * - `notes_words` - word index for BM25 over the same three columns, segmented
27
+ * (docs/journal.md#search).
26
28
  */
29
+ import { indexText, resolveSegmenter } from "@4fu/arbor-lexical";
27
30
  /** Bumped when a change makes an existing database unusable. */
28
- export const SCHEMA_VERSION = 1;
31
+ export const SCHEMA_VERSION = 2;
29
32
  /** The FTS5 table note search reads. */
30
33
  export const FTS_TABLE = "notes_fts";
34
+ /** The word index note search reads. */
35
+ export const WORD_TABLE = "notes_words";
36
+ /** The trigram index: exact substrings, for Chinese and mixed text. */
37
+ const TRIGRAM_INDEX = `CREATE VIRTUAL TABLE IF NOT EXISTS ${FTS_TABLE}
38
+ USING fts5(title, summary, body, tokenize = 'trigram')`;
39
+ /**
40
+ * The word index: one column per searched field, each holding that field's
41
+ * segments, one per line. The columns are what the BM25 weights rank by.
42
+ */
43
+ const WORD_INDEX = `CREATE VIRTUAL TABLE IF NOT EXISTS ${WORD_TABLE}
44
+ USING fts5(title, summary, body, tokenize = 'unicode61')`;
31
45
  const STATEMENTS = [
32
46
  `PRAGMA journal_mode = WAL`,
33
47
  `PRAGMA foreign_keys = ON`,
@@ -114,25 +128,61 @@ const STATEMENTS = [
114
128
  time INTEGER NOT NULL,
115
129
  PRIMARY KEY (epoch, note)
116
130
  )`,
117
- // Exact substrings, for Chinese and mixed text.
118
- `CREATE VIRTUAL TABLE IF NOT EXISTS ${FTS_TABLE}
119
- USING fts5(title, summary, body, tokenize = 'trigram')`,
131
+ TRIGRAM_INDEX,
132
+ WORD_INDEX,
120
133
  ];
121
- /** Create the schema when it is missing, and check the version when it is not. */
122
- export function createSchema(db) {
134
+ /**
135
+ * Version 2 added the word index (docs/journal.md#search): a segmented word
136
+ * index with BM25 beside the trigram index, which is what lets a multi-word
137
+ * query find a note that holds the words apart from each other. Every note,
138
+ * every ref and the trigram index stay, so a version-1 database opens where it
139
+ * left off: the notes it already holds are segmented into the new index in the
140
+ * same step that records the version, and a delete leaves a note out of it.
141
+ */
142
+ function migrateToVersion2(db, segmenter) {
143
+ // One transaction: the index and the version that says it is there land
144
+ // together, so a database that stops half way through is still the version it
145
+ // was and migrates again, whole, instead of half-indexing twice.
146
+ db.exec("BEGIN IMMEDIATE");
147
+ try {
148
+ db.exec(WORD_INDEX);
149
+ const insert = db.prepare(`INSERT INTO ${WORD_TABLE} (rowid, title, summary, body) VALUES (?, ?, ?, ?)`);
150
+ const rows = db
151
+ .prepare("SELECT id, title, summary, body FROM notes WHERE status <> 'deleted'")
152
+ .all();
153
+ for (const row of rows) {
154
+ insert.run(Number(row.id), indexText(row.title, segmenter), indexText(row.summary, segmenter), indexText(row.body, segmenter));
155
+ }
156
+ db.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES ('schema_version', ?)").run(String(SCHEMA_VERSION));
157
+ db.exec("COMMIT");
158
+ }
159
+ catch (error) {
160
+ try {
161
+ db.exec("ROLLBACK");
162
+ }
163
+ catch {
164
+ // The transaction is already gone; the original error is what matters.
165
+ }
166
+ throw error;
167
+ }
168
+ }
169
+ /** Create the schema when it is missing, migrate an older one, and record the version. */
170
+ export function createSchema(db, segmenter = resolveSegmenter()) {
123
171
  // The version lives in the one table that has to exist before anything else.
124
172
  db.exec(`CREATE TABLE IF NOT EXISTS meta (
125
173
  key TEXT PRIMARY KEY,
126
174
  value TEXT NOT NULL
127
175
  )`);
128
176
  const version = readMeta(db, "schema_version");
129
- if (version !== null && Number(version) !== SCHEMA_VERSION) {
130
- throw new Error(`notes schema ${version} is not supported by this build (${SCHEMA_VERSION}); migrate or rebuild the database`);
177
+ if (version !== null && Number(version) > SCHEMA_VERSION) {
178
+ throw new Error(`notes schema ${version} is newer than this build (${SCHEMA_VERSION}); upgrade arbor, or migrate the database`);
131
179
  }
180
+ if (Number(version) === SCHEMA_VERSION - 1)
181
+ migrateToVersion2(db, segmenter);
132
182
  for (const statement of STATEMENTS)
133
183
  db.exec(statement);
134
- if (version === null) {
135
- db.prepare("INSERT INTO meta (key, value) VALUES ('schema_version', ?)").run(String(SCHEMA_VERSION));
184
+ if (version === null || Number(version) !== SCHEMA_VERSION) {
185
+ db.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES ('schema_version', ?)").run(String(SCHEMA_VERSION));
136
186
  }
137
187
  }
138
188
  export function readMeta(db, key) {
package/search.d.ts CHANGED
@@ -1,27 +1,60 @@
1
1
  /**
2
2
  * Lexical search over notes (docs/journal.md#search, docs/memory.md).
3
3
  *
4
- * Lists show summaries and never bodies, so note search is a lookup of the
5
- * notes whose title, summary or body holds the query as an exact substring:
6
- * the trigram index answers three characters and more, which is what makes a
7
- * Chinese substring findable, and a `LIKE` scan answers a one- or
8
- * two-character query, which has no trigram of its own. Results come back in id
9
- * order, so a page is byte-stable however it is paged.
4
+ * The channels the journal has, merged the same way, and one more when a
5
+ * deployment installed it:
6
+ *
7
+ * - a word index, segmented and ranked with BM25 over title, summary and body,
8
+ * the title and the summary weighted above the body;
9
+ * - a trigram index for exact substrings, which is what makes a Chinese
10
+ * substring findable;
11
+ * - a `LIKE` scan, for one- and two-character queries, which have no trigram of
12
+ * their own;
13
+ * - a semantic channel, note text embedded and searched by meaning, when
14
+ * `@4fu/arbor-search-zvec` is installed.
15
+ *
16
+ * Exact substring hits come first, then word hits that are not already there,
17
+ * then the hits only the semantic channel found, each channel by its own
18
+ * relevance. The `LIKE` scan has no relevance of its own, so it answers in id
19
+ * order and a page is byte-stable however it is paged.
10
20
  */
11
21
  import type { DatabaseSync, SQLInputValue } from "node:sqlite";
12
- /** A query shorter than this cannot use the trigram index. */
13
- export declare const MIN_TRIGRAM_CHARS = 3;
22
+ import type { NoteStatus, Segmenter } from "@4fu/arbor-interfaces";
23
+ import { MIN_TRIGRAM_CHARS, escapeLike, likePattern, quoteFts } from "@4fu/arbor-lexical";
24
+ import type { Ranked } from "@4fu/arbor-lexical";
25
+ export { MIN_TRIGRAM_CHARS, escapeLike, likePattern, quoteFts };
26
+ /**
27
+ * The BM25 column weights both note indexes are ranked with, in the order the
28
+ * FTS5 tables carry their columns: title, summary, body. The title names the
29
+ * note and the summary is its one line, so a hit in either counts for more
30
+ * than one in the body: ten times as much for the title, four times for the
31
+ * summary.
32
+ */
33
+ export declare const COLUMN_WEIGHTS = "10.0, 4.0, 1.0";
14
34
  /** One condition over `notes n`, and the values it is bound to. */
15
35
  export interface SubstringClause {
16
36
  sql: string;
17
37
  params: readonly SQLInputValue[];
18
38
  }
19
- /** The notes holding the query, as a condition over `notes n`. */
39
+ /** The notes holding the query as an exact substring, as a condition over `notes n`. */
20
40
  export declare function substringClause(query: string): SubstringClause;
21
- /** The ids of several notes that hold the query, by id. */
22
- export declare function holdingIds(db: DatabaseSync, query: string, ids: readonly number[]): number[];
23
- /** Escape a literal for `LIKE ... ESCAPE '\'`. */
24
- export declare function escapeLike(value: string): string;
25
- export declare function likePattern(query: string): string;
26
- /** Quote one phrase for an FTS5 `MATCH`, so outside text is never syntax. */
27
- export declare function quoteFts(value: string): string;
41
+ /**
42
+ * The notes a text query matches, in the order search returns them: exact
43
+ * substring hits first, then word hits that are not already there, each channel
44
+ * by its own relevance, then the hits only the semantic channel found. A
45
+ * one- or two-character query has no trigram and no word match of its own, so
46
+ * the fallback scan answers it in id order, and the semantic channel's hits
47
+ * follow it.
48
+ *
49
+ * `semantic` is the channel's own candidates as ranks, in the order it gave
50
+ * them: the note store asks it for the notes nearest the query and applies its
51
+ * own filters to what comes back, so a channel that filters less than the store
52
+ * does still answers the read it was asked (docs/memory.md#search).
53
+ */
54
+ export declare function matchedIds(db: DatabaseSync, query: string, segmenter: Segmenter, status?: NoteStatus, semantic?: readonly Ranked[]): number[];
55
+ /**
56
+ * The ids of several notes that hold the query: an exact substring or the
57
+ * query's words. `about` and `linked` bound a lookup to the notes the journal
58
+ * resolved, in their own weight order, so this answers a set and not an order.
59
+ */
60
+ export declare function holdingIds(db: DatabaseSync, query: string, ids: readonly number[], segmenter: Segmenter): number[];
package/search.js CHANGED
@@ -1,50 +1,127 @@
1
1
  /**
2
2
  * Lexical search over notes (docs/journal.md#search, docs/memory.md).
3
3
  *
4
- * Lists show summaries and never bodies, so note search is a lookup of the
5
- * notes whose title, summary or body holds the query as an exact substring:
6
- * the trigram index answers three characters and more, which is what makes a
7
- * Chinese substring findable, and a `LIKE` scan answers a one- or
8
- * two-character query, which has no trigram of its own. Results come back in id
9
- * order, so a page is byte-stable however it is paged.
4
+ * The channels the journal has, merged the same way, and one more when a
5
+ * deployment installed it:
6
+ *
7
+ * - a word index, segmented and ranked with BM25 over title, summary and body,
8
+ * the title and the summary weighted above the body;
9
+ * - a trigram index for exact substrings, which is what makes a Chinese
10
+ * substring findable;
11
+ * - a `LIKE` scan, for one- and two-character queries, which have no trigram of
12
+ * their own;
13
+ * - a semantic channel, note text embedded and searched by meaning, when
14
+ * `@4fu/arbor-search-zvec` is installed.
15
+ *
16
+ * Exact substring hits come first, then word hits that are not already there,
17
+ * then the hits only the semantic channel found, each channel by its own
18
+ * relevance. The `LIKE` scan has no relevance of its own, so it answers in id
19
+ * order and a page is byte-stable however it is paged.
10
20
  */
11
- import { FTS_TABLE } from "./schema.js";
12
- /** A query shorter than this cannot use the trigram index. */
13
- export const MIN_TRIGRAM_CHARS = 3;
14
- /** The notes holding the query, as a condition over `notes n`. */
21
+ import { MIN_TRIGRAM_CHARS, escapeLike, likePattern, mergeHits, quoteFts, trigramMatch, wordMatch, } from "@4fu/arbor-lexical";
22
+ import { FTS_TABLE, WORD_TABLE } from "./schema.js";
23
+ export { MIN_TRIGRAM_CHARS, escapeLike, likePattern, quoteFts };
24
+ /**
25
+ * The BM25 column weights both note indexes are ranked with, in the order the
26
+ * FTS5 tables carry their columns: title, summary, body. The title names the
27
+ * note and the summary is its one line, so a hit in either counts for more
28
+ * than one in the body: ten times as much for the title, four times for the
29
+ * summary.
30
+ */
31
+ export const COLUMN_WEIGHTS = "10.0, 4.0, 1.0";
32
+ /** The `LIKE` scan's condition, for a query the trigram index cannot answer. */
33
+ function likeClause(query) {
34
+ const pattern = likePattern(query);
35
+ return {
36
+ sql: `(n.title LIKE ? ESCAPE '\\' OR n.summary LIKE ? ESCAPE '\\' OR n.body LIKE ? ESCAPE '\\')`,
37
+ params: [pattern, pattern, pattern],
38
+ };
39
+ }
40
+ /** The word index's condition: every one of the query's segments. */
41
+ function wordClause(tokens) {
42
+ return {
43
+ sql: `n.id IN (SELECT rowid FROM ${WORD_TABLE} WHERE ${WORD_TABLE} MATCH ?)`,
44
+ params: [wordMatch(tokens)],
45
+ };
46
+ }
47
+ /** The notes holding the query as an exact substring, as a condition over `notes n`. */
15
48
  export function substringClause(query) {
16
49
  if (Array.from(query).length >= MIN_TRIGRAM_CHARS) {
17
50
  return {
18
51
  sql: `n.id IN (SELECT rowid FROM ${FTS_TABLE} WHERE ${FTS_TABLE} MATCH ?)`,
19
- params: [quoteFts(query)],
52
+ params: [trigramMatch(query)],
20
53
  };
21
54
  }
22
- const pattern = likePattern(query);
23
- return {
24
- sql: `(n.title LIKE ? ESCAPE '\\' OR n.summary LIKE ? ESCAPE '\\' OR n.body LIKE ? ESCAPE '\\')`,
25
- params: [pattern, pattern, pattern],
26
- };
55
+ return likeClause(query);
27
56
  }
28
- /** The ids of several notes that hold the query, by id. */
29
- export function holdingIds(db, query, ids) {
57
+ /** Every match from one index, by relevance, with the status filter applied. */
58
+ function ranked(db, table, match, status) {
59
+ const filtered = status === undefined ? "" : " AND n.status = ?";
60
+ const params = status === undefined ? [match] : [match, status];
61
+ const rows = db
62
+ .prepare(`SELECT f.rowid AS rowid, bm25(${table}, ${COLUMN_WEIGHTS}) AS rank
63
+ FROM ${table} f JOIN notes n ON n.id = f.rowid
64
+ WHERE ${table} MATCH ?${filtered}
65
+ ORDER BY rank, f.rowid`)
66
+ .all(...params);
67
+ return rows.map((row) => ({ rowid: Number(row.rowid), rank: Number(row.rank) }));
68
+ }
69
+ /** The notes a one- or two-character query holds, in id order. */
70
+ function scanIds(db, query, status) {
71
+ const clause = likeClause(query);
72
+ // A delete leaves the row and marks it, and the scan reads the notes table
73
+ // itself, so it leaves the deleted ones out on its own (docs/memory.md).
74
+ const filtered = status === undefined ? " AND n.status <> 'deleted'" : " AND n.status = ?";
75
+ const rows = db
76
+ .prepare(`SELECT n.id AS id FROM notes n WHERE ${clause.sql}${filtered} ORDER BY n.id`)
77
+ .all(...clause.params, ...(status === undefined ? [] : [status]));
78
+ return rows.map((row) => Number(row.id));
79
+ }
80
+ /**
81
+ * The notes a text query matches, in the order search returns them: exact
82
+ * substring hits first, then word hits that are not already there, each channel
83
+ * by its own relevance, then the hits only the semantic channel found. A
84
+ * one- or two-character query has no trigram and no word match of its own, so
85
+ * the fallback scan answers it in id order, and the semantic channel's hits
86
+ * follow it.
87
+ *
88
+ * `semantic` is the channel's own candidates as ranks, in the order it gave
89
+ * them: the note store asks it for the notes nearest the query and applies its
90
+ * own filters to what comes back, so a channel that filters less than the store
91
+ * does still answers the read it was asked (docs/memory.md#search).
92
+ */
93
+ export function matchedIds(db, query, segmenter, status, semantic = []) {
94
+ if (Array.from(query).length < MIN_TRIGRAM_CHARS) {
95
+ // A one- or two-character query has no trigram and no word match of its
96
+ // own: the scan answers it in id order, and the semantic channel's hits
97
+ // follow it.
98
+ const scanned = scanIds(db, query, status).map((rowid) => ({ rowid, rank: 0 }));
99
+ return mergeHits({ channel: "substring", hits: scanned }, { channel: "semantic", hits: semantic }).map((hit) => hit.rowid);
100
+ }
101
+ const substrings = ranked(db, FTS_TABLE, trigramMatch(query), status);
102
+ const match = wordMatch(segmenter.segment(query));
103
+ const words = match === "" ? [] : ranked(db, WORD_TABLE, match, status);
104
+ return mergeHits({ channel: "substring", hits: substrings }, { channel: "word", hits: words }, { channel: "semantic", hits: semantic }).map((hit) => hit.rowid);
105
+ }
106
+ /**
107
+ * The ids of several notes that hold the query: an exact substring or the
108
+ * query's words. `about` and `linked` bound a lookup to the notes the journal
109
+ * resolved, in their own weight order, so this answers a set and not an order.
110
+ */
111
+ export function holdingIds(db, query, ids, segmenter) {
30
112
  if (ids.length === 0)
31
113
  return [];
32
- const clause = substringClause(query);
114
+ const clauses = [substringClause(query)];
115
+ if (Array.from(query).length >= MIN_TRIGRAM_CHARS) {
116
+ const tokens = segmenter.segment(query);
117
+ if (wordMatch(tokens) !== "")
118
+ clauses.push(wordClause(tokens));
119
+ }
33
120
  const rows = db
34
121
  .prepare(`SELECT n.id AS id FROM notes n
35
- WHERE ${clause.sql} AND n.id IN (${ids.map(() => "?").join(", ")})
122
+ WHERE (${clauses.map((clause) => clause.sql).join(" OR ")})
123
+ AND n.id IN (${ids.map(() => "?").join(", ")})
36
124
  ORDER BY n.id`)
37
- .all(...clause.params, ...ids);
125
+ .all(...clauses.flatMap((clause) => clause.params), ...ids);
38
126
  return rows.map((row) => Number(row.id));
39
127
  }
40
- /** Escape a literal for `LIKE ... ESCAPE '\'`. */
41
- export function escapeLike(value) {
42
- return value.replace(/[\\%_]/g, (character) => `\\${character}`);
43
- }
44
- export function likePattern(query) {
45
- return `%${escapeLike(query)}%`;
46
- }
47
- /** Quote one phrase for an FTS5 `MATCH`, so outside text is never syntax. */
48
- export function quoteFts(value) {
49
- return `"${value.replace(/"/g, '""')}"`;
50
- }
package/semantic.d.ts ADDED
@@ -0,0 +1,29 @@
1
+ /**
2
+ * The semantic channel the note store uses when the caller names none
3
+ * (docs/memory.md#search).
4
+ *
5
+ * The spec calls for note text to be embedded and searched by meaning beside the
6
+ * lexical indexes, and the default is zvec when `@4fu/arbor-search-zvec` is
7
+ * installed: `resolveSemanticIndex` loads it, and the store uses what it returns
8
+ * when the caller names no channel. The package is an optional peer dependency,
9
+ * so a deployment that leaves it out keeps working.
10
+ *
11
+ * What is left here is the absence that answers when the package is not there:
12
+ * no channel, and search is exactly lexical. That is also the answer when the
13
+ * package is installed but its vector store will not open - zvec's prebuilt
14
+ * binary does not load on this platform, or another host holds the collection -
15
+ * and when no embedding model is available, because the package says so by
16
+ * handing back nothing.
17
+ */
18
+ import type { SemanticIndex } from "@4fu/arbor-interfaces";
19
+ /**
20
+ * The semantic channel a note store uses when the caller names none: the zvec
21
+ * one when the package is installed and its store opens, and none otherwise.
22
+ *
23
+ * `load` replaces how the zvec package is loaded. A host that resolves modules
24
+ * its own way passes its own; so does a test that has no zvec package to load. A
25
+ * load that fails falls back rather than stopping the store, because the package
26
+ * is optional: it may be missing, or its prebuilt binary may not load on this
27
+ * platform.
28
+ */
29
+ export declare function resolveSemanticIndex(path: string, load?: (path: string) => SemanticIndex | null): SemanticIndex | null;
package/semantic.js ADDED
@@ -0,0 +1,57 @@
1
+ /**
2
+ * The semantic channel the note store uses when the caller names none
3
+ * (docs/memory.md#search).
4
+ *
5
+ * The spec calls for note text to be embedded and searched by meaning beside the
6
+ * lexical indexes, and the default is zvec when `@4fu/arbor-search-zvec` is
7
+ * installed: `resolveSemanticIndex` loads it, and the store uses what it returns
8
+ * when the caller names no channel. The package is an optional peer dependency,
9
+ * so a deployment that leaves it out keeps working.
10
+ *
11
+ * What is left here is the absence that answers when the package is not there:
12
+ * no channel, and search is exactly lexical. That is also the answer when the
13
+ * package is installed but its vector store will not open - zvec's prebuilt
14
+ * binary does not load on this platform, or another host holds the collection -
15
+ * and when no embedding model is available, because the package says so by
16
+ * handing back nothing.
17
+ */
18
+ import { createRequire } from "node:module";
19
+ /** The package that holds the zvec semantic index, when a deployment installed it. */
20
+ const ZVEC_PACKAGE = "@4fu/arbor-search-zvec";
21
+ /**
22
+ * Load the zvec semantic index over a store beside the notes database's own, or
23
+ * `null` when it is not there: the package is not installed, or it holds no
24
+ * index.
25
+ */
26
+ function loadZvec(path) {
27
+ const require = createRequire(import.meta.url);
28
+ const loaded = require(ZVEC_PACKAGE);
29
+ const create = loaded?.createZvecIndex;
30
+ if (typeof create !== "function")
31
+ return null;
32
+ const index = create({ path });
33
+ if (index === null || typeof index !== "object")
34
+ return null;
35
+ const shape = index;
36
+ if (typeof shape.add !== "function" || typeof shape.search !== "function")
37
+ return null;
38
+ return index;
39
+ }
40
+ /**
41
+ * The semantic channel a note store uses when the caller names none: the zvec
42
+ * one when the package is installed and its store opens, and none otherwise.
43
+ *
44
+ * `load` replaces how the zvec package is loaded. A host that resolves modules
45
+ * its own way passes its own; so does a test that has no zvec package to load. A
46
+ * load that fails falls back rather than stopping the store, because the package
47
+ * is optional: it may be missing, or its prebuilt binary may not load on this
48
+ * platform.
49
+ */
50
+ export function resolveSemanticIndex(path, load = loadZvec) {
51
+ try {
52
+ return load(path);
53
+ }
54
+ catch {
55
+ return null;
56
+ }
57
+ }