@4fu/arbor-notes-sqlite 2026.1106.0 → 2026.1107.0

package/schema.d.ts

@@ -23,12 +23,17 @@
  *   it froze them at, so a window's prefix never changes under it.
  * - `notes_fts` - trigram index for exact substrings over title, summary and
  *   body.
+ * - `notes_words` - word index for BM25 over the same three columns, segmented
+ *   (docs/journal.md#search).
  */
 import type { DatabaseSync } from "node:sqlite";
+import type { Segmenter } from "@4fu/arbor-interfaces";
 /** Bumped when a change makes an existing database unusable. */
-export declare const SCHEMA_VERSION = 1;
+export declare const SCHEMA_VERSION = 2;
 /** The FTS5 table note search reads. */
 export declare const FTS_TABLE = "notes_fts";
-/** Create the schema when it is missing, and check the version when it is not. */
-export declare function createSchema(db: DatabaseSync): void;
+/** The word index note search reads. */
+export declare const WORD_TABLE = "notes_words";
+/** Create the schema when it is missing, migrate an older one, and record the version. */
+export declare function createSchema(db: DatabaseSync, segmenter?: Segmenter): void;
 export declare function readMeta(db: DatabaseSync, key: string): string | null;

package/schema.js

@@ -23,11 +23,25 @@
  *   it froze them at, so a window's prefix never changes under it.
  * - `notes_fts` - trigram index for exact substrings over title, summary and
  *   body.
+ * - `notes_words` - word index for BM25 over the same three columns, segmented
+ *   (docs/journal.md#search).
  */
+import { indexText, resolveSegmenter } from "@4fu/arbor-lexical";
 /** Bumped when a change makes an existing database unusable. */
-export const SCHEMA_VERSION = 1;
+export const SCHEMA_VERSION = 2;
 /** The FTS5 table note search reads. */
 export const FTS_TABLE = "notes_fts";
+/** The word index note search reads. */
+export const WORD_TABLE = "notes_words";
+/** The trigram index: exact substrings, for Chinese and mixed text. */
+const TRIGRAM_INDEX = `CREATE VIRTUAL TABLE IF NOT EXISTS ${FTS_TABLE}
+   USING fts5(title, summary, body, tokenize = 'trigram')`;
+/**
+ * The word index: one column per searched field, each holding that field's
+ * segments, one per line. The columns are what the BM25 weights rank by.
+ */
+const WORD_INDEX = `CREATE VIRTUAL TABLE IF NOT EXISTS ${WORD_TABLE}
+   USING fts5(title, summary, body, tokenize = 'unicode61')`;
 const STATEMENTS = [
     `PRAGMA journal_mode = WAL`,
     `PRAGMA foreign_keys = ON`,
@@ -114,25 +128,61 @@ const STATEMENTS = [
      time INTEGER NOT NULL,
      PRIMARY KEY (epoch, note)
    )`,
-    // Exact substrings, for Chinese and mixed text.
-    `CREATE VIRTUAL TABLE IF NOT EXISTS ${FTS_TABLE}
-     USING fts5(title, summary, body, tokenize = 'trigram')`,
+    TRIGRAM_INDEX,
+    WORD_INDEX,
 ];
-/** Create the schema when it is missing, and check the version when it is not. */
-export function createSchema(db) {
+/**
+ * Version 2 added the word index (docs/journal.md#search): a segmented word
+ * index with BM25 beside the trigram index, which is what lets a multi-word
+ * query find a note that holds the words apart from each other. Every note,
+ * every ref and the trigram index stay, so a version-1 database opens where it
+ * left off: the notes it already holds are segmented into the new index in the
+ * same step that records the version, and a delete leaves a note out of it.
+ */
+function migrateToVersion2(db, segmenter) {
+    // One transaction: the index and the version that says it is there land
+    // together, so a database that stops half way through is still the version it
+    // was and migrates again, whole, instead of half-indexing twice.
+    db.exec("BEGIN IMMEDIATE");
+    try {
+        db.exec(WORD_INDEX);
+        const insert = db.prepare(`INSERT INTO ${WORD_TABLE} (rowid, title, summary, body) VALUES (?, ?, ?, ?)`);
+        const rows = db
+            .prepare("SELECT id, title, summary, body FROM notes WHERE status <> 'deleted'")
+            .all();
+        for (const row of rows) {
+            insert.run(Number(row.id), indexText(row.title, segmenter), indexText(row.summary, segmenter), indexText(row.body, segmenter));
+        }
+        db.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES ('schema_version', ?)").run(String(SCHEMA_VERSION));
+        db.exec("COMMIT");
+    }
+    catch (error) {
+        try {
+            db.exec("ROLLBACK");
+        }
+        catch {
+            // The transaction is already gone; the original error is what matters.
+        }
+        throw error;
+    }
+}
+/** Create the schema when it is missing, migrate an older one, and record the version. */
+export function createSchema(db, segmenter = resolveSegmenter()) {
     // The version lives in the one table that has to exist before anything else.
     db.exec(`CREATE TABLE IF NOT EXISTS meta (
        key TEXT PRIMARY KEY,
        value TEXT NOT NULL
      )`);
     const version = readMeta(db, "schema_version");
-    if (version !== null && Number(version) !== SCHEMA_VERSION) {
-        throw new Error(`notes schema ${version} is not supported by this build (${SCHEMA_VERSION}); migrate or rebuild the database`);
+    if (version !== null && Number(version) > SCHEMA_VERSION) {
+        throw new Error(`notes schema ${version} is newer than this build (${SCHEMA_VERSION}); upgrade arbor, or migrate the database`);
     }
+    if (Number(version) === SCHEMA_VERSION - 1)
+        migrateToVersion2(db, segmenter);
     for (const statement of STATEMENTS)
         db.exec(statement);
-    if (version === null) {
-        db.prepare("INSERT INTO meta (key, value) VALUES ('schema_version', ?)").run(String(SCHEMA_VERSION));
+    if (version === null || Number(version) !== SCHEMA_VERSION) {
+        db.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES ('schema_version', ?)").run(String(SCHEMA_VERSION));
     }
 }
 export function readMeta(db, key) {

package/search.d.ts

@@ -1,27 +1,60 @@
 /**
  * Lexical search over notes (docs/journal.md#search, docs/memory.md).
  *
- * Lists show summaries and never bodies, so note search is a lookup of the
- * notes whose title, summary or body holds the query as an exact substring:
- * the trigram index answers three characters and more, which is what makes a
- * Chinese substring findable, and a `LIKE` scan answers a one- or
- * two-character query, which has no trigram of its own. Results come back in id
- * order, so a page is byte-stable however it is paged.
+ * The channels the journal has, merged the same way, and one more when a
+ * deployment installed it:
+ *
+ * - a word index, segmented and ranked with BM25 over title, summary and body,
+ *   the title and the summary weighted above the body;
+ * - a trigram index for exact substrings, which is what makes a Chinese
+ *   substring findable;
+ * - a `LIKE` scan, for one- and two-character queries, which have no trigram of
+ *   their own;
+ * - a semantic channel, note text embedded and searched by meaning, when
+ *   `@4fu/arbor-search-zvec` is installed.
+ *
+ * Exact substring hits come first, then word hits that are not already there,
+ * then the hits only the semantic channel found, each channel by its own
+ * relevance. The `LIKE` scan has no relevance of its own, so it answers in id
+ * order and a page is byte-stable however it is paged.
  */
 import type { DatabaseSync, SQLInputValue } from "node:sqlite";
-/** A query shorter than this cannot use the trigram index. */
-export declare const MIN_TRIGRAM_CHARS = 3;
+import type { NoteStatus, Segmenter } from "@4fu/arbor-interfaces";
+import { MIN_TRIGRAM_CHARS, escapeLike, likePattern, quoteFts } from "@4fu/arbor-lexical";
+import type { Ranked } from "@4fu/arbor-lexical";
+export { MIN_TRIGRAM_CHARS, escapeLike, likePattern, quoteFts };
+/**
+ * The BM25 column weights both note indexes are ranked with, in the order the
+ * FTS5 tables carry their columns: title, summary, body. The title names the
+ * note and the summary is its one line, so a hit in either counts for more
+ * than one in the body: ten times as much for the title, four times for the
+ * summary.
+ */
+export declare const COLUMN_WEIGHTS = "10.0, 4.0, 1.0";
 /** One condition over `notes n`, and the values it is bound to. */
 export interface SubstringClause {
     sql: string;
     params: readonly SQLInputValue[];
 }
-/** The notes holding the query, as a condition over `notes n`. */
+/** The notes holding the query as an exact substring, as a condition over `notes n`. */
 export declare function substringClause(query: string): SubstringClause;
-/** The ids of several notes that hold the query, by id. */
-export declare function holdingIds(db: DatabaseSync, query: string, ids: readonly number[]): number[];
-/** Escape a literal for `LIKE ... ESCAPE '\'`. */
-export declare function escapeLike(value: string): string;
-export declare function likePattern(query: string): string;
-/** Quote one phrase for an FTS5 `MATCH`, so outside text is never syntax. */
-export declare function quoteFts(value: string): string;
+/**
+ * The notes a text query matches, in the order search returns them: exact
+ * substring hits first, then word hits that are not already there, each channel
+ * by its own relevance, then the hits only the semantic channel found. A
+ * one- or two-character query has no trigram and no word match of its own, so
+ * the fallback scan answers it in id order, and the semantic channel's hits
+ * follow it.
+ *
+ * `semantic` is the channel's own candidates as ranks, in the order it gave
+ * them: the note store asks it for the notes nearest the query and applies its
+ * own filters to what comes back, so a channel that filters less than the store
+ * does still answers the read it was asked (docs/memory.md#search).
+ */
+export declare function matchedIds(db: DatabaseSync, query: string, segmenter: Segmenter, status?: NoteStatus, semantic?: readonly Ranked[]): number[];
+/**
+ * The ids of several notes that hold the query: an exact substring or the
+ * query's words. `about` and `linked` bound a lookup to the notes the journal
+ * resolved, in their own weight order, so this answers a set and not an order.
+ */
+export declare function holdingIds(db: DatabaseSync, query: string, ids: readonly number[], segmenter: Segmenter): number[];

package/search.js

@@ -1,50 +1,127 @@
 /**
  * Lexical search over notes (docs/journal.md#search, docs/memory.md).
  *
- * Lists show summaries and never bodies, so note search is a lookup of the
- * notes whose title, summary or body holds the query as an exact substring:
- * the trigram index answers three characters and more, which is what makes a
- * Chinese substring findable, and a `LIKE` scan answers a one- or
- * two-character query, which has no trigram of its own. Results come back in id
- * order, so a page is byte-stable however it is paged.
+ * The channels the journal has, merged the same way, and one more when a
+ * deployment installed it:
+ *
+ * - a word index, segmented and ranked with BM25 over title, summary and body,
+ *   the title and the summary weighted above the body;
+ * - a trigram index for exact substrings, which is what makes a Chinese
+ *   substring findable;
+ * - a `LIKE` scan, for one- and two-character queries, which have no trigram of
+ *   their own;
+ * - a semantic channel, note text embedded and searched by meaning, when
+ *   `@4fu/arbor-search-zvec` is installed.
+ *
+ * Exact substring hits come first, then word hits that are not already there,
+ * then the hits only the semantic channel found, each channel by its own
+ * relevance. The `LIKE` scan has no relevance of its own, so it answers in id
+ * order and a page is byte-stable however it is paged.
  */
-import { FTS_TABLE } from "./schema.js";
-/** A query shorter than this cannot use the trigram index. */
-export const MIN_TRIGRAM_CHARS = 3;
-/** The notes holding the query, as a condition over `notes n`. */
+import { MIN_TRIGRAM_CHARS, escapeLike, likePattern, mergeHits, quoteFts, trigramMatch, wordMatch, } from "@4fu/arbor-lexical";
+import { FTS_TABLE, WORD_TABLE } from "./schema.js";
+export { MIN_TRIGRAM_CHARS, escapeLike, likePattern, quoteFts };
+/**
+ * The BM25 column weights both note indexes are ranked with, in the order the
+ * FTS5 tables carry their columns: title, summary, body. The title names the
+ * note and the summary is its one line, so a hit in either counts for more
+ * than one in the body: ten times as much for the title, four times for the
+ * summary.
+ */
+export const COLUMN_WEIGHTS = "10.0, 4.0, 1.0";
+/** The `LIKE` scan's condition, for a query the trigram index cannot answer. */
+function likeClause(query) {
+    const pattern = likePattern(query);
+    return {
+        sql: `(n.title LIKE ? ESCAPE '\\' OR n.summary LIKE ? ESCAPE '\\' OR n.body LIKE ? ESCAPE '\\')`,
+        params: [pattern, pattern, pattern],
+    };
+}
+/** The word index's condition: every one of the query's segments. */
+function wordClause(tokens) {
+    return {
+        sql: `n.id IN (SELECT rowid FROM ${WORD_TABLE} WHERE ${WORD_TABLE} MATCH ?)`,
+        params: [wordMatch(tokens)],
+    };
+}
+/** The notes holding the query as an exact substring, as a condition over `notes n`. */
 export function substringClause(query) {
     if (Array.from(query).length >= MIN_TRIGRAM_CHARS) {
         return {
             sql: `n.id IN (SELECT rowid FROM ${FTS_TABLE} WHERE ${FTS_TABLE} MATCH ?)`,
-            params: [quoteFts(query)],
+            params: [trigramMatch(query)],
         };
     }
-    const pattern = likePattern(query);
-    return {
-        sql: `(n.title LIKE ? ESCAPE '\\' OR n.summary LIKE ? ESCAPE '\\' OR n.body LIKE ? ESCAPE '\\')`,
-        params: [pattern, pattern, pattern],
-    };
+    return likeClause(query);
 }
-/** The ids of several notes that hold the query, by id. */
-export function holdingIds(db, query, ids) {
+/** Every match from one index, by relevance, with the status filter applied. */
+function ranked(db, table, match, status) {
+    const filtered = status === undefined ? "" : " AND n.status = ?";
+    const params = status === undefined ? [match] : [match, status];
+    const rows = db
+        .prepare(`SELECT f.rowid AS rowid, bm25(${table}, ${COLUMN_WEIGHTS}) AS rank
+       FROM ${table} f JOIN notes n ON n.id = f.rowid
+       WHERE ${table} MATCH ?${filtered}
+       ORDER BY rank, f.rowid`)
+        .all(...params);
+    return rows.map((row) => ({ rowid: Number(row.rowid), rank: Number(row.rank) }));
+}
+/** The notes a one- or two-character query holds, in id order. */
+function scanIds(db, query, status) {
+    const clause = likeClause(query);
+    // A delete leaves the row and marks it, and the scan reads the notes table
+    // itself, so it leaves the deleted ones out on its own (docs/memory.md).
+    const filtered = status === undefined ? " AND n.status <> 'deleted'" : " AND n.status = ?";
+    const rows = db
+        .prepare(`SELECT n.id AS id FROM notes n WHERE ${clause.sql}${filtered} ORDER BY n.id`)
+        .all(...clause.params, ...(status === undefined ? [] : [status]));
+    return rows.map((row) => Number(row.id));
+}
+/**
+ * The notes a text query matches, in the order search returns them: exact
+ * substring hits first, then word hits that are not already there, each channel
+ * by its own relevance, then the hits only the semantic channel found. A
+ * one- or two-character query has no trigram and no word match of its own, so
+ * the fallback scan answers it in id order, and the semantic channel's hits
+ * follow it.
+ *
+ * `semantic` is the channel's own candidates as ranks, in the order it gave
+ * them: the note store asks it for the notes nearest the query and applies its
+ * own filters to what comes back, so a channel that filters less than the store
+ * does still answers the read it was asked (docs/memory.md#search).
+ */
+export function matchedIds(db, query, segmenter, status, semantic = []) {
+    if (Array.from(query).length < MIN_TRIGRAM_CHARS) {
+        // A one- or two-character query has no trigram and no word match of its
+        // own: the scan answers it in id order, and the semantic channel's hits
+        // follow it.
+        const scanned = scanIds(db, query, status).map((rowid) => ({ rowid, rank: 0 }));
+        return mergeHits({ channel: "substring", hits: scanned }, { channel: "semantic", hits: semantic }).map((hit) => hit.rowid);
+    }
+    const substrings = ranked(db, FTS_TABLE, trigramMatch(query), status);
+    const match = wordMatch(segmenter.segment(query));
+    const words = match === "" ? [] : ranked(db, WORD_TABLE, match, status);
+    return mergeHits({ channel: "substring", hits: substrings }, { channel: "word", hits: words }, { channel: "semantic", hits: semantic }).map((hit) => hit.rowid);
+}
+/**
+ * The ids of several notes that hold the query: an exact substring or the
+ * query's words. `about` and `linked` bound a lookup to the notes the journal
+ * resolved, in their own weight order, so this answers a set and not an order.
+ */
+export function holdingIds(db, query, ids, segmenter) {
     if (ids.length === 0)
         return [];
-    const clause = substringClause(query);
+    const clauses = [substringClause(query)];
+    if (Array.from(query).length >= MIN_TRIGRAM_CHARS) {
+        const tokens = segmenter.segment(query);
+        if (wordMatch(tokens) !== "")
+            clauses.push(wordClause(tokens));
+    }
     const rows = db
         .prepare(`SELECT n.id AS id FROM notes n
-       WHERE ${clause.sql} AND n.id IN (${ids.map(() => "?").join(", ")})
+       WHERE (${clauses.map((clause) => clause.sql).join(" OR ")})
+         AND n.id IN (${ids.map(() => "?").join(", ")})
        ORDER BY n.id`)
-        .all(...clause.params, ...ids);
+        .all(...clauses.flatMap((clause) => clause.params), ...ids);
     return rows.map((row) => Number(row.id));
 }
-/** Escape a literal for `LIKE ... ESCAPE '\'`. */
-export function escapeLike(value) {
-    return value.replace(/[\\%_]/g, (character) => `\\${character}`);
-}
-export function likePattern(query) {
-    return `%${escapeLike(query)}%`;
-}
-/** Quote one phrase for an FTS5 `MATCH`, so outside text is never syntax. */
-export function quoteFts(value) {
-    return `"${value.replace(/"/g, '""')}"`;
-}

package/semantic.d.ts

@@ -0,0 +1,29 @@
+/**
+ * The semantic channel the note store uses when the caller names none
+ * (docs/memory.md#search).
+ *
+ * The spec calls for note text to be embedded and searched by meaning beside the
+ * lexical indexes, and the default is zvec when `@4fu/arbor-search-zvec` is
+ * installed: `resolveSemanticIndex` loads it, and the store uses what it returns
+ * when the caller names no channel. The package is an optional peer dependency,
+ * so a deployment that leaves it out keeps working.
+ *
+ * What is left here is the absence that answers when the package is not there:
+ * no channel, and search is exactly lexical. That is also the answer when the
+ * package is installed but its vector store will not open - zvec's prebuilt
+ * binary does not load on this platform, or another host holds the collection -
+ * and when no embedding model is available, because the package says so by
+ * handing back nothing.
+ */
+import type { SemanticIndex } from "@4fu/arbor-interfaces";
+/**
+ * The semantic channel a note store uses when the caller names none: the zvec
+ * one when the package is installed and its store opens, and none otherwise.
+ *
+ * `load` replaces how the zvec package is loaded. A host that resolves modules
+ * its own way passes its own; so does a test that has no zvec package to load. A
+ * load that fails falls back rather than stopping the store, because the package
+ * is optional: it may be missing, or its prebuilt binary may not load on this
+ * platform.
+ */
+export declare function resolveSemanticIndex(path: string, load?: (path: string) => SemanticIndex | null): SemanticIndex | null;

package/semantic.js

@@ -0,0 +1,57 @@
+/**
+ * The semantic channel the note store uses when the caller names none
+ * (docs/memory.md#search).
+ *
+ * The spec calls for note text to be embedded and searched by meaning beside the
+ * lexical indexes, and the default is zvec when `@4fu/arbor-search-zvec` is
+ * installed: `resolveSemanticIndex` loads it, and the store uses what it returns
+ * when the caller names no channel. The package is an optional peer dependency,
+ * so a deployment that leaves it out keeps working.
+ *
+ * What is left here is the absence that answers when the package is not there:
+ * no channel, and search is exactly lexical. That is also the answer when the
+ * package is installed but its vector store will not open - zvec's prebuilt
+ * binary does not load on this platform, or another host holds the collection -
+ * and when no embedding model is available, because the package says so by
+ * handing back nothing.
+ */
+import { createRequire } from "node:module";
+/** The package that holds the zvec semantic index, when a deployment installed it. */
+const ZVEC_PACKAGE = "@4fu/arbor-search-zvec";
+/**
+ * Load the zvec semantic index over a store beside the notes database's own, or
+ * `null` when it is not there: the package is not installed, or it holds no
+ * index.
+ */
+function loadZvec(path) {
+    const require = createRequire(import.meta.url);
+    const loaded = require(ZVEC_PACKAGE);
+    const create = loaded?.createZvecIndex;
+    if (typeof create !== "function")
+        return null;
+    const index = create({ path });
+    if (index === null || typeof index !== "object")
+        return null;
+    const shape = index;
+    if (typeof shape.add !== "function" || typeof shape.search !== "function")
+        return null;
+    return index;
+}
+/**
+ * The semantic channel a note store uses when the caller names none: the zvec
+ * one when the package is installed and its store opens, and none otherwise.
+ *
+ * `load` replaces how the zvec package is loaded. A host that resolves modules
+ * its own way passes its own; so does a test that has no zvec package to load. A
+ * load that fails falls back rather than stopping the store, because the package
+ * is optional: it may be missing, or its prebuilt binary may not load on this
+ * platform.
+ */
+export function resolveSemanticIndex(path, load = loadZvec) {
+    try {
+        return load(path);
+    }
+    catch {
+        return null;
+    }
+}