@4fu/arbor-journal-sqlite 2026.1106.1 → 2026.1107.1

package/README.md

@@ -56,7 +56,9 @@ npm install @4fu/arbor-core @4fu/arbor-journal-sqlite @4fu/arbor-notes-sqlite \
 ```
 
 `@4fu/arbor-plugin-http` is optional; it is the example plugin below.
-`@4fu/arbor-segmenter-jieba` is an optional peer of the journal.
+`@4fu/arbor-segmenter-jieba` is an optional peer of the lexical package, and
+`@4fu/arbor-search-zvec` an optional peer of the journal: absent either one, the
+journal searches lexically and nothing else changes.
 
 ## A minimal host
 
@@ -120,6 +122,8 @@ window gets, and how a continuation keeps working in it, are in
 | [`@4fu/arbor-journal-sqlite`](packages/journal-sqlite) | the default `Journal` and `BlobStore` on SQLite |
 | [`@4fu/arbor-notes-sqlite`](packages/notes-sqlite) | the default `NoteStore` on SQLite, beside the journal |
 | [`@4fu/arbor-segmenter-jieba`](packages/segmenter-jieba) | the default `Segmenter`: jieba's dictionary segmentation for the word index, optional |
+| [`@4fu/arbor-lexical`](packages/lexical) | the lexical search the journal and notes share: word segmentation, the FTS5 helpers and the merge order |
+| [`@4fu/arbor-search-zvec`](packages/search-zvec) | the optional semantic channel: event text embedded and searched by meaning with zvec |
 | [`@4fu/arbor-workspace-dir`](packages/workspace-dir) | the fallback `Workspace`: one directory per window, with a continuation working in its predecessor's directory in place |
 | [`@4fu/arbor-workspace-ovnfs`](packages/workspace-ovnfs) | the default `Workspace` when ovnfs is installed: one shared folder, per-window copy-on-write checkouts the model opens and applies back |
 | [`@4fu/arbor-environment-dir`](packages/environment-dir) | the default `Environment`: one `/env` directory, a revision per item, validation and deltas |
@@ -171,6 +175,7 @@ tarball would contain ([docs/release.md](docs/release.md)).
 | `packages/errors` | the error shape every refusal uses: the rule and the next action |
 | `packages/journal-sqlite` | the default `Journal` and `BlobStore` on SQLite |
 | `packages/segmenter-jieba` | the default `Segmenter`: jieba's dictionary segmentation for the word index, optional |
+| `packages/lexical` | the lexical search the journal and notes share: word segmentation, the FTS5 helpers and the merge order |
 | `packages/notes-sqlite` | the default `NoteStore` on SQLite, beside the journal |
 | `packages/judge-jev` | the default `Judge`: Jev, TypeSafe AI's System One classifier, answering the dispatch decisions |
 | `packages/workspace-dir` | the default `Workspace`: one directory per window, with a continuation working in its predecessor's directory in place, and the lookup of another window's directory it contributes to the catalog |

package/index.d.ts

@@ -5,8 +5,8 @@ export { SqliteJournal, deriveLinks, parseEventRef, eventRefOf, summaryOf } from
 export type { SqliteJournalOptions } from "./sqlite-journal.ts";
 export { FsBlobStore, blobIdOf, blobRefOf } from "./blob-store.ts";
 export type { BlobEntry, BlobRegistry } from "./blob-store.ts";
-export { fallbackSegmenter, resolveSegmenter, indexText } from "./segmenter.ts";
+export { resolveSemanticIndex } from "./semantic.ts";
 export { SCHEMA_VERSION, IMPLICIT_CITE_WEIGHT, TRIGRAM_TABLE, WORD_TABLE, createSchema, } from "./schema.ts";
-export { buildFilters, escapeLike, likePattern, quoteFts, ranked, substring, trigramMatch, wordMatch, } from "./search.ts";
-export type { FilterSql, Ranked, SearchFilters } from "./search.ts";
-export { MIN_TRIGRAM_CHARS } from "./sqlite-journal.ts";
+export { buildFilters } from "./search.ts";
+export type { FilterSql, SearchFilters } from "./search.ts";
+export { MIN_TRIGRAM_CHARS } from "@4fu/arbor-lexical";

package/index.js

@@ -3,7 +3,7 @@
  */
 export { SqliteJournal, deriveLinks, parseEventRef, eventRefOf, summaryOf } from "./sqlite-journal.js";
 export { FsBlobStore, blobIdOf, blobRefOf } from "./blob-store.js";
-export { fallbackSegmenter, resolveSegmenter, indexText } from "./segmenter.js";
+export { resolveSemanticIndex } from "./semantic.js";
 export { SCHEMA_VERSION, IMPLICIT_CITE_WEIGHT, TRIGRAM_TABLE, WORD_TABLE, createSchema, } from "./schema.js";
-export { buildFilters, escapeLike, likePattern, quoteFts, ranked, substring, trigramMatch, wordMatch, } from "./search.js";
-export { MIN_TRIGRAM_CHARS } from "./sqlite-journal.js";
+export { buildFilters } from "./search.js";
+export { MIN_TRIGRAM_CHARS } from "@4fu/arbor-lexical";

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@4fu/arbor-journal-sqlite",
-  "version": "2026.1106.1",
+  "version": "2026.1107.1",
   "type": "module",
-  "description": "The default Journal: append-only events with a global sequence, targets, native ids, refs, blobs and lexical search on SQLite.",
+  "description": "The default Journal: append-only events with a global sequence, targets, native ids, refs, blobs and search on SQLite: lexical, and semantic when the optional channel is installed.",
   "exports": {
     ".": {
       "types": "./index.d.ts",
@@ -20,8 +20,8 @@
     "schema.js",
     "search.d.ts",
     "search.js",
-    "segmenter.d.ts",
-    "segmenter.js",
+    "semantic.d.ts",
+    "semantic.js",
     "sqlite-journal.d.ts",
     "sqlite-journal.js",
     "stored.d.ts",
@@ -40,14 +40,15 @@
     "access": "public"
   },
   "dependencies": {
-    "@4fu/arbor-errors": "2026.1106.1",
-    "@4fu/arbor-interfaces": "2026.1106.1"
+    "@4fu/arbor-errors": "2026.1107.0",
+    "@4fu/arbor-interfaces": "2026.1107.1",
+    "@4fu/arbor-lexical": "2026.1107.1"
   },
   "peerDependencies": {
-    "@4fu/arbor-segmenter-jieba": "2026.1106.1"
+    "@4fu/arbor-search-zvec": "2026.1107.1"
   },
   "peerDependenciesMeta": {
-    "@4fu/arbor-segmenter-jieba": {
+    "@4fu/arbor-search-zvec": {
       "optional": true
     }
   }

package/search.d.ts

@@ -1,30 +1,34 @@
 /**
  * Lexical search over the journal's events (docs/journal.md#search).
  *
- * Three indexes, merged:
+ * Three indexes, and a fourth channel when a deployment installed one:
  *
  * - a word index, segmented and ranked with BM25;
  * - a trigram index for exact substrings, which is what makes a Chinese
  *   substring findable;
  * - a substring scan, for one- and two-character queries and for queries the
- *   trigram index cannot answer.
+ *   trigram index cannot answer;
+ * - a semantic channel, event text embedded and searched by meaning, when
+ *   `@4fu/arbor-search-zvec` is installed.
  *
- * Exact substring hits come first, then word hits that are not already there.
+ * What the journal and the notes store share - how a literal is quoted and
+ * escaped, how a query becomes one index's match, and how the channels merge -
+ * lives in `@4fu/arbor-lexical`. What lives here is what is the journal's own:
+ * the filters a read applies, and reading the rows an index matched.
  */
 import type { DatabaseSync, SQLInputValue } from "node:sqlite";
+import type { WindowRef } from "@4fu/arbor-interfaces";
+import { MIN_TRIGRAM_CHARS, likePattern, quoteFts, trigramMatch, wordMatch } from "@4fu/arbor-lexical";
+import type { Ranked } from "@4fu/arbor-lexical";
+export { MIN_TRIGRAM_CHARS, likePattern, quoteFts, trigramMatch, wordMatch };
 export interface SearchFilters {
     kinds?: readonly string[];
-    window?: string;
+    window?: WindowRef;
     targets?: readonly string[];
     external?: string;
     afterSeq?: number;
     beforeSeq?: number;
 }
-/** One matching row, with its raw relevance. Lower `rank` is better. */
-export interface Ranked {
-    seq: number;
-    rank: number;
-}
 export interface FilterSql {
     sql: string;
     params: SQLInputValue[];
@@ -38,12 +42,3 @@ export declare function buildFilters(filters: SearchFilters): FilterSql;
 export declare function ranked(db: DatabaseSync, table: string, match: string, filters: SearchFilters): Ranked[];
 /** Every substring match, newest first. The scan has no relevance of its own. */
 export declare function substring(db: DatabaseSync, pattern: string, filters: SearchFilters): Ranked[];
-/** Escape a literal for `LIKE ... ESCAPE '\'`. */
-export declare function escapeLike(value: string): string;
-export declare function likePattern(query: string): string;
-/** Quote one term for an FTS5 `MATCH`, so outside text is never syntax. */
-export declare function quoteFts(value: string): string;
-/** The trigram index takes one phrase. */
-export declare function trigramMatch(query: string): string;
-/** The word index takes the query's segments, all of them required. */
-export declare function wordMatch(tokens: readonly string[]): string;

package/search.js

@@ -1,16 +1,23 @@
 /**
  * Lexical search over the journal's events (docs/journal.md#search).
  *
- * Three indexes, merged:
+ * Three indexes, and a fourth channel when a deployment installed one:
  *
  * - a word index, segmented and ranked with BM25;
  * - a trigram index for exact substrings, which is what makes a Chinese
  *   substring findable;
  * - a substring scan, for one- and two-character queries and for queries the
- *   trigram index cannot answer.
+ *   trigram index cannot answer;
+ * - a semantic channel, event text embedded and searched by meaning, when
+ *   `@4fu/arbor-search-zvec` is installed.
  *
- * Exact substring hits come first, then word hits that are not already there.
+ * What the journal and the notes store share - how a literal is quoted and
+ * escaped, how a query becomes one index's match, and how the channels merge -
+ * lives in `@4fu/arbor-lexical`. What lives here is what is the journal's own:
+ * the filters a read applies, and reading the rows an index matched.
  */
+import { MIN_TRIGRAM_CHARS, likePattern, quoteFts, trigramMatch, wordMatch } from "@4fu/arbor-lexical";
+export { MIN_TRIGRAM_CHARS, likePattern, quoteFts, trigramMatch, wordMatch };
 /**
  * The filters a read applies, as SQL over `events e`. A target filter is an
  * `EXISTS`, so one event with three targets still counts once.
@@ -48,42 +55,20 @@ export function buildFilters(filters) {
 export function ranked(db, table, match, filters) {
     const built = buildFilters(filters);
     const rows = db
-        .prepare(`SELECT f.rowid AS seq, bm25(${table}) AS rank
+        .prepare(`SELECT f.rowid AS rowid, bm25(${table}) AS rank
        FROM ${table} f JOIN events e ON e.seq = f.rowid
        WHERE ${table} MATCH ?${built.sql}
        ORDER BY rank, f.rowid`)
         .all(match, ...built.params);
-    return rows.map((row) => ({ seq: Number(row.seq), rank: Number(row.rank) }));
+    return rows.map((row) => ({ rowid: Number(row.rowid), rank: Number(row.rank) }));
 }
 /** Every substring match, newest first. The scan has no relevance of its own. */
 export function substring(db, pattern, filters) {
     const built = buildFilters(filters);
     const rows = db
-        .prepare(`SELECT e.seq AS seq, 1.0 AS rank FROM events e
+        .prepare(`SELECT e.seq AS rowid, 1.0 AS rank FROM events e
        WHERE e.body LIKE ? ESCAPE '\\'${built.sql}
        ORDER BY e.seq DESC`)
         .all(pattern, ...built.params);
-    return rows.map((row) => ({ seq: Number(row.seq), rank: Number(row.rank) }));
-}
-/** Escape a literal for `LIKE ... ESCAPE '\'`. */
-export function escapeLike(value) {
-    return value.replace(/[\\%_]/g, (character) => `\\${character}`);
-}
-export function likePattern(query) {
-    return `%${escapeLike(query)}%`;
-}
-/** Quote one term for an FTS5 `MATCH`, so outside text is never syntax. */
-export function quoteFts(value) {
-    return `"${value.replace(/"/g, '""')}"`;
-}
-/** The trigram index takes one phrase. */
-export function trigramMatch(query) {
-    return quoteFts(query);
-}
-/** The word index takes the query's segments, all of them required. */
-export function wordMatch(tokens) {
-    return tokens
-        .filter((token) => token.length > 0)
-        .map(quoteFts)
-        .join(" ");
+    return rows.map((row) => ({ rowid: Number(row.rowid), rank: Number(row.rank) }));
 }

package/semantic.d.ts

@@ -0,0 +1,29 @@
+/**
+ * The semantic channel the journal uses when the caller names none
+ * (docs/journal.md#search).
+ *
+ * The spec calls for event text to be embedded and searched by meaning beside
+ * the lexical indexes, and the default is zvec when `@4fu/arbor-search-zvec` is
+ * installed: `resolveSemanticIndex` loads it, and the journal uses what it
+ * returns when the caller names no channel. The package is an optional peer
+ * dependency, so a deployment that leaves it out keeps working.
+ *
+ * What is left here is the absence that answers when the package is not there:
+ * no channel, and search is exactly lexical. That is also the answer when the
+ * package is installed but its vector store will not open - zvec's prebuilt
+ * binary does not load on this platform, or another host holds the collection -
+ * and when no embedding model is available, because the package says so by
+ * handing back nothing.
+ */
+import type { SemanticIndex } from "@4fu/arbor-interfaces";
+/**
+ * The semantic channel a journal uses when the caller names none: the zvec one
+ * when the package is installed and its store opens, and none otherwise.
+ *
+ * `load` replaces how the zvec package is loaded. A host that resolves modules
+ * its own way passes its own; so does a test that has no zvec package to load. A
+ * load that fails falls back rather than stopping the journal, because the
+ * package is optional: it may be missing, or its prebuilt binary may not load on
+ * this platform.
+ */
+export declare function resolveSemanticIndex(path: string, load?: (path: string) => SemanticIndex | null): SemanticIndex | null;

package/semantic.js

@@ -0,0 +1,56 @@
+/**
+ * The semantic channel the journal uses when the caller names none
+ * (docs/journal.md#search).
+ *
+ * The spec calls for event text to be embedded and searched by meaning beside
+ * the lexical indexes, and the default is zvec when `@4fu/arbor-search-zvec` is
+ * installed: `resolveSemanticIndex` loads it, and the journal uses what it
+ * returns when the caller names no channel. The package is an optional peer
+ * dependency, so a deployment that leaves it out keeps working.
+ *
+ * What is left here is the absence that answers when the package is not there:
+ * no channel, and search is exactly lexical. That is also the answer when the
+ * package is installed but its vector store will not open - zvec's prebuilt
+ * binary does not load on this platform, or another host holds the collection -
+ * and when no embedding model is available, because the package says so by
+ * handing back nothing.
+ */
+import { createRequire } from "node:module";
+/** The package that holds the zvec semantic index, when a deployment installed it. */
+const ZVEC_PACKAGE = "@4fu/arbor-search-zvec";
+/**
+ * Load the zvec semantic index over a store beside the journal's own, or `null`
+ * when it is not there: the package is not installed, or it holds no index.
+ */
+function loadZvec(path) {
+    const require = createRequire(import.meta.url);
+    const loaded = require(ZVEC_PACKAGE);
+    const create = loaded?.createZvecIndex;
+    if (typeof create !== "function")
+        return null;
+    const index = create({ path });
+    if (index === null || typeof index !== "object")
+        return null;
+    const shape = index;
+    if (typeof shape.add !== "function" || typeof shape.search !== "function")
+        return null;
+    return index;
+}
+/**
+ * The semantic channel a journal uses when the caller names none: the zvec one
+ * when the package is installed and its store opens, and none otherwise.
+ *
+ * `load` replaces how the zvec package is loaded. A host that resolves modules
+ * its own way passes its own; so does a test that has no zvec package to load. A
+ * load that fails falls back rather than stopping the journal, because the
+ * package is optional: it may be missing, or its prebuilt binary may not load on
+ * this platform.
+ */
+export function resolveSemanticIndex(path, load = loadZvec) {
+    try {
+        return load(path);
+    }
+    catch {
+        return null;
+    }
+}

package/sqlite-journal.d.ts

@@ -18,9 +18,9 @@
  * the sequence an append assigns is atomic. The interface is asynchronous
  * because a `Journal` may live behind a process boundary.
  */
-import type { AroundOptions, BlobStore, Event, EventInput, EventPage, EventRef, Journal, Link, LinkInput, LinkPage, LinksOptions, ListOptions, Ref, SearchOptions, SearchPage, Segmenter, Target, TargetGap, TraceNode, TraceOptions, WindowRef } from "@4fu/arbor-interfaces";
-/** A query shorter than this cannot use the trigram index. */
-export declare const MIN_TRIGRAM_CHARS = 3;
+import type { AroundOptions, BlobStore, Event, EventInput, EventPage, EventRef, Journal, Link, LinkInput, LinkPage, LinksOptions, ListOptions, Ref, SearchOptions, SearchPage, Segmenter, SemanticIndex, Target, TargetGap, TraceNode, TraceOptions, WindowRef } from "@4fu/arbor-interfaces";
+import { MIN_TRIGRAM_CHARS } from "@4fu/arbor-lexical";
+export { MIN_TRIGRAM_CHARS };
 export interface SqliteJournalOptions {
     /** A file path, or `:memory:`. */
     path?: string;
@@ -28,6 +28,14 @@ export interface SqliteJournalOptions {
     blobs?: BlobStore;
     /** Word segmentation for the word index. */
     segmenter?: Segmenter;
+    /**
+     * The semantic channel: event text embedded and searched by meaning
+     * (docs/journal.md#search). Omitted, the zvec channel is used when
+     * `@4fu/arbor-search-zvec` is installed and its store opens, and there is no
+     * channel otherwise. `null` forces lexical search even when the package is
+     * installed.
+     */
+    semantic?: SemanticIndex | null;
     /** The clock. Tests pass a fixed one. */
     now?: () => number;
 }
@@ -40,8 +48,11 @@ export declare class SqliteJournal implements Journal {
     readonly path: string;
     private readonly db;
     private readonly segmenter;
+    private readonly semantic;
     private readonly clock;
     private readonly ownedBlobDirectory;
+    private semanticReady;
+    private semanticCaughtUp;
     private closed;
     constructor(options?: SqliteJournalOptions);
     append(input: EventInput): Promise<Event>;
@@ -57,6 +68,35 @@ export declare class SqliteJournal implements Journal {
     targetsBehind(window: WindowRef): Promise<TargetGap[]>;
     redact(ref: EventRef, reason: string): Promise<Event>;
     close(): Promise<void>;
+    /**
+     * Index the events the semantic channel does not hold yet, and say how many.
+     *
+     * A restart over an existing database, a channel that was not ready when the
+     * events arrived, and an append whose indexing failed all leave events to
+     * catch up on. A channel that is not ready yet holds what it was given and
+     * indexes that itself, so this answers 0 until it is.
+     */
+    catchUpSemantic(): Promise<number>;
+    /** Hand one event's text to the semantic channel, when there is one. */
+    private addToSemantic;
+    /** Take one event's text back out of the semantic channel, when there is one. */
+    private removeFromSemantic;
+    /**
+     * The semantic channel's candidates, as ranks in the order it gave them.
+     *
+     * An absent channel, one that is not ready yet, and one that refuses all
+     * answer with nothing, and the search is exactly the lexical one. The
+     * journal's own filters decide which candidates count, so a channel that
+     * filters less than the journal does still answers the read it was asked for.
+     */
+    private semanticHits;
+    /**
+     * The sequences among `refs` the journal's own filters allow, in the order
+     * given. A redacted event and one with no text are not among them, so a
+     * vector that outlived its content never reaches a page. The channel holds
+     * the ref a store gave it - an event ref here - so anything else is skipped.
+     */
+    private allowedSeqs;
     private ensureOpen;
     private insertChildren;
     private readEvent;

package/sqlite-journal.js

@@ -26,12 +26,19 @@ import { join } from "node:path";
 import { arborError } from "@4fu/arbor-errors";
 import { DEFAULT_PAGE_LIMIT, MAX_PAGE_LIMIT } from "@4fu/arbor-interfaces";
 import { FsBlobStore, blobIdOf } from "./blob-store.js";
-import { indexText, resolveSegmenter } from "./segmenter.js";
+import { indexText, mergeHits, resolveSegmenter, MIN_TRIGRAM_CHARS } from "@4fu/arbor-lexical";
+import { resolveSemanticIndex } from "./semantic.js";
 import { IMPLICIT_CITE_WEIGHT, TRIGRAM_TABLE, WORD_TABLE, createSchema } from "./schema.js";
 import { buildFilters, likePattern, ranked, substring, trigramMatch, wordMatch } from "./search.js";
 import { storedBlobRef, storedExternalRef, storedRef, storedWindowRef } from "./stored.js";
-/** A query shorter than this cannot use the trigram index. */
-export const MIN_TRIGRAM_CHARS = 3;
+export { MIN_TRIGRAM_CHARS };
+/**
+ * The fewest candidates the semantic channel is asked for. A page of one still
+ * fuses the channels over a handful of candidates, so a hit the word index
+ * ranked first is not lost to a semantic hit the channel would have ranked
+ * second.
+ */
+const MIN_SEMANTIC_CANDIDATES = 10;
 /** How many related items one read appends (docs/prompting.md). */
 const RELATED_LIMIT = 3;
 const AROUND_BEFORE = 5;
@@ -125,8 +132,11 @@ export class SqliteJournal {
     path;
     db;
     segmenter;
+    semantic;
     clock;
     ownedBlobDirectory;
+    semanticReady = false;
+    semanticCaughtUp = false;
     closed = false;
     constructor(options = {}) {
         this.path = options.path ?? ":memory:";
@@ -134,14 +144,15 @@ export class SqliteJournal {
         createSchema(this.db);
         this.segmenter = options.segmenter ?? resolveSegmenter();
         this.clock = options.now ?? (() => Date.now());
+        // An in-memory journal has no directory of its own; give it a temporary one
+        // and take it away on close. The blob store and the semantic channel's own
+        // store both live under it.
+        const root = this.path === ":memory:" ? join(tmpdir(), `arbor-journal-${randomUUID()}`) : null;
         if (options.blobs !== undefined) {
             this.blobs = options.blobs;
-            this.ownedBlobDirectory = null;
+            this.ownedBlobDirectory = root;
         }
-        else if (this.path === ":memory:") {
-            // An in-memory journal has no directory of its own; give it a temporary
-            // one and take it away on close.
-            const root = join(tmpdir(), `arbor-journal-${randomUUID()}`);
+        else if (root !== null) {
             this.blobs = new FsBlobStore(join(root, "blobs"), new SqlBlobRegistry(this.db, join(root, "blobs")));
             this.ownedBlobDirectory = root;
         }
@@ -150,6 +161,10 @@ export class SqliteJournal {
             this.blobs = new FsBlobStore(directory, new SqlBlobRegistry(this.db, directory));
             this.ownedBlobDirectory = null;
         }
+        this.semantic =
+            options.semantic === undefined
+                ? resolveSemanticIndex(root === null ? `${this.path}.vectors` : join(root, "vectors"))
+                : options.semantic;
     }
     async append(input) {
         this.ensureOpen();
@@ -174,6 +189,14 @@ export class SqliteJournal {
             this.insertChildren(assigned, input, deriveLinks(input.links, input.refs, eventRefOf(assigned), time));
             return assigned;
         });
+        // The semantic channel is asynchronous, so it is indexed once the event is
+        // committed: an append never waits on a model, and a channel that is not
+        // ready yet holds the text until it is. With no channel there is nothing to
+        // wait for, and the append takes the path it always did.
+        const body = input.body;
+        if (this.semantic !== null && body !== null && body !== undefined && body !== "") {
+            await this.addToSemantic(seq, body, input.kind, input.window ?? null);
+        }
         return this.readEvent(seq);
     }
     async get(ref) {
@@ -260,7 +283,11 @@ export class SqliteJournal {
             // scan is its fallback.
             substrings = substring(this.db, likePattern(trimmed), filters);
         }
-        const merged = mergeHits(substrings, words);
+        const semantic = this.semantic === null ? [] : await this.semanticHits(trimmed, limit, filters);
+        // Exact substring hits first, then word hits, then the hits only the
+        // semantic channel found: the vector channel adds what the lexical indexes
+        // missed and never moves what they found (docs/journal.md#search).
+        const merged = mergeHits({ channel: "substring", hits: substrings }, { channel: "word", hits: words }, { channel: "semantic", hits: semantic });
         const page = merged.slice(offset, offset + limit);
         const items = page.map((hit) => this.toSearchHit(hit));
         return {
@@ -435,6 +462,10 @@ export class SqliteJournal {
         });
         for (const blob of redaction.erased)
             await this.blobs.delete(storedBlobRef(blob));
+        // The vector the event left behind goes with its content: a redaction erases
+        // what the journal holds, and the semantic channel is one of those places.
+        if (this.semantic !== null)
+            await this.removeFromSemantic(seq);
         return this.readEvent(redaction.by);
     }
     async close() {
@@ -442,11 +473,124 @@ export class SqliteJournal {
             return;
         this.closed = true;
         this.db.close();
+        if (this.semantic !== null)
+            await this.semantic.close();
         if (this.ownedBlobDirectory !== null) {
             await rm(this.ownedBlobDirectory, { recursive: true, force: true });
         }
     }
+    /**
+     * Index the events the semantic channel does not hold yet, and say how many.
+     *
+     * A restart over an existing database, a channel that was not ready when the
+     * events arrived, and an append whose indexing failed all leave events to
+     * catch up on. A channel that is not ready yet holds what it was given and
+     * indexes that itself, so this answers 0 until it is.
+     */
+    async catchUpSemantic() {
+        this.ensureOpen();
+        if (this.semantic === null || !(await this.semantic.ready()))
+            return 0;
+        const held = await this.semantic.held();
+        const rows = this.db
+            .prepare(`SELECT seq, kind, window, body FROM events
+         WHERE body IS NOT NULL AND redacted IS NULL ORDER BY seq`)
+            .all();
+        let indexed = 0;
+        for (const row of rows) {
+            const seq = toNumber(row.seq);
+            if (held.has(eventRefOf(seq)))
+                continue;
+            await this.addToSemantic(seq, row.body, row.kind, storedWindowRef(row.window));
+            indexed += 1;
+        }
+        return indexed;
+    }
     // --- internals ---------------------------------------------------------
+    /** Hand one event's text to the semantic channel, when there is one. */
+    async addToSemantic(seq, body, kind, window) {
+        if (this.semantic === null)
+            return;
+        try {
+            await this.semantic.add(eventRefOf(seq), body, { kind, window });
+        }
+        catch {
+            // An optional channel that refuses leaves the event searchable lexically;
+            // a catch-up indexes it once the channel answers again.
+        }
+    }
+    /** Take one event's text back out of the semantic channel, when there is one. */
+    async removeFromSemantic(seq) {
+        if (this.semantic === null)
+            return;
+        try {
+            await this.semantic.remove(eventRefOf(seq));
+        }
+        catch {
+            // As with an append: the channel is optional, and the event is gone from
+            // every index the journal owns.
+        }
+    }
+    /**
+     * The semantic channel's candidates, as ranks in the order it gave them.
+     *
+     * An absent channel, one that is not ready yet, and one that refuses all
+     * answer with nothing, and the search is exactly the lexical one. The
+     * journal's own filters decide which candidates count, so a channel that
+     * filters less than the journal does still answers the read it was asked for.
+     */
+    async semanticHits(query, limit, filters) {
+        if (this.semantic === null)
+            return [];
+        const ready = await this.semantic.ready();
+        if (ready !== this.semanticReady) {
+            // The channel just became usable, or stopped being: index what it missed
+            // while it was not.
+            this.semanticReady = ready;
+            this.semanticCaughtUp = false;
+        }
+        if (!this.semanticCaughtUp) {
+            this.semanticCaughtUp = true;
+            await this.catchUpSemantic();
+        }
+        if (!ready)
+            return [];
+        const asked = {
+            ...(filters.kinds === undefined ? {} : { kinds: filters.kinds }),
+            ...(filters.window === undefined ? {} : { window: filters.window }),
+            ...(filters.afterSeq === undefined ? {} : { afterSeq: filters.afterSeq }),
+            ...(filters.beforeSeq === undefined ? {} : { beforeSeq: filters.beforeSeq }),
+        };
+        const refs = await this.semantic.search(query, Math.max(limit, MIN_SEMANTIC_CANDIDATES), asked);
+        const allowed = this.allowedSeqs(refs, filters);
+        // The channel's order, as ranks the shared merge reads: the nearest hit
+        // carries the most negative rank, so it normalizes to the best score.
+        return allowed.map((rowid, index) => ({ rowid, rank: index - allowed.length }));
+    }
+    /**
+     * The sequences among `refs` the journal's own filters allow, in the order
+     * given. A redacted event and one with no text are not among them, so a
+     * vector that outlived its content never reaches a page. The channel holds
+     * the ref a store gave it - an event ref here - so anything else is skipped.
+     */
+    allowedSeqs(refs, filters) {
+        const seqs = [];
+        for (const ref of refs) {
+            const seq = parseEventRef(ref);
+            if (seq !== null)
+                seqs.push(seq);
+        }
+        if (seqs.length === 0)
+            return [];
+        const built = buildFilters(filters);
+        const placeholders = seqs.map(() => "?").join(", ");
+        const rows = this.db
+            .prepare(`SELECT e.seq AS seq FROM events e
+         WHERE e.seq IN (${placeholders}) AND e.body IS NOT NULL AND e.redacted IS NULL${built.sql}`)
+            .all(...seqs, ...built.params);
+        const allowed = new Set(rows.map((row) => toNumber(row.seq)));
+        return seqs.filter((seq) => allowed.has(seq));
+    }
     ensureOpen() {
         if (this.closed) {
             throw arborError({
@@ -567,7 +711,7 @@ export class SqliteJournal {
         return { window: event.window, time: event.time, summary: summaryOf(event.body) };
     }
     toSearchHit(hit) {
-        const event = this.readEvent(hit.seq);
+        const event = this.readEvent(hit.rowid);
         if (event === null) {
             throw arborError({
                 code: "internal",
@@ -582,7 +726,7 @@ export class SqliteJournal {
             window: event.window,
             kind: event.kind,
             time: event.time,
-            match: hit.match,
+            match: hit.channel,
             score: hit.score,
             summary: summaryOf(event.body),
         };
@@ -612,32 +756,6 @@ export function deriveLinks(declared, refs, from, time) {
     }
     return links;
 }
-/**
- * Merge the two indexes: exact substring hits first, then word hits that are
- * not already there, each ordered by its own relevance.
- */
-function mergeHits(substrings, words) {
-    const seen = new Set();
-    const merged = [];
-    const push = (hits, match) => {
-        const best = Math.min(...hits.map((hit) => hit.rank));
-        for (const hit of hits) {
-            if (seen.has(hit.seq))
-                continue;
-            seen.add(hit.seq);
-            merged.push({ seq: hit.seq, match, score: normalize(hit.rank, best) });
-        }
-    };
-    push(substrings, "substring");
-    push(words, "word");
-    return merged;
-}
-/** Relevance in 0..1 within one index; higher is better. */
-function normalize(rank, best) {
-    if (best === 0)
-        return 1;
-    return Math.max(0, Math.min(1, rank / best));
-}
 function filtersOf(options) {
     return {
         kinds: options.kinds?.map(String),

package/segmenter.d.ts

@@ -1,30 +0,0 @@
-/**
- * Word segmentation for the journal's word index (docs/journal.md#search).
- *
- * The spec calls for jieba segmentation with BM25, and the default is jieba
- * when `@4fu/arbor-segmenter-jieba` is installed: `resolveSegmenter` loads it, and
- * the journal uses what it returns when the caller names no segmenter. The
- * package is an optional peer dependency, so a deployment that leaves it out
- * keeps working.
- *
- * What is left here is the approximation that answers when jieba is not there:
- * Chinese, Japanese and Korean runs are indexed per character and per bigram,
- * and every other script per word. Character and bigram indexing is what makes
- * a two-character Chinese query find the phrase it appears in, without a
- * dictionary.
- */
-import type { Segmenter } from "@4fu/arbor-interfaces";
-export declare const fallbackSegmenter: Segmenter;
-/**
- * The segmenter a journal uses when the caller names none: jieba when the
- * package is installed, the per-character-and-bigram approximation otherwise.
- *
- * `load` replaces how the jieba package is loaded. A host that resolves modules
- * its own way passes its own; so does a test that has no jieba package to load.
- * A load that fails falls back rather than stopping the journal, because the
- * package is optional: it may be missing, or its prebuilt binary may not load
- * on this platform.
- */
-export declare function resolveSegmenter(load?: () => Segmenter | null): Segmenter;
-/** The text the word index stores: the segments, one per line. */
-export declare function indexText(text: string, segmenter: Segmenter): string;

package/segmenter.js

@@ -1,107 +0,0 @@
-/**
- * Word segmentation for the journal's word index (docs/journal.md#search).
- *
- * The spec calls for jieba segmentation with BM25, and the default is jieba
- * when `@4fu/arbor-segmenter-jieba` is installed: `resolveSegmenter` loads it, and
- * the journal uses what it returns when the caller names no segmenter. The
- * package is an optional peer dependency, so a deployment that leaves it out
- * keeps working.
- *
- * What is left here is the approximation that answers when jieba is not there:
- * Chinese, Japanese and Korean runs are indexed per character and per bigram,
- * and every other script per word. Character and bigram indexing is what makes
- * a two-character Chinese query find the phrase it appears in, without a
- * dictionary.
- */
-import { createRequire } from "node:module";
-/** The package that holds the jieba segmenter, when a deployment installed it. */
-const JIEBA_PACKAGE = "@4fu/arbor-segmenter-jieba";
-/** CJK unified ideographs, extensions, compatibility, kana and hangul. */
-function isCjk(codePoint) {
-    return ((codePoint >= 0x3400 && codePoint <= 0x4dbf) ||
-        (codePoint >= 0x4e00 && codePoint <= 0x9fff) ||
-        (codePoint >= 0xf900 && codePoint <= 0xfaff) ||
-        (codePoint >= 0x3040 && codePoint <= 0x30ff) ||
-        (codePoint >= 0xac00 && codePoint <= 0xd7af) ||
-        (codePoint >= 0x20000 && codePoint <= 0x2a6df) ||
-        (codePoint >= 0x2a700 && codePoint <= 0x2ebef));
-}
-/** Whitespace and punctuation separate words; everything else is part of one. */
-const SEPARATOR = /[\s\p{P}\p{S}\p{C}\p{Z}]/u;
-export const fallbackSegmenter = {
-    segment(text) {
-        const chars = Array.from(text);
-        const tokens = [];
-        let index = 0;
-        while (index < chars.length) {
-            const first = chars[index];
-            const codePoint = first.codePointAt(0);
-            if (isCjk(codePoint)) {
-                const run = [];
-                while (index < chars.length) {
-                    const char = chars[index];
-                    const point = char.codePointAt(0);
-                    if (!isCjk(point))
-                        break;
-                    run.push(char);
-                    index += 1;
-                }
-                tokens.push(...run);
-                for (let offset = 0; offset + 1 < run.length; offset += 1) {
-                    tokens.push(run[offset] + run[offset + 1]);
-                }
-                continue;
-            }
-            if (SEPARATOR.test(first)) {
-                index += 1;
-                continue;
-            }
-            const run = [];
-            while (index < chars.length) {
-                const char = chars[index];
-                const point = char.codePointAt(0);
-                if (isCjk(point) || SEPARATOR.test(char))
-                    break;
-                run.push(char);
-                index += 1;
-            }
-            tokens.push(run.join("").toLowerCase());
-        }
-        return tokens;
-    },
-};
-/**
- * Load the jieba segmenter, or `null` when it is not there: the package is not
- * installed, or it holds no segmenter.
- */
-function loadJieba() {
-    const require = createRequire(import.meta.url);
-    const loaded = require(JIEBA_PACKAGE);
-    const candidate = loaded?.jiebaSegmenter;
-    if (candidate === null || typeof candidate !== "object")
-        return null;
-    const shape = candidate;
-    return typeof shape.segment === "function" ? candidate : null;
-}
-/**
- * The segmenter a journal uses when the caller names none: jieba when the
- * package is installed, the per-character-and-bigram approximation otherwise.
- *
- * `load` replaces how the jieba package is loaded. A host that resolves modules
- * its own way passes its own; so does a test that has no jieba package to load.
- * A load that fails falls back rather than stopping the journal, because the
- * package is optional: it may be missing, or its prebuilt binary may not load
- * on this platform.
- */
-export function resolveSegmenter(load = loadJieba) {
-    try {
-        return load() ?? fallbackSegmenter;
-    }
-    catch {
-        return fallbackSegmenter;
-    }
-}
-/** The text the word index stores: the segments, one per line. */
-export function indexText(text, segmenter) {
-    return segmenter.segment(text).join("\n");
-}