@4fu/arbor-journal-sqlite 0.0.0-stage → 0.1.0

package/search.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Lexical search over the journal's events (docs/journal.md#search).
+ *
+ * Three indexes, merged:
+ *
+ * - a word index, segmented and ranked with BM25;
+ * - a trigram index for exact substrings, which is what makes a Chinese
+ *   substring findable;
+ * - a substring scan, for one- and two-character queries and for queries the
+ *   trigram index cannot answer.
+ *
+ * Exact substring hits come first, then word hits that are not already there.
+ */
+import type { DatabaseSync, SQLInputValue } from "node:sqlite";
+export interface SearchFilters {
+    kinds?: readonly string[];
+    window?: string;
+    targets?: readonly string[];
+    external?: string;
+    afterSeq?: number;
+    beforeSeq?: number;
+}
+/** One matching row, with its raw relevance. Lower `rank` is better. */
+export interface Ranked {
+    seq: number;
+    rank: number;
+}
+export interface FilterSql {
+    sql: string;
+    params: SQLInputValue[];
+}
+/**
+ * The filters a read applies, as SQL over `events e`. A target filter is an
+ * `EXISTS`, so one event with three targets still counts once.
+ */
+export declare function buildFilters(filters: SearchFilters): FilterSql;
+/** Every match from one FTS5 index, ordered by relevance. */
+export declare function ranked(db: DatabaseSync, table: string, match: string, filters: SearchFilters): Ranked[];
+/** Every substring match, newest first. The scan has no relevance of its own. */
+export declare function substring(db: DatabaseSync, pattern: string, filters: SearchFilters): Ranked[];
+/** Escape a literal for `LIKE ... ESCAPE '\'`. */
+export declare function escapeLike(value: string): string;
+export declare function likePattern(query: string): string;
+/** Quote one term for an FTS5 `MATCH`, so outside text is never syntax. */
+export declare function quoteFts(value: string): string;
+/** The trigram index takes one phrase. */
+export declare function trigramMatch(query: string): string;
+/** The word index takes the query's segments, all of them required. */
+export declare function wordMatch(tokens: readonly string[]): string;

package/search.js

@@ -0,0 +1,89 @@
+/**
+ * Lexical search over the journal's events (docs/journal.md#search).
+ *
+ * Three indexes, merged:
+ *
+ * - a word index, segmented and ranked with BM25;
+ * - a trigram index for exact substrings, which is what makes a Chinese
+ *   substring findable;
+ * - a substring scan, for one- and two-character queries and for queries the
+ *   trigram index cannot answer.
+ *
+ * Exact substring hits come first, then word hits that are not already there.
+ */
+/**
+ * The filters a read applies, as SQL over `events e`. A target filter is an
+ * `EXISTS`, so one event with three targets still counts once.
+ */
+export function buildFilters(filters) {
+    const clauses = [];
+    const params = [];
+    if (filters.kinds !== undefined && filters.kinds.length > 0) {
+        clauses.push(`AND e.kind IN (${filters.kinds.map(() => "?").join(", ")})`);
+        params.push(...filters.kinds);
+    }
+    if (filters.window !== undefined) {
+        clauses.push("AND e.window = ?");
+        params.push(filters.window);
+    }
+    if (filters.external !== undefined) {
+        clauses.push("AND e.external = ?");
+        params.push(filters.external);
+    }
+    for (const target of filters.targets ?? []) {
+        clauses.push("AND EXISTS (SELECT 1 FROM event_targets et WHERE et.seq = e.seq AND et.target = ?)");
+        params.push(target);
+    }
+    if (filters.afterSeq !== undefined) {
+        clauses.push("AND e.seq > ?");
+        params.push(filters.afterSeq);
+    }
+    if (filters.beforeSeq !== undefined) {
+        clauses.push("AND e.seq < ?");
+        params.push(filters.beforeSeq);
+    }
+    return { sql: clauses.length === 0 ? "" : ` ${clauses.join(" ")}`, params };
+}
+/** Every match from one FTS5 index, ordered by relevance. */
+export function ranked(db, table, match, filters) {
+    const built = buildFilters(filters);
+    const rows = db
+        .prepare(`SELECT f.rowid AS seq, bm25(${table}) AS rank
+       FROM ${table} f JOIN events e ON e.seq = f.rowid
+       WHERE ${table} MATCH ?${built.sql}
+       ORDER BY rank, f.rowid`)
+        .all(match, ...built.params);
+    return rows.map((row) => ({ seq: Number(row.seq), rank: Number(row.rank) }));
+}
+/** Every substring match, newest first. The scan has no relevance of its own. */
+export function substring(db, pattern, filters) {
+    const built = buildFilters(filters);
+    const rows = db
+        .prepare(`SELECT e.seq AS seq, 1.0 AS rank FROM events e
+       WHERE e.body LIKE ? ESCAPE '\\'${built.sql}
+       ORDER BY e.seq DESC`)
+        .all(pattern, ...built.params);
+    return rows.map((row) => ({ seq: Number(row.seq), rank: Number(row.rank) }));
+}
+/** Escape a literal for `LIKE ... ESCAPE '\'`. */
+export function escapeLike(value) {
+    return value.replace(/[\\%_]/g, (character) => `\\${character}`);
+}
+export function likePattern(query) {
+    return `%${escapeLike(query)}%`;
+}
+/** Quote one term for an FTS5 `MATCH`, so outside text is never syntax. */
+export function quoteFts(value) {
+    return `"${value.replace(/"/g, '""')}"`;
+}
+/** The trigram index takes one phrase. */
+export function trigramMatch(query) {
+    return quoteFts(query);
+}
+/** The word index takes the query's segments, all of them required. */
+export function wordMatch(tokens) {
+    return tokens
+        .filter((token) => token.length > 0)
+        .map(quoteFts)
+        .join(" ");
+}

package/segmenter.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Word segmentation for the journal's word index (docs/journal.md#search).
+ *
+ * The spec calls for jieba segmentation with BM25, and the default is jieba
+ * when `@4fu/arbor-segmenter-jieba` is installed: `resolveSegmenter` loads it, and
+ * the journal uses what it returns when the caller names no segmenter. The
+ * package is an optional peer dependency, so a deployment that leaves it out
+ * keeps working.
+ *
+ * What is left here is the approximation that answers when jieba is not there:
+ * Chinese, Japanese and Korean runs are indexed per character and per bigram,
+ * and every other script per word. Character and bigram indexing is what makes
+ * a two-character Chinese query find the phrase it appears in, without a
+ * dictionary.
+ */
+import type { Segmenter } from "@4fu/arbor-interfaces";
+export declare const fallbackSegmenter: Segmenter;
+/**
+ * The segmenter a journal uses when the caller names none: jieba when the
+ * package is installed, the per-character-and-bigram approximation otherwise.
+ *
+ * `load` replaces how the jieba package is loaded. A host that resolves modules
+ * its own way passes its own; so does a test that has no jieba package to load.
+ * A load that fails falls back rather than stopping the journal, because the
+ * package is optional: it may be missing, or its prebuilt binary may not load
+ * on this platform.
+ */
+export declare function resolveSegmenter(load?: () => Segmenter | null): Segmenter;
+/** The text the word index stores: the segments, one per line. */
+export declare function indexText(text: string, segmenter: Segmenter): string;

package/segmenter.js

@@ -0,0 +1,107 @@
+/**
+ * Word segmentation for the journal's word index (docs/journal.md#search).
+ *
+ * The spec calls for jieba segmentation with BM25, and the default is jieba
+ * when `@4fu/arbor-segmenter-jieba` is installed: `resolveSegmenter` loads it, and
+ * the journal uses what it returns when the caller names no segmenter. The
+ * package is an optional peer dependency, so a deployment that leaves it out
+ * keeps working.
+ *
+ * What is left here is the approximation that answers when jieba is not there:
+ * Chinese, Japanese and Korean runs are indexed per character and per bigram,
+ * and every other script per word. Character and bigram indexing is what makes
+ * a two-character Chinese query find the phrase it appears in, without a
+ * dictionary.
+ */
+import { createRequire } from "node:module";
+/** The package that holds the jieba segmenter, when a deployment installed it. */
+const JIEBA_PACKAGE = "@4fu/arbor-segmenter-jieba";
+/** CJK unified ideographs, extensions, compatibility, kana and hangul. */
+function isCjk(codePoint) {
+    return ((codePoint >= 0x3400 && codePoint <= 0x4dbf) ||
+        (codePoint >= 0x4e00 && codePoint <= 0x9fff) ||
+        (codePoint >= 0xf900 && codePoint <= 0xfaff) ||
+        (codePoint >= 0x3040 && codePoint <= 0x30ff) ||
+        (codePoint >= 0xac00 && codePoint <= 0xd7af) ||
+        (codePoint >= 0x20000 && codePoint <= 0x2a6df) ||
+        (codePoint >= 0x2a700 && codePoint <= 0x2ebef));
+}
+/** Whitespace and punctuation separate words; everything else is part of one. */
+const SEPARATOR = /[\s\p{P}\p{S}\p{C}\p{Z}]/u;
+export const fallbackSegmenter = {
+    segment(text) {
+        const chars = Array.from(text);
+        const tokens = [];
+        let index = 0;
+        while (index < chars.length) {
+            const first = chars[index];
+            const codePoint = first.codePointAt(0);
+            if (isCjk(codePoint)) {
+                const run = [];
+                while (index < chars.length) {
+                    const char = chars[index];
+                    const point = char.codePointAt(0);
+                    if (!isCjk(point))
+                        break;
+                    run.push(char);
+                    index += 1;
+                }
+                tokens.push(...run);
+                for (let offset = 0; offset + 1 < run.length; offset += 1) {
+                    tokens.push(run[offset] + run[offset + 1]);
+                }
+                continue;
+            }
+            if (SEPARATOR.test(first)) {
+                index += 1;
+                continue;
+            }
+            const run = [];
+            while (index < chars.length) {
+                const char = chars[index];
+                const point = char.codePointAt(0);
+                if (isCjk(point) || SEPARATOR.test(char))
+                    break;
+                run.push(char);
+                index += 1;
+            }
+            tokens.push(run.join("").toLowerCase());
+        }
+        return tokens;
+    },
+};
+/**
+ * Load the jieba segmenter, or `null` when it is not there: the package is not
+ * installed, or it holds no segmenter.
+ */
+function loadJieba() {
+    const require = createRequire(import.meta.url);
+    const loaded = require(JIEBA_PACKAGE);
+    const candidate = loaded?.jiebaSegmenter;
+    if (candidate === null || typeof candidate !== "object")
+        return null;
+    const shape = candidate;
+    return typeof shape.segment === "function" ? candidate : null;
+}
+/**
+ * The segmenter a journal uses when the caller names none: jieba when the
+ * package is installed, the per-character-and-bigram approximation otherwise.
+ *
+ * `load` replaces how the jieba package is loaded. A host that resolves modules
+ * its own way passes its own; so does a test that has no jieba package to load.
+ * A load that fails falls back rather than stopping the journal, because the
+ * package is optional: it may be missing, or its prebuilt binary may not load
+ * on this platform.
+ */
+export function resolveSegmenter(load = loadJieba) {
+    try {
+        return load() ?? fallbackSegmenter;
+    }
+    catch {
+        return fallbackSegmenter;
+    }
+}
+/** The text the word index stores: the segments, one per line. */
+export function indexText(text, segmenter) {
+    return segmenter.segment(text).join("\n");
+}

package/sqlite-journal.d.ts

@@ -0,0 +1,75 @@
+/**
+ * The default `Journal`: SQLite (docs/journal.md).
+ *
+ * What this implementation owns:
+ *
+ * - the one global order: `seq`, assigned at append and only growing, with the
+ *   ref `r<seq>`;
+ * - targets, native ids with a unique index, refs and the links the host
+ *   records between them;
+ * - media held by id, with the bytes in a blob store beside the database;
+ * - lexical search: a word index with BM25, a trigram index for exact
+ *   substrings, and a substring scan for one- and two-character queries,
+ *   merged with exact substring hits first;
+ * - watermarks per window and target, and the targets a window is behind on;
+ * - redaction, the one thing that erases content.
+ *
+ * It is synchronous inside: one `DatabaseSync`, one transaction per append, so
+ * the sequence an append assigns is atomic. The interface is asynchronous
+ * because a `Journal` may live behind a process boundary.
+ */
+import type { AroundOptions, BlobStore, Event, EventInput, EventPage, EventRef, Journal, Link, LinkInput, LinkPage, LinksOptions, ListOptions, Ref, SearchOptions, SearchPage, Segmenter, Target, TargetGap, TraceNode, TraceOptions, WindowRef } from "@4fu/arbor-interfaces";
+/** A query shorter than this cannot use the trigram index. */
+export declare const MIN_TRIGRAM_CHARS = 3;
+export interface SqliteJournalOptions {
+    /** A file path, or `:memory:`. */
+    path?: string;
+    /** Media storage. Defaults to a directory beside the database. */
+    blobs?: BlobStore;
+    /** Word segmentation for the word index. */
+    segmenter?: Segmenter;
+    /** The clock. Tests pass a fixed one. */
+    now?: () => number;
+}
+export declare function parseEventRef(ref: string): number | null;
+export declare function eventRefOf(seq: number): EventRef;
+/** The opening words of a body, for a one-line result. */
+export declare function summaryOf(body: string | null, max?: number): string;
+export declare class SqliteJournal implements Journal {
+    readonly blobs: BlobStore;
+    readonly path: string;
+    private readonly db;
+    private readonly segmenter;
+    private readonly clock;
+    private readonly ownedBlobDirectory;
+    private closed;
+    constructor(options?: SqliteJournalOptions);
+    append(input: EventInput): Promise<Event>;
+    get(ref: EventRef): Promise<Event | null>;
+    getMany(refs: readonly EventRef[]): Promise<Event[]>;
+    list(options?: ListOptions): Promise<EventPage>;
+    around(ref: EventRef, options?: AroundOptions): Promise<Event[]>;
+    search(query: string, options?: SearchOptions): Promise<SearchPage>;
+    links(ref: Ref, options?: LinksOptions): Promise<LinkPage>;
+    trace(ref: Ref, options?: TraceOptions): Promise<TraceNode>;
+    watermark(window: WindowRef, target: Target): Promise<number | null>;
+    advanceWatermark(window: WindowRef, target: Target, seq: number): Promise<number>;
+    targetsBehind(window: WindowRef): Promise<TargetGap[]>;
+    redact(ref: EventRef, reason: string): Promise<Event>;
+    close(): Promise<void>;
+    private ensureOpen;
+    private insertChildren;
+    private readEvent;
+    private readEventByRef;
+    /** One column of one table, for the child rows of an event. */
+    private strings;
+    private countEvents;
+    private latestOn;
+    /** Where a ref came from: the links that go out of it, strongest first. */
+    private sourcesOf;
+    private resolveRelated;
+    private toSearchHit;
+}
+/** The links one append records: what it declared, plus a bare cite per ref. */
+export declare function deriveLinks(declared: readonly LinkInput[] | undefined, refs: readonly Ref[] | undefined, from: EventRef, time: number): Link[];
+export { FsBlobStore, blobIdOf } from "./blob-store.ts";