@4fu/arbor-lexical 0.0.0-stage → 2026.1107.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 4fuu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,3 +1,196 @@
-# Temporary Holding Version
+# arbor
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+arbor is a framework for an agent without sessions: many context windows run
+at once over one shared record, so work moves in parallel, replies come fast,
+and every window can find everything that was ever done.
+
+A deployment composes the pieces it wants and registers plugins for the domains
+it serves. The framework guarantees what an agent cannot guarantee for itself -
+the record, isolation, exactly-once effects, the checks, dispatch - and leaves
+everything that takes judgment, such as git, merges and conflicts, to the agent.
+
+## Core ideas
+
+- **No sessions to manage.** Work runs in disposable context windows. Any
+  number run at the same time; none of them owns anything, and a window that
+  ends loses nothing. The model looks up what it needs instead of carrying it.
+- **One record, shared live.** Every input, tool call, result, send and note
+  revision goes into one journal and is visible to every window as
+  soon as it is written. Records never merge and are never lost.
+- **Dispatch by a judge.** A judge (Jev by default, replaceable) decides which
+  live window takes a new input, or opens a new one. There is no router session
+  and no main line.
+- **Current without locks.** Windows run in parallel. New events on a
+  conversation a window is in are steered into it as they arrive; only a note
+  write is checked against the revision the window read.
+- **Mechanisms in the framework, policy in the agent.** The framework
+  guarantees what an agent cannot guarantee for itself - the record, isolation,
+  exactly-once effects, the checks, dispatch. Everything that takes judgment,
+  such as git, merges and conflicts, is the agent's.
+- **A shared workspace with per-window checkouts.** The default workspace is
+  ovnfs: one shared folder every window reads, and a window opens its own
+  copy-on-write checkout of the project it needs, works there and merges back
+  with `apply`. It falls back to a plain directory per window when ovnfs is not
+  installed. It is not a sandbox: ovnfs isolates files, not processes.
+- **A self-changing environment.** The model adds and changes skills, MCP
+  servers and resident services itself; a `reload` validates the change and
+  broadcasts it to every live window as a delta of names and descriptions.
+- **Minimal limits.** The framework sets no policy. Permissions, privacy
+  review, budgets and approvals belong to deployments built on it, through
+  hooks.
+- **Direct tools plus code mode.** Decisions are direct tools; everything else
+  is a module called from `code_exec`. There is no CLI.
+- **Every module is replaceable.** The agent backend (pi by default), the
+  journal, notes, workspaces, the judge, channels and the rest are interfaces
+  ([docs/architecture.md](docs/architecture.md#interfaces)).
+
+## Install
+
+Node 24 or newer. Every package is published separately, so a deployment
+installs the core and the defaults it wants:
+
+```sh
+npm install @4fu/arbor-core @4fu/arbor-journal-sqlite @4fu/arbor-notes-sqlite \
+  @4fu/arbor-environment-dir @4fu/arbor-supervisor-process @4fu/arbor-agent-pi \
+  @4fu/arbor-workspace-dir
+```
+
+`@4fu/arbor-plugin-http` is optional; it is the example plugin below.
+`@4fu/arbor-segmenter-jieba` is an optional peer of the lexical package, and
+`@4fu/arbor-search-zvec` an optional peer of the journal: absent either one, the
+journal searches lexically and nothing else changes.
+
+## A minimal host
+
+The smallest host needs a journal and a backend. Adding the defaults and one
+plugin is what a deployment usually starts from:
+
+```js
+import { mkdtemp } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { Arbor } from "@4fu/arbor-core";
+import { DirectoryEnvironment } from "@4fu/arbor-environment-dir";
+import { SqliteJournal } from "@4fu/arbor-journal-sqlite";
+import { SqliteNoteStore } from "@4fu/arbor-notes-sqlite";
+import { ProcessSupervisor } from "@4fu/arbor-supervisor-process";
+import { DirectoryWorkspace } from "@4fu/arbor-workspace-dir";
+import { PiAgentBackend } from "@4fu/arbor-agent-pi";
+import { HttpChannel, httpPlugin } from "@4fu/arbor-plugin-http";
+
+const root = await mkdtemp(join(tmpdir(), "arbor-"));
+
+// Every module below is a default implementation of an interface. Replace any
+// one by passing your own; the host does not change.
+const journal = new SqliteJournal({ path: join(root, "journal.sqlite") });
+const notes = new SqliteNoteStore({ path: join(root, "notes.sqlite"), journal });
+const supervisor = new ProcessSupervisor({ root: join(root, "services") });
+const environment = new DirectoryEnvironment({ root: join(root, "env"), supervisor });
+const workspace = new DirectoryWorkspace({ root: join(root, "workspaces") });
+
+// A plugin adds; it never patches the core. This one is a chat channel: a
+// webhook in, a window's sends out, and a page that drives the whole host.
+let arbor;
+const channel = new HttpChannel({ host: "127.0.0.1", port: 8787, board: () => arbor.boardLines() });
+arbor = new Arbor({
+  journal,
+  notes,
+  backend: new PiAgentBackend(),
+  workspace,
+  environment,
+  supervisor,
+  plugins: [httpPlugin({ channel })],
+});
+
+await arbor.start();
+await channel.listen();
+```
+
+`arbor.open(task)` dispatches one task, and the journal is where everything that
+happened can be read back ([docs/journal.md](docs/journal.md)). The workspace a
+window gets, and how a continuation keeps working in it, are in
+[docs/workspaces.md](docs/workspaces.md).
+
+## Packages
+
+| Package | What it is |
+| --- | --- |
+| [`@4fu/arbor-interfaces`](packages/interfaces) | every replaceable interface, plus the core's own shapes. Types only. |
+| [`@4fu/arbor-errors`](packages/errors) | the error shape every refusal uses: the rule and the next action |
+| [`@4fu/arbor-prompt`](packages/prompt) | typed blocks, the one deterministic renderer and the replaceable text table |
+| [`@4fu/arbor-tools`](packages/tools) | the entry points: direct tools, code mode modules, the host API |
+| [`@4fu/arbor-journal-sqlite`](packages/journal-sqlite) | the default `Journal` and `BlobStore` on SQLite |
+| [`@4fu/arbor-notes-sqlite`](packages/notes-sqlite) | the default `NoteStore` on SQLite, beside the journal |
+| [`@4fu/arbor-segmenter-jieba`](packages/segmenter-jieba) | the default `Segmenter`: jieba's dictionary segmentation for the word index, optional |
+| [`@4fu/arbor-lexical`](packages/lexical) | the lexical search the journal and notes share: word segmentation, the FTS5 helpers and the merge order |
+| [`@4fu/arbor-search-zvec`](packages/search-zvec) | the optional semantic channel: event text embedded and searched by meaning with zvec |
+| [`@4fu/arbor-workspace-dir`](packages/workspace-dir) | the fallback `Workspace`: one directory per window, with a continuation working in its predecessor's directory in place |
+| [`@4fu/arbor-workspace-ovnfs`](packages/workspace-ovnfs) | the default `Workspace` when ovnfs is installed: one shared folder, per-window copy-on-write checkouts the model opens and applies back |
+| [`@4fu/arbor-environment-dir`](packages/environment-dir) | the default `Environment`: one `/env` directory, a revision per item, validation and deltas |
+| [`@4fu/arbor-supervisor-process`](packages/supervisor-process) | the default `Supervisor`: one host process per service, restart with backoff, logs |
+| [`@4fu/arbor-judge-jev`](packages/judge-jev) | the default `Judge`: Jev, TypeSafe AI's System One classifier, answering the dispatch decisions |
+| [`@4fu/arbor-agent-pi`](packages/agent-pi) | the default `AgentBackend`: pi-agent-core in process |
+| [`@4fu/arbor-plugin-http`](packages/plugin-http) | the HTTP test channel: a webhook in, sends out, a page that drives the whole host |
+| [`@4fu/arbor-core`](packages/core) | the service layer: one implementation of every operation - the window lifecycle, dispatch, coordination, the board, notes access, workspaces, effects and schedules |
+
+### Defaults are examples
+
+Every default above is a reference implementation of its interface, written to
+be read and replaced. A real deployment is expected to swap out the ones that do
+not fit: the SQLite journal and notes, the directory workspace, the process
+supervisor, the environment directory, the HTTP test plugin and the Jev judge
+are all examples of what an interface can be satisfied by, not commitments.
+What a deployment keeps is its own choice, and the interfaces are the contract
+([docs/architecture.md](docs/architecture.md#interfaces)).
+
+Each package's own documentation holds the limits of that default, so a
+deployment can see what it would be taking on before it adopts one.
+
+## Documents
+
+| Page | Contents |
+| --- | --- |
+| [docs/overview.md](docs/overview.md) | goals, concepts, principles, routes considered |
+| [docs/architecture.md](docs/architecture.md) | layers, interfaces, stack, models, plugins, hooks, data placement |
+| [docs/journal.md](docs/journal.md) | events and their order, targets, watermarks, external ids, refs, tracing, media, redaction |
+| [docs/windows.md](docs/windows.md) | window lifecycle, prompt layout, reminders, continuation |
+| [docs/dispatch.md](docs/dispatch.md) | task sources, judge decisions, insertion timing, backpressure, scheduled tasks |
+| [docs/coordination.md](docs/coordination.md) | the board, expectations, deliveries and broadcasts, staying current, effects |
+| [docs/environment.md](docs/environment.md) | skills, MCP servers and services: using, changing, reload, built-in skills, deltas |
+| [docs/memory.md](docs/memory.md) | notes, the active set, anchors, links |
+| [docs/workspaces.md](docs/workspaces.md) | the shared folder and per-window checkouts: isolation, continuity, what the agent does |
+| [docs/tools.md](docs/tools.md) | direct tools and code mode modules |
+| [docs/prompting.md](docs/prompting.md) | how every model-facing text is written: layers, notes, deliveries, errors, judge questions, terms |
+| [docs/release.md](docs/release.md) | calendar versioning, the manual release steps, how a package is published |
+
+## Repository layout
+
+`pnpm check` typechecks and tests every package on its TypeScript sources.
+`pnpm build` emits the `dist` publishing needs, and `pnpm pack` shows what each
+tarball would contain ([docs/release.md](docs/release.md)).
+
+| Path | Layer |
+| --- | --- |
+| `packages/interfaces` | the interfaces layer: every replaceable interface, plus the core's own shapes. Types only. |
+| `packages/errors` | the error shape every refusal uses: the rule and the next action |
+| `packages/journal-sqlite` | the default `Journal` and `BlobStore` on SQLite |
+| `packages/segmenter-jieba` | the default `Segmenter`: jieba's dictionary segmentation for the word index, optional |
+| `packages/lexical` | the lexical search the journal and notes share: word segmentation, the FTS5 helpers and the merge order |
+| `packages/notes-sqlite` | the default `NoteStore` on SQLite, beside the journal |
+| `packages/judge-jev` | the default `Judge`: Jev, TypeSafe AI's System One classifier, answering the dispatch decisions |
+| `packages/workspace-dir` | the default `Workspace`: one directory per window, with a continuation working in its predecessor's directory in place, and the lookup of another window's directory it contributes to the catalog |
+| `packages/environment-dir` | the default `Environment`: one `/env` directory, a revision per item, validation and deltas |
+| `packages/supervisor-process` | the default `Supervisor`: one host process per service, restart with backoff, logs |
+| `packages/plugin-http` | the HTTP test channel: a webhook in, sends out, a page that drives the whole host |
+| `packages/prompt` | typed blocks, the one deterministic renderer and the replaceable text table |
+| `packages/agent-pi` | the default `AgentBackend`: pi-agent-core in process |
+| `packages/tools` | the entry points: direct tools, code mode modules, the host API |
+| `packages/core` | the service layer: one implementation of every operation - the window lifecycle, dispatch, coordination, the board, notes access, workspaces, effects, schedules and the snapshot epochs |
+
+`mise.toml` pins the toolchain - Node and pnpm - installed under
+`~/.local/share/mise`, and `.agents/setup` prepares a machine from it. A machine
+without mise is expected to carry Node 24 or newer and pnpm 12 already.
+
+## License
+
+MIT, see [LICENSE](LICENSE).

package/index.d.ts

@@ -0,0 +1,6 @@
+/**
+ * @4fu/arbor-lexical - the lexical search the journal and notes share.
+ */
+export { fallbackSegmenter, resolveSegmenter, indexText } from "./segmenter.ts";
+export { MIN_TRIGRAM_CHARS, escapeLike, likePattern, quoteFts, trigramMatch, wordMatch, mergeHits, } from "./search.ts";
+export type { ChannelHits, MergedHit, Ranked, SearchChannel } from "./search.ts";

package/index.js

@@ -0,0 +1,5 @@
+/**
+ * @4fu/arbor-lexical - the lexical search the journal and notes share.
+ */
+export { fallbackSegmenter, resolveSegmenter, indexText } from "./segmenter.js";
+export { MIN_TRIGRAM_CHARS, escapeLike, likePattern, quoteFts, trigramMatch, wordMatch, mergeHits, } from "./search.js";

package/package.json

@@ -1,6 +1,45 @@
 {
   "name": "@4fu/arbor-lexical",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "2026.1107.1",
+  "type": "module",
+  "description": "The lexical search the journal and notes share: word segmentation, the FTS5 helpers and the merge order.",
+  "exports": {
+    ".": {
+      "types": "./index.d.ts",
+      "default": "./index.js"
+    }
+  },
+  "types": "./index.d.ts",
+  "main": "./index.js",
+  "files": [
+    "index.d.ts",
+    "index.js",
+    "search.d.ts",
+    "search.js",
+    "segmenter.d.ts",
+    "segmenter.js"
+  ],
+  "engines": {
+    "node": ">=24"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/4fuu/arbor.git",
+    "directory": "packages/lexical"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@4fu/arbor-interfaces": "2026.1107.1"
+  },
+  "peerDependencies": {
+    "@4fu/arbor-segmenter-jieba": "2026.1107.1"
+  },
+  "peerDependenciesMeta": {
+    "@4fu/arbor-segmenter-jieba": {
+      "optional": true
+    }
+  }
+}

package/search.d.ts

@@ -0,0 +1,62 @@
+/**
+ * The pieces of lexical search every store shares (docs/journal.md#search).
+ *
+ * All text search is lexical and built for Chinese and mixed text: a word index
+ * by jieba segmentation with BM25, a trigram index for exact substrings, and a
+ * fallback for one- or two-character queries. Results are merged, exact
+ * substring hits first, and then by relevance.
+ *
+ * What lives here is what that model needs and no store should own alone: how a
+ * literal is quoted for an FTS5 `MATCH` and escaped for `LIKE`, how a query
+ * becomes one index's match, and how the channels are merged. Each store keeps
+ * its own schema, its own filters and its own read of the rows a match names.
+ */
+import type { MatchKind } from "@4fu/arbor-interfaces";
+/** A query shorter than this cannot use the trigram index. */
+export declare const MIN_TRIGRAM_CHARS = 3;
+/**
+ * Which channel a match came from, and the `match` a hit carries. The two
+ * lexical channels are `substring` and `word`; `semantic` is the one a store adds
+ * when a deployment gave it an embedding channel (docs/journal.md#search).
+ */
+export type SearchChannel = MatchKind;
+/** One match from one index, with its raw relevance. Lower `rank` is better. */
+export interface Ranked {
+    /** The row the index matched: an event's `seq`, a note's `id`. */
+    rowid: number;
+    rank: number;
+}
+/** One index's matches, named by the channel they are merged as. */
+export interface ChannelHits {
+    channel: SearchChannel;
+    hits: readonly Ranked[];
+}
+/** One merged match, with the channel it came from and its relevance in 0..1. */
+export interface MergedHit {
+    rowid: number;
+    channel: SearchChannel;
+    /** Relevance within the match's own channel; higher is better. */
+    score: number;
+}
+/** Escape a literal for `LIKE ... ESCAPE '\'`. */
+export declare function escapeLike(value: string): string;
+export declare function likePattern(query: string): string;
+/** Quote one term for an FTS5 `MATCH`, so outside text is never syntax. */
+export declare function quoteFts(value: string): string;
+/** The trigram index takes one phrase. */
+export declare function trigramMatch(query: string): string;
+/** The word index takes the query's segments, all of them required. */
+export declare function wordMatch(tokens: readonly string[]): string;
+/**
+ * Merge the channels the way lexical search does: the first channel's hits
+ * first, then the next channel's hits that are not already there. Each channel
+ * keeps the order it arrives in, which is its own relevance order - the query
+ * that fills a channel orders it - and a row two channels found stays where the
+ * first one put it. The channels are named at the call site in the order they
+ * merge, so a later one - a vector index, say - plugs in as one more argument
+ * and nothing here changes.
+ *
+ * A store with a semantic channel names it last, so the vector channel adds the
+ * hits no lexical index found and never moves one it did.
+ */
+export declare function mergeHits(...channels: readonly ChannelHits[]): MergedHit[];

package/search.js

@@ -0,0 +1,71 @@
+/**
+ * The pieces of lexical search every store shares (docs/journal.md#search).
+ *
+ * All text search is lexical and built for Chinese and mixed text: a word index
+ * by jieba segmentation with BM25, a trigram index for exact substrings, and a
+ * fallback for one- or two-character queries. Results are merged, exact
+ * substring hits first, and then by relevance.
+ *
+ * What lives here is what that model needs and no store should own alone: how a
+ * literal is quoted for an FTS5 `MATCH` and escaped for `LIKE`, how a query
+ * becomes one index's match, and how the channels are merged. Each store keeps
+ * its own schema, its own filters and its own read of the rows a match names.
+ */
+/** A query shorter than this cannot use the trigram index. */
+export const MIN_TRIGRAM_CHARS = 3;
+/** Escape a literal for `LIKE ... ESCAPE '\'`. */
+export function escapeLike(value) {
+    return value.replace(/[\\%_]/g, (character) => `\\${character}`);
+}
+export function likePattern(query) {
+    return `%${escapeLike(query)}%`;
+}
+/** Quote one term for an FTS5 `MATCH`, so outside text is never syntax. */
+export function quoteFts(value) {
+    return `"${value.replace(/"/g, '""')}"`;
+}
+/** The trigram index takes one phrase. */
+export function trigramMatch(query) {
+    return quoteFts(query);
+}
+/** The word index takes the query's segments, all of them required. */
+export function wordMatch(tokens) {
+    return tokens
+        .filter((token) => token.length > 0)
+        .map(quoteFts)
+        .join(" ");
+}
+/**
+ * Merge the channels the way lexical search does: the first channel's hits
+ * first, then the next channel's hits that are not already there. Each channel
+ * keeps the order it arrives in, which is its own relevance order - the query
+ * that fills a channel orders it - and a row two channels found stays where the
+ * first one put it. The channels are named at the call site in the order they
+ * merge, so a later one - a vector index, say - plugs in as one more argument
+ * and nothing here changes.
+ *
+ * A store with a semantic channel names it last, so the vector channel adds the
+ * hits no lexical index found and never moves one it did.
+ */
+export function mergeHits(...channels) {
+    const seen = new Set();
+    const merged = [];
+    for (const { channel, hits } of channels) {
+        if (hits.length === 0)
+            continue;
+        const best = Math.min(...hits.map((hit) => hit.rank));
+        for (const hit of hits) {
+            if (seen.has(hit.rowid))
+                continue;
+            seen.add(hit.rowid);
+            merged.push({ rowid: hit.rowid, channel, score: normalize(hit.rank, best) });
+        }
+    }
+    return merged;
+}
+/** Relevance in 0..1 within one channel; higher is better. */
+function normalize(rank, best) {
+    if (best === 0)
+        return 1;
+    return Math.max(0, Math.min(1, rank / best));
+}

package/segmenter.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Word segmentation for the word index (docs/journal.md#search).
+ *
+ * The spec calls for jieba segmentation with BM25, and the default is jieba
+ * when `@4fu/arbor-segmenter-jieba` is installed: `resolveSegmenter` loads it, and
+ * a store that keeps a word index uses what it returns when the caller names no
+ * segmenter. The package is an optional peer dependency, so a deployment that
+ * leaves it out keeps working.
+ *
+ * What is left here is the approximation that answers when jieba is not there:
+ * Chinese, Japanese and Korean runs are indexed per character and per bigram,
+ * and every other script per word. Character and bigram indexing is what makes
+ * a two-character Chinese query find the phrase it appears in, without a
+ * dictionary.
+ */
+import type { Segmenter } from "@4fu/arbor-interfaces";
+export declare const fallbackSegmenter: Segmenter;
+/**
+ * The segmenter a store uses when the caller names none: jieba when the
+ * package is installed, the per-character-and-bigram approximation otherwise.
+ * `load` replaces how the jieba package is loaded. A host that resolves modules
+ * its own way passes its own; so does a test that has no jieba package to load.
+ * A load that fails falls back rather than stopping the store, because the
+ * package is optional: it may be missing, or its prebuilt binary may not load
+ * on this platform.
+ */
+export declare function resolveSegmenter(load?: () => Segmenter | null): Segmenter;
+/** The text the word index stores: the segments, one per line. */
+export declare function indexText(text: string, segmenter: Segmenter): string;

package/segmenter.js

@@ -0,0 +1,106 @@
+/**
+ * Word segmentation for the word index (docs/journal.md#search).
+ *
+ * The spec calls for jieba segmentation with BM25, and the default is jieba
+ * when `@4fu/arbor-segmenter-jieba` is installed: `resolveSegmenter` loads it, and
+ * a store that keeps a word index uses what it returns when the caller names no
+ * segmenter. The package is an optional peer dependency, so a deployment that
+ * leaves it out keeps working.
+ *
+ * What is left here is the approximation that answers when jieba is not there:
+ * Chinese, Japanese and Korean runs are indexed per character and per bigram,
+ * and every other script per word. Character and bigram indexing is what makes
+ * a two-character Chinese query find the phrase it appears in, without a
+ * dictionary.
+ */
+import { createRequire } from "node:module";
+/** The package that holds the jieba segmenter, when a deployment installed it. */
+const JIEBA_PACKAGE = "@4fu/arbor-segmenter-jieba";
+/** CJK unified ideographs, extensions, compatibility, kana and hangul. */
+function isCjk(codePoint) {
+    return ((codePoint >= 0x3400 && codePoint <= 0x4dbf) ||
+        (codePoint >= 0x4e00 && codePoint <= 0x9fff) ||
+        (codePoint >= 0xf900 && codePoint <= 0xfaff) ||
+        (codePoint >= 0x3040 && codePoint <= 0x30ff) ||
+        (codePoint >= 0xac00 && codePoint <= 0xd7af) ||
+        (codePoint >= 0x20000 && codePoint <= 0x2a6df) ||
+        (codePoint >= 0x2a700 && codePoint <= 0x2ebef));
+}
+/** Whitespace and punctuation separate words; everything else is part of one. */
+const SEPARATOR = /[\s\p{P}\p{S}\p{C}\p{Z}]/u;
+export const fallbackSegmenter = {
+    segment(text) {
+        const chars = Array.from(text);
+        const tokens = [];
+        let index = 0;
+        while (index < chars.length) {
+            const first = chars[index];
+            const codePoint = first.codePointAt(0);
+            if (isCjk(codePoint)) {
+                const run = [];
+                while (index < chars.length) {
+                    const char = chars[index];
+                    const point = char.codePointAt(0);
+                    if (!isCjk(point))
+                        break;
+                    run.push(char);
+                    index += 1;
+                }
+                tokens.push(...run);
+                for (let offset = 0; offset + 1 < run.length; offset += 1) {
+                    tokens.push(run[offset] + run[offset + 1]);
+                }
+                continue;
+            }
+            if (SEPARATOR.test(first)) {
+                index += 1;
+                continue;
+            }
+            const run = [];
+            while (index < chars.length) {
+                const char = chars[index];
+                const point = char.codePointAt(0);
+                if (isCjk(point) || SEPARATOR.test(char))
+                    break;
+                run.push(char);
+                index += 1;
+            }
+            tokens.push(run.join("").toLowerCase());
+        }
+        return tokens;
+    },
+};
+/**
+ * Load the jieba segmenter, or `null` when it is not there: the package is not
+ * installed, or it holds no segmenter.
+ */
+function loadJieba() {
+    const require = createRequire(import.meta.url);
+    const loaded = require(JIEBA_PACKAGE);
+    const candidate = loaded?.jiebaSegmenter;
+    if (candidate === null || typeof candidate !== "object")
+        return null;
+    const shape = candidate;
+    return typeof shape.segment === "function" ? candidate : null;
+}
+/**
+ * The segmenter a store uses when the caller names none: jieba when the
+ * package is installed, the per-character-and-bigram approximation otherwise.
+ * `load` replaces how the jieba package is loaded. A host that resolves modules
+ * its own way passes its own; so does a test that has no jieba package to load.
+ * A load that fails falls back rather than stopping the store, because the
+ * package is optional: it may be missing, or its prebuilt binary may not load
+ * on this platform.
+ */
+export function resolveSegmenter(load = loadJieba) {
+    try {
+        return load() ?? fallbackSegmenter;
+    }
+    catch {
+        return fallbackSegmenter;
+    }
+}
+/** The text the word index stores: the segments, one per line. */
+export function indexText(text, segmenter) {
+    return segmenter.segment(text).join("\n");
+}