dsh-plugin-dbgraph-service 0.0.0-stage → 0.1.0

package/lib/schema.d.ts

@@ -0,0 +1,150 @@
+/**
+ * The on-disk index format: where the files live, the schema version, the DDL, and the sidecar the
+ * system prompt reads synchronously.
+ *
+ * Three files, all under the global index root (`~/.dsh/dbgraph`):
+ *
+ * - `connections.json` — the connection list. Structure and host facts only; every secret lives in
+ *   the host's credential seam, referenced by name.
+ * - `<connection>/index.db` — one SQLite file per connection, replaced wholesale on every run. That
+ *   single decision removes a whole class of bugs: no incremental diffing, no deletes, no
+ *   half-updated index. A run writes `<connection>/index.db.<pid>-<nonce>.tmp` and renames it into
+ *   place, so two runs never share a staging file.
+ * - `status.json` — a digest of every connection's index state plus its object list, small enough to
+ *   read synchronously on every prompt assembly. It exists so the model learns the schema without
+ *   spending a tool call, and so nothing opens SQLite during assembly.
+ *
+ * The index holds structure and never a business value. The one data-derived field anywhere is
+ * `columns.enum_values_json`, a bounded sample of distinct literals (see `DEFAULT_ENUM_SAMPLE_LIMIT`).
+ * @module dsh-plugin-dbgraph-service/schema
+ */
+/** Directory name under the harness home that holds every dbgraph file. */
+export declare const INDEX_ROOT_DIRNAME = "dbgraph";
+/** File name of one connection's index, inside its own directory. */
+export declare const INDEX_FILE_NAME = "index.db";
+/** The connection list, inside the index root. */
+export declare const CONNECTIONS_FILE_NAME = "connections.json";
+/** The prompt digest sidecar, inside the index root. */
+export declare const STATUS_FILE_NAME = "status.json";
+/** Schema version stored in `meta.schema_version`. A store refuses a file it does not know. */
+export declare const INDEX_SCHEMA_VERSION = 1;
+/** Default number of distinct literals sampled per column for `columns.enum_values_json`. */
+export declare const DEFAULT_ENUM_SAMPLE_LIMIT = 10;
+/**
+ * Default object count carried in `status.json` for the prompt digest. A connection with more
+ * objects than this lists the largest ones and says how many were left out.
+ */
+export declare const DEFAULT_STATUS_DIGEST_OBJECTS = 120;
+/** Default hours after which an L2 snapshot is called stale in the prompt. */
+export declare const DEFAULT_INDEX_STALE_HOURS = 6;
+/**
+ * The statements that create a fresh index. Applied in order inside one transaction; the writer
+ * executes them against a `.tmp` file that only becomes the index after the rename.
+ *
+ * `search` keeps its rows (a contentful FTS5 table) so the identity columns read back; the inline
+ * comment beside the DDL below records why a contentless table was rejected rather than kept.
+ */
+export declare const INDEX_SCHEMA_DDL: readonly string[];
+/**
+ * The `meta` keys every writer writes and every reader looks up. One list, so the JavaScript and the
+ * native writer cannot drift: the reader treats an absent key as unknown rather than as a default.
+ */
+export declare const INDEX_META_KEYS: {
+    /** {@link INDEX_SCHEMA_VERSION} the file was written for. */
+    readonly schemaVersion: "schema_version";
+    /** Connection id the file describes. */
+    readonly connection: "connection";
+    /** Database family. */
+    readonly engine: "engine";
+    /** Database name, or file path. */
+    readonly database: "database";
+    /** Server version string, when the server reported one. */
+    readonly serverVersion: "server_version";
+    /** `l1` or `l2` — which level the snapshot covers. */
+    readonly level: "level";
+    /** Epoch milliseconds when the snapshot was taken off the server. */
+    readonly fetchedAt: "fetched_at";
+    /** Epoch milliseconds of the last structural snapshot. */
+    readonly l1IndexedAt: "l1_indexed_at";
+    /** Epoch milliseconds of the last enrichment pass; absent on an L1-only file. */
+    readonly l2IndexedAt: "l2_indexed_at";
+    /** `js` or `rust` — which writer produced this file. */
+    readonly writer: "writer";
+    /** Epoch milliseconds when the file was written. */
+    readonly writtenAt: "written_at";
+};
+/** One entry of the prompt digest: everything needed to name a table without a query. */
+export interface StatusDigestEntry {
+    /** Object name, schema-qualified when the engine has a schema. */
+    readonly object: string;
+    /** Object comment, when indexed. */
+    readonly comment?: string;
+    /** Column count. */
+    readonly columns: number;
+}
+/** One connection's state as the prompt sees it. */
+export interface StatusConnectionEntry {
+    /** Database family, for the sentence's parenthetical. */
+    readonly engine: string;
+    /** Human label, when configured. */
+    readonly label?: string;
+    /** Epoch milliseconds of the last structural snapshot. */
+    readonly l1IndexedAt?: number;
+    /** Epoch milliseconds of the last enrichment pass. */
+    readonly l2IndexedAt?: number;
+    /** Objects in the index. */
+    readonly objects: number;
+    /** Columns in the index. */
+    readonly columns: number;
+    /** Fraction of columns carrying a comment, 0..1. */
+    readonly commentCoverage: number;
+    /** Largest objects, capped at `DEFAULT_STATUS_DIGEST_OBJECTS`. */
+    readonly digest: readonly StatusDigestEntry[];
+    /** Objects left out of {@link digest}. */
+    readonly digestOmitted: number;
+}
+/**
+ * The sidecar `status.json` holds. Written after the index rename, so a reader never describes an
+ * index that is not there yet.
+ */
+export interface StatusFile {
+    /** Format version. A reader that sees a newer one falls back to the weaker sentence. */
+    readonly version: number;
+    /** Epoch milliseconds of the last write. */
+    readonly updatedAt: number;
+    /** Per-connection entries, keyed by connection id. */
+    readonly connections: Readonly<Record<string, StatusConnectionEntry>>;
+}
+/** One persisted connection, as `connections.json` stores it. Never carries a secret. */
+export interface StoredConnection {
+    /** Connection identity. */
+    readonly id: string;
+    /** Database family. */
+    readonly engine: string;
+    /** Human label. */
+    readonly label?: string;
+    /** Host. */
+    readonly host?: string;
+    /** Port. */
+    readonly port?: number;
+    /** Database name, file path, or Redis database index. */
+    readonly database?: string;
+    /** Login user. */
+    readonly user?: string;
+    /** Credential-seam reference holding the password. */
+    readonly credentialRef?: string;
+    /** Sensitive-column handling for this connection. */
+    readonly sensitiveColumns?: string;
+    /** Extra sensitive column patterns. */
+    readonly sensitivePatterns?: readonly string[];
+    /** Epoch milliseconds when the connection was added. */
+    readonly addedAt?: number;
+}
+/** The `connections.json` document. */
+export interface ConnectionsFile {
+    /** Format version. */
+    readonly version: number;
+    /** Configured connections, in the order the user arranged them. */
+    readonly connections: readonly StoredConnection[];
+}
+//# sourceMappingURL=schema.d.ts.map

package/lib/schema.js

@@ -0,0 +1,152 @@
+/**
+ * The on-disk index format: where the files live, the schema version, the DDL, and the sidecar the
+ * system prompt reads synchronously.
+ *
+ * Three files, all under the global index root (`~/.dsh/dbgraph`):
+ *
+ * - `connections.json` — the connection list. Structure and host facts only; every secret lives in
+ *   the host's credential seam, referenced by name.
+ * - `<connection>/index.db` — one SQLite file per connection, replaced wholesale on every run. That
+ *   single decision removes a whole class of bugs: no incremental diffing, no deletes, no
+ *   half-updated index. A run writes `<connection>/index.db.<pid>-<nonce>.tmp` and renames it into
+ *   place, so two runs never share a staging file.
+ * - `status.json` — a digest of every connection's index state plus its object list, small enough to
+ *   read synchronously on every prompt assembly. It exists so the model learns the schema without
+ *   spending a tool call, and so nothing opens SQLite during assembly.
+ *
+ * The index holds structure and never a business value. The one data-derived field anywhere is
+ * `columns.enum_values_json`, a bounded sample of distinct literals (see `DEFAULT_ENUM_SAMPLE_LIMIT`).
+ * @module dsh-plugin-dbgraph-service/schema
+ */
+/** Directory name under the harness home that holds every dbgraph file. */
+export const INDEX_ROOT_DIRNAME = 'dbgraph';
+/** File name of one connection's index, inside its own directory. */
+export const INDEX_FILE_NAME = 'index.db';
+/** The connection list, inside the index root. */
+export const CONNECTIONS_FILE_NAME = 'connections.json';
+/** The prompt digest sidecar, inside the index root. */
+export const STATUS_FILE_NAME = 'status.json';
+/** Schema version stored in `meta.schema_version`. A store refuses a file it does not know. */
+export const INDEX_SCHEMA_VERSION = 1;
+/** Default number of distinct literals sampled per column for `columns.enum_values_json`. */
+export const DEFAULT_ENUM_SAMPLE_LIMIT = 10;
+/**
+ * Default object count carried in `status.json` for the prompt digest. A connection with more
+ * objects than this lists the largest ones and says how many were left out.
+ */
+export const DEFAULT_STATUS_DIGEST_OBJECTS = 120;
+/** Default hours after which an L2 snapshot is called stale in the prompt. */
+export const DEFAULT_INDEX_STALE_HOURS = 6;
+/**
+ * The statements that create a fresh index. Applied in order inside one transaction; the writer
+ * executes them against a `.tmp` file that only becomes the index after the rename.
+ *
+ * `search` keeps its rows (a contentful FTS5 table) so the identity columns read back; the inline
+ * comment beside the DDL below records why a contentless table was rejected rather than kept.
+ */
+export const INDEX_SCHEMA_DDL = [
+    `CREATE TABLE IF NOT EXISTS meta (
+     key   TEXT PRIMARY KEY,
+     value TEXT NOT NULL
+   )`,
+    `CREATE TABLE IF NOT EXISTS objects (
+     id           INTEGER PRIMARY KEY,
+     kind         TEXT    NOT NULL,
+     schema_name  TEXT    NOT NULL DEFAULT '',
+     name         TEXT    NOT NULL,
+     comment      TEXT,
+     row_estimate INTEGER,
+     updated_at   INTEGER NOT NULL
+   )`,
+    `CREATE UNIQUE INDEX IF NOT EXISTS objects_identity ON objects (schema_name, name)`,
+    `CREATE TABLE IF NOT EXISTS columns (
+     id               INTEGER PRIMARY KEY,
+     object_id        INTEGER NOT NULL REFERENCES objects (id) ON DELETE CASCADE,
+     ordinal          INTEGER NOT NULL,
+     name             TEXT    NOT NULL,
+     type             TEXT    NOT NULL,
+     type_class       TEXT    NOT NULL,
+     nullable         INTEGER NOT NULL DEFAULT 1,
+     default_value    TEXT,
+     comment          TEXT,
+     is_primary_key   INTEGER NOT NULL DEFAULT 0,
+     is_unique        INTEGER NOT NULL DEFAULT 0,
+     enum_values_json TEXT,
+     updated_at       INTEGER NOT NULL
+   )`,
+    `CREATE UNIQUE INDEX IF NOT EXISTS columns_identity ON columns (object_id, name)`,
+    `CREATE TABLE IF NOT EXISTS edges (
+     id             INTEGER PRIMARY KEY,
+     src_object     INTEGER NOT NULL REFERENCES objects (id) ON DELETE CASCADE,
+     src_column_id  INTEGER REFERENCES columns (id) ON DELETE CASCADE,
+     kind           TEXT    NOT NULL,
+     dst_object     INTEGER REFERENCES objects (id) ON DELETE CASCADE,
+     dst_column_id  INTEGER REFERENCES columns (id) ON DELETE CASCADE,
+     name           TEXT
+   )`,
+    `CREATE INDEX IF NOT EXISTS edges_src ON edges (src_object)`,
+    `CREATE INDEX IF NOT EXISTS edges_dst ON edges (dst_object)`,
+    `CREATE TABLE IF NOT EXISTS terms (
+     term      TEXT    NOT NULL,
+     weight    REAL    NOT NULL,
+     object_id INTEGER REFERENCES objects (id) ON DELETE CASCADE,
+     column_id INTEGER REFERENCES columns (id) ON DELETE CASCADE
+   )`,
+    `CREATE INDEX IF NOT EXISTS terms_term ON terms (term)`,
+    // FTS5, contentful, with the two identity columns declared UNINDEXED.
+    //
+    // Two facts were measured on node v22.21.0 / SQLite 3.50.4 rather than assumed, and they pick this
+    // shape over the alternatives:
+    //
+    // 1. A **contentless** table (`content = ''`) answers with rowid and rank only — every declared
+    //    column, UNINDEXED ones included, reads back as NULL. Storing `object_id`/`column_id` there is
+    //    useless, so a contentless design has to smuggle identity through rowid.
+    // 2. A **contentful** table with `object_id UNINDEXED` returns those columns verbatim
+    //    (`{rowid: 7, object_id: 3, column_id: 42}`). UNINDEXED means "stored but not tokenized", not
+    //    "stored but unreadable".
+    //
+    // So: one table, readable ids, no rowid arithmetic. The size a contentless table would save does not
+    // pay for that.
+    //
+    // Note the FTS5 column grammar while adding a column: `column-name [UNINDEXED]`. A type name in
+    // front of UNINDEXED is a parse error (`parse error in "object_id INTEGER UNINDEXED"`).
+    //
+    // `text` is **not** the raw name and comment — it is the token stream {@link tokenizeForSearch}
+    // produces, because unicode61 treats a run of CJK as one token: store `广告点击` verbatim and a
+    // search for `广告` matches nothing. Verified.
+    `CREATE VIRTUAL TABLE IF NOT EXISTS search USING fts5 (
+     text,
+     object_id UNINDEXED,
+     column_id UNINDEXED,
+     tokenize = 'unicode61'
+   )`,
+];
+/**
+ * The `meta` keys every writer writes and every reader looks up. One list, so the JavaScript and the
+ * native writer cannot drift: the reader treats an absent key as unknown rather than as a default.
+ */
+export const INDEX_META_KEYS = {
+    /** {@link INDEX_SCHEMA_VERSION} the file was written for. */
+    schemaVersion: 'schema_version',
+    /** Connection id the file describes. */
+    connection: 'connection',
+    /** Database family. */
+    engine: 'engine',
+    /** Database name, or file path. */
+    database: 'database',
+    /** Server version string, when the server reported one. */
+    serverVersion: 'server_version',
+    /** `l1` or `l2` — which level the snapshot covers. */
+    level: 'level',
+    /** Epoch milliseconds when the snapshot was taken off the server. */
+    fetchedAt: 'fetched_at',
+    /** Epoch milliseconds of the last structural snapshot. */
+    l1IndexedAt: 'l1_indexed_at',
+    /** Epoch milliseconds of the last enrichment pass; absent on an L1-only file. */
+    l2IndexedAt: 'l2_indexed_at',
+    /** `js` or `rust` — which writer produced this file. */
+    writer: 'writer',
+    /** Epoch milliseconds when the file was written. */
+    writtenAt: 'written_at',
+};
+//# sourceMappingURL=schema.js.map

package/lib/tokenize.d.ts

@@ -0,0 +1,45 @@
+/**
+ * The one tokenizer both index writers and the resolver must agree on.
+ *
+ * FTS5's `unicode61` tokenizer splits on non-alphanumerics, which means a run of Chinese becomes a
+ * **single** token: store `广告点击` verbatim and `MATCH '广告'` returns nothing (verified on SQLite
+ * 3.50.4). So the indexed text is not the raw name and comment — it is the token stream this module
+ * produces, and the query side is the same function's output joined with `OR`.
+ *
+ * Two rules, deliberately few:
+ *
+ * - **ASCII identifiers** split on every change of character class, so `createdAt` → `created`, `at`
+ *   and `password_hash` → `password`, `hash`; everything is lowercased.
+ * - **CJK runs** split into overlapping bigrams (`广告点击` → `广告`, `告点`, `点击`). A run of one
+ *   character is its own token.
+ *
+ * The native writer reimplements these two rules in Rust. `packages/store/tests/tokenize-parity.spec.ts`
+ * is what keeps the two implementations honest: it asserts token-for-token equality on a fixture
+ * corpus, so a change here that is not mirrored there fails the build rather than silently degrading
+ * recall.
+ * @module dsh-plugin-dbgraph-service/tokenize
+ */
+/**
+ * The token stream indexed for one text field, and the vocabulary a query is built from.
+ * @param input - a name, a comment, or a user's words.
+ * @returns deduplicated tokens in first-seen order; empty when the input holds no usable token.
+ */
+export declare function tokenizeForSearch(input: string): string[];
+/**
+ * Build the `MATCH` expression for a set of user words.
+ *
+ * Each token is quoted, because a bare token that happens to be an FTS5 keyword (`or`, `not`, `near`)
+ * is a syntax error. Tokens join with `OR`, not a space: in FTS5 a space means "same phrase, in
+ * order", which is far stricter than "any of these words".
+ * @param words - the user's words, already tokenized or not.
+ * @param maxTokens - ceiling on terms in one expression (default 32).
+ * @returns an FTS5 `MATCH` expression, or an empty string when the words hold no usable token.
+ */
+export declare function searchMatchQuery(words: readonly string[], maxTokens?: number): string;
+/**
+ * The indexable text for one object or column: its name and its comment, tokenized and joined.
+ * @param parts - the fields to fold in, in the order they should contribute tokens.
+ * @returns a space-joined token stream, ready for the `search.text` column.
+ */
+export declare function searchDocument(parts: readonly (string | undefined)[]): string;
+//# sourceMappingURL=tokenize.d.ts.map

package/lib/tokenize.js

@@ -0,0 +1,205 @@
+/**
+ * The one tokenizer both index writers and the resolver must agree on.
+ *
+ * FTS5's `unicode61` tokenizer splits on non-alphanumerics, which means a run of Chinese becomes a
+ * **single** token: store `广告点击` verbatim and `MATCH '广告'` returns nothing (verified on SQLite
+ * 3.50.4). So the indexed text is not the raw name and comment — it is the token stream this module
+ * produces, and the query side is the same function's output joined with `OR`.
+ *
+ * Two rules, deliberately few:
+ *
+ * - **ASCII identifiers** split on every change of character class, so `createdAt` → `created`, `at`
+ *   and `password_hash` → `password`, `hash`; everything is lowercased.
+ * - **CJK runs** split into overlapping bigrams (`广告点击` → `广告`, `告点`, `点击`). A run of one
+ *   character is its own token.
+ *
+ * The native writer reimplements these two rules in Rust. `packages/store/tests/tokenize-parity.spec.ts`
+ * is what keeps the two implementations honest: it asserts token-for-token equality on a fixture
+ * corpus, so a change here that is not mirrored there fails the build rather than silently degrading
+ * recall.
+ * @module dsh-plugin-dbgraph-service/tokenize
+ */
+/** CJK ideographs and kana — the scripts whose tokens need bigram expansion. */
+const CJK = /[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]/;
+/**
+ * Classify one character.
+ * @param ch - the character.
+ * @returns its class.
+ */
+function classOf(ch) {
+    if (CJK.test(ch))
+        return 'cjk';
+    if (ch >= 'a' && ch <= 'z')
+        return 'lower';
+    if (ch >= 'A' && ch <= 'Z')
+        return 'upper';
+    if (ch >= '0' && ch <= '9')
+        return 'digit';
+    return 'other';
+}
+/**
+ * Split one identifier-ish run into its ASCII parts, at camel-case and letter/digit boundaries.
+ *
+ * The boundary needs one character of lookahead, which is why this walks indices instead of folding
+ * over characters: `createdAt` splits before `A` (a lower-case letter precedes an upper-case one),
+ * while `HTMLParser` splits before `P` (an upper-case run is followed by a lower-case letter). A
+ * class-change-per-character rule gets the first one wrong, emitting `created` `a` `t`.
+ * @param word - the run to split.
+ * @returns lowercase parts, in order.
+ */
+function splitIdentifier(word) {
+    const chars = [...word];
+    const parts = [];
+    let current = '';
+    for (let index = 0; index < chars.length; index += 1) {
+        const ch = chars[index];
+        const previous = index === 0 ? '' : chars[index - 1];
+        const next = index + 1 < chars.length ? chars[index + 1] : '';
+        if (current !== '' && isBoundary(previous, ch, next)) {
+            parts.push(current);
+            current = '';
+        }
+        current += ch;
+    }
+    if (current !== '')
+        parts.push(current);
+    return parts.map(part => part.toLowerCase());
+}
+/**
+ * Whether a new part starts at `ch`.
+ * @param previous - the character before `ch`, or `''` at the start of the run.
+ * @param ch - the character being placed.
+ * @param next - the character after `ch`, or `''` at the end of the run.
+ * @returns true when `ch` begins a new part.
+ */
+function isBoundary(previous, ch, next) {
+    const chClass = classOf(ch);
+    const previousClass = classOf(previous === '' ? ' ' : previous);
+    if (chClass !== previousClass) {
+        if (chClass === 'digit')
+            return true;
+        if (chClass === 'upper')
+            return previousClass === 'lower' || previousClass === 'digit';
+        // A lower-case letter after an upper-case run does not start a part by itself — the part already
+        // started at the upper-case letter that precedes it (see the acronym rule below).
+        if (chClass === 'lower' && previousClass === 'upper')
+            return false;
+        return true;
+    }
+    // Acronym boundary: inside a run of capitals, the last capital before a lower-case letter starts a
+    // new part, so `HTMLParser` → `HTML` + `Parser` rather than one run.
+    if (chClass === 'upper' && next !== '' && classOf(next) === 'lower')
+        return true;
+    return false;
+}
+/**
+ * Expand a CJK run into overlapping bigrams. A single-character run is its own token.
+ * @param run - the run, one or more CJK characters.
+ * @returns bigrams, in order.
+ */
+function cjkBigrams(run) {
+    const chars = [...run];
+    if (chars.length === 1)
+        return chars;
+    const out = [];
+    for (let index = 0; index + 1 < chars.length; index += 1) {
+        out.push(`${chars[index]}${chars[index + 1]}`);
+    }
+    return out;
+}
+/**
+ * The token stream indexed for one text field, and the vocabulary a query is built from.
+ * @param input - a name, a comment, or a user's words.
+ * @returns deduplicated tokens in first-seen order; empty when the input holds no usable token.
+ */
+export function tokenizeForSearch(input) {
+    const seen = new Set();
+    const tokens = [];
+    const push = (token) => {
+        if (token === '' || seen.has(token))
+            return;
+        seen.add(token);
+        tokens.push(token);
+    };
+    const normalized = input.normalize('NFKC');
+    let ascii = '';
+    let cjk = '';
+    const flushAscii = () => {
+        if (ascii === '')
+            return;
+        for (const part of splitIdentifier(ascii))
+            push(part);
+        ascii = '';
+    };
+    const flushCjk = () => {
+        if (cjk === '')
+            return;
+        for (const bigram of cjkBigrams(cjk))
+            push(bigram);
+        cjk = '';
+    };
+    for (const ch of normalized) {
+        const cls = classOf(ch);
+        if (cls === 'cjk') {
+            flushAscii();
+            cjk += ch;
+            continue;
+        }
+        flushCjk();
+        if (cls === 'other') {
+            flushAscii();
+            continue;
+        }
+        ascii += ch;
+    }
+    flushAscii();
+    flushCjk();
+    return tokens;
+}
+/**
+ * Build the `MATCH` expression for a set of user words.
+ *
+ * Each token is quoted, because a bare token that happens to be an FTS5 keyword (`or`, `not`, `near`)
+ * is a syntax error. Tokens join with `OR`, not a space: in FTS5 a space means "same phrase, in
+ * order", which is far stricter than "any of these words".
+ * @param words - the user's words, already tokenized or not.
+ * @param maxTokens - ceiling on terms in one expression (default 32).
+ * @returns an FTS5 `MATCH` expression, or an empty string when the words hold no usable token.
+ */
+export function searchMatchQuery(words, maxTokens = 32) {
+    const tokens = [];
+    const seen = new Set();
+    for (const word of words) {
+        for (const token of tokenizeForSearch(word)) {
+            if (seen.has(token))
+                continue;
+            seen.add(token);
+            tokens.push(token);
+            if (tokens.length >= maxTokens) {
+                return tokens.map(token => `"${token.replace(/"/g, '""')}"`).join(' OR ');
+            }
+        }
+    }
+    return tokens.map(token => `"${token.replace(/"/g, '""')}"`).join(' OR ');
+}
+/**
+ * The indexable text for one object or column: its name and its comment, tokenized and joined.
+ * @param parts - the fields to fold in, in the order they should contribute tokens.
+ * @returns a space-joined token stream, ready for the `search.text` column.
+ */
+export function searchDocument(parts) {
+    const tokens = [];
+    const seen = new Set();
+    for (const part of parts) {
+        if (part === undefined || part === '')
+            continue;
+        for (const token of tokenizeForSearch(part)) {
+            if (seen.has(token))
+                continue;
+            seen.add(token);
+            tokens.push(token);
+        }
+    }
+    return tokens.join(' ');
+}
+//# sourceMappingURL=tokenize.js.map