kibi-plugin-search-hints 0.0.0-stage → 0.1.0

package/dist/cache.d.ts

@@ -0,0 +1,25 @@
+import type { SearchHintsEntity } from "kibi-plugin-sdk";
+/**
+ * Entity vectors keyed by type::id and the host content hash. A changed
+ * entity, a different model or a damaged file means that vector (or the
+ * whole file) is rebuilt; nothing stale is ever returned.
+ */
+export declare class VectorCache {
+    private readonly file;
+    private readonly model;
+    private readonly dims;
+    private readonly vectors;
+    private dirty;
+    private constructor();
+    static open(directory: string, model: string, dims: number): VectorCache;
+    private read;
+    /** The vector of each entity, computing only the ones not cached. */
+    vectorsFor(entities: readonly SearchHintsEntity[], encode: (text: string) => Float32Array): Float32Array[];
+    /** Keep only the given entities, so deleted ones do not grow the file. */
+    retain(entities: readonly SearchHintsEntity[]): void;
+    /** Cached entity keys in file order; for inspection and tests. */
+    keys(): string[];
+    /** Write atomically: a reader sees the old file or the new one. */
+    save(): void;
+}
+//# sourceMappingURL=cache.d.ts.map

package/dist/cache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../src/cache.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAezD;;;;GAIG;AAEH,qBAAa,WAAW;IAKpB,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,IAAI;IANvB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAC3D,OAAO,CAAC,KAAK,CAAS;IAEtB,OAAO;IAMP,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,WAAW;IAMxE,OAAO,CAAC,IAAI;IA6BZ,qEAAqE;IACrE,UAAU,CACR,QAAQ,EAAE,SAAS,iBAAiB,EAAE,EACtC,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,YAAY,GACrC,YAAY,EAAE;IAajB,0EAA0E;IAC1E,MAAM,CAAC,QAAQ,EAAE,SAAS,iBAAiB,EAAE,GAAG,IAAI;IASpD,kEAAkE;IAClE,IAAI,IAAI,MAAM,EAAE;IAIhB,mEAAmE;IACnE,IAAI,IAAI,IAAI;CAgCb"}

package/dist/cache.js

@@ -0,0 +1,112 @@
+import { readFileSync, renameSync, writeFileSync } from "node:fs";
+import path from "node:path";
+const MAGIC = "KSHC";
+const FORMAT_VERSION = 1;
+const FILE_NAME = "vectors.bin";
+/**
+ * Entity vectors keyed by type::id and the host content hash. A changed
+ * entity, a different model or a damaged file means that vector (or the
+ * whole file) is rebuilt; nothing stale is ever returned.
+ */
+// implements REQ-kibi-plugin-search-hints
+export class VectorCache {
+    file;
+    model;
+    dims;
+    vectors = new Map();
+    dirty = false;
+    constructor(file, model, dims) {
+        this.file = file;
+        this.model = model;
+        this.dims = dims;
+    }
+    static open(directory, model, dims) {
+        const cache = new VectorCache(path.join(directory, FILE_NAME), model, dims);
+        cache.read();
+        return cache;
+    }
+    read() {
+        // A missing, foreign or damaged file leaves the cache empty: every
+        // vector is then rebuilt, and nothing stale is ever returned.
+        try {
+            const bytes = readFileSync(this.file);
+            if (bytes.toString("ascii", 0, 4) !== MAGIC ||
+                bytes.readUInt32LE(4) !== FORMAT_VERSION)
+                return;
+            const headerLength = bytes.readUInt32LE(8);
+            const header = JSON.parse(bytes.toString("utf8", 12, 12 + headerLength));
+            // A file of other dimensions fails the length check below.
+            if (header.model !== this.model)
+                return;
+            const start = 12 + headerLength;
+            const width = this.dims * 4;
+            if (bytes.length !== start + header.entries.length * width)
+                return;
+            header.entries.forEach(([key, hash], index) => {
+                const offset = start + index * width;
+                const vector = Float32Array.from({ length: this.dims }, (_, col) => bytes.readFloatLE(offset + col * 4));
+                this.vectors.set(key, { hash, vector });
+            });
+        }
+        catch { }
+    }
+    /** The vector of each entity, computing only the ones not cached. */
+    vectorsFor(entities, encode) {
+        return entities.map((entity) => {
+            const key = `${entity.type}::${entity.id}`;
+            const cached = this.vectors.get(key);
+            if (cached !== undefined && cached.hash === entity.hash)
+                return cached.vector;
+            const vector = encode(entity.text);
+            this.vectors.set(key, { hash: entity.hash, vector });
+            this.dirty = true;
+            return vector;
+        });
+    }
+    /** Keep only the given entities, so deleted ones do not grow the file. */
+    retain(entities) {
+        const keep = new Set(entities.map(cacheKey));
+        for (const key of this.vectors.keys())
+            if (!keep.has(key)) {
+                this.vectors.delete(key);
+                this.dirty = true;
+            }
+    }
+    /** Cached entity keys in file order; for inspection and tests. */
+    keys() {
+        return [...this.vectors.keys()].sort();
+    }
+    /** Write atomically: a reader sees the old file or the new one. */
+    save() {
+        if (!this.dirty)
+            return;
+        const entries = this.keys().map((key) => [key, this.vectors.get(key)]);
+        const header = Buffer.from(JSON.stringify({
+            model: this.model,
+            dims: this.dims,
+            entries: entries.map(([key, { hash }]) => [key, hash]),
+        }), "utf8");
+        const width = this.dims * 4;
+        const out = Buffer.alloc(12 + header.length + entries.length * width);
+        out.write(MAGIC, 0);
+        out.writeUInt32LE(FORMAT_VERSION, 4);
+        out.writeUInt32LE(header.length, 8);
+        header.copy(out, 12);
+        entries.forEach(([, { vector }], index) => {
+            const offset = 12 + header.length + index * width;
+            vector.forEach((value, col) => out.writeFloatLE(value, offset + col * 4));
+        });
+        const temporary = `${this.file}.${process.pid}.tmp`;
+        try {
+            writeFileSync(temporary, out, { mode: 0o600 });
+            renameSync(temporary, this.file);
+            this.dirty = false;
+        }
+        catch {
+            // An unwritable cache only costs a rebuild next time.
+        }
+    }
+}
+function cacheKey(entity) {
+    return `${entity.type}::${entity.id}`;
+}

package/dist/index.d.ts

@@ -0,0 +1,33 @@
+import { type SearchHint, type SearchHintsInput, type SearchHintsV1 } from "kibi-plugin-sdk";
+import { VectorCache } from "./cache.js";
+import { type StaticModel } from "./model.js";
+export { VectorCache } from "./cache.js";
+export { MODEL_ID, MODEL_LABEL, MODEL_REVISION, loadStaticModel, } from "./model.js";
+export type { StaticModel } from "./model.js";
+export { WordPieceTokenizer, normalize, preTokenize } from "./tokenizer.js";
+/**
+ * Rank the supplied entities by cosine similarity to the query. Scores are
+ * cosine clamped to [0, 1]; ties are ordered by id, so the same input always
+ * gives the same hints.
+ */
+export declare function rankByMeaning(staticModel: StaticModel, cache: VectorCache, input: SearchHintsInput): SearchHint[];
+export declare function createSearchHints(): SearchHintsV1;
+/**
+ * Offline meaning-based hints for weak kb_search answers, from the bundled
+ * potion-base-8M static embedding. No network, no native code.
+ */
+export declare const kibiPlugin: {
+    apiVersion: "kibi.plugin.v1";
+    id: string;
+    version: string;
+    permissions: {
+        network: false;
+        metered: false;
+        secrets: never[];
+    };
+    capabilities: {
+        searchHints: SearchHintsV1;
+    };
+};
+export default kibiPlugin;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAEL,KAAK,UAAU,EACf,KAAK,gBAAgB,EAErB,KAAK,aAAa,EAEnB,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAe,KAAK,WAAW,EAAmB,MAAM,YAAY,CAAC;AAE5E,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EACL,QAAQ,EACR,WAAW,EACX,cAAc,EACd,eAAe,GAChB,MAAM,YAAY,CAAC;AACpB,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAkB5E;;;;GAIG;AAEH,wBAAgB,aAAa,CAC3B,WAAW,EAAE,WAAW,EACxB,KAAK,EAAE,WAAW,EAClB,KAAK,EAAE,gBAAgB,GACtB,UAAU,EAAE,CA0Bd;AAGD,wBAAgB,iBAAiB,IAAI,aAAa,CAoBjD;AAED;;;GAGG;AAEH,eAAO,MAAM,UAAU;;;;;;;;;;;;CAMrB,CAAC;AAEH,eAAe,UAAU,CAAC"}

package/dist/index.js

@@ -0,0 +1,71 @@
+import { createRequire } from "node:module";
+import { KIBI_PLUGIN_API_VERSION, defineKibiPlugin, } from "kibi-plugin-sdk";
+import { VectorCache } from "./cache.js";
+import { MODEL_LABEL, loadStaticModel } from "./model.js";
+export { VectorCache } from "./cache.js";
+export { MODEL_ID, MODEL_LABEL, MODEL_REVISION, loadStaticModel, } from "./model.js";
+export { WordPieceTokenizer, normalize, preTokenize } from "./tokenizer.js";
+const packageJson = createRequire(import.meta.url)("../package.json");
+let model;
+const caches = new Map();
+/** kibi.search-hints.v1 caps a request at 20 hints. */
+const MAX_LIMIT = 20;
+function dot(left, right) {
+    return left.reduce((sum, value, index) => sum + value * right[index], 0);
+}
+/**
+ * Rank the supplied entities by cosine similarity to the query. Scores are
+ * cosine clamped to [0, 1]; ties are ordered by id, so the same input always
+ * gives the same hints.
+ */
+// implements REQ-kibi-plugin-search-hints
+export function rankByMeaning(staticModel, cache, input) {
+    const vectors = cache.vectorsFor(input.entities, (text) => staticModel.encode(text));
+    cache.retain(input.entities);
+    cache.save();
+    const query = staticModel.encode(input.query);
+    const limit = Math.max(0, Math.min(input.limit, MAX_LIMIT));
+    return input.entities
+        .map((entity, index) => ({
+        id: entity.id,
+        score: Math.min(1, Math.max(0, dot(query, vectors[index]))),
+    }))
+        .filter((hint) => hint.score > 0)
+        .sort((left, right) => right.score - left.score ||
+        // rationale: the host sends each id once, so ids never tie and < and
+        // <= order alike.
+        // Stryker disable next-line EqualityOperator
+        (left.id < right.id ? -1 : 1))
+        .slice(0, limit);
+}
+// implements REQ-kibi-plugin-search-hints
+export function createSearchHints() {
+    return {
+        id: "kibi-plugin-search-hints.potion-base-8M",
+        model: MODEL_LABEL,
+        hint(input) {
+            // The table loads on the first hint, not when the plugin is resolved.
+            model ??= loadStaticModel();
+            const staticModel = model;
+            let cache = caches.get(input.cacheDirectory);
+            if (cache === undefined) {
+                cache = VectorCache.open(input.cacheDirectory, staticModel.id, staticModel.dims);
+                caches.set(input.cacheDirectory, cache);
+            }
+            return { hints: rankByMeaning(staticModel, cache, input) };
+        },
+    };
+}
+/**
+ * Offline meaning-based hints for weak kb_search answers, from the bundled
+ * potion-base-8M static embedding. No network, no native code.
+ */
+// implements REQ-kibi-plugin-search-hints
+export const kibiPlugin = defineKibiPlugin({
+    apiVersion: KIBI_PLUGIN_API_VERSION,
+    id: "kibi-plugin-search-hints",
+    version: packageJson.version,
+    permissions: { network: false, metered: false, secrets: [] },
+    capabilities: { searchHints: createSearchHints() },
+});
+export default kibiPlugin;

package/dist/model.d.ts

@@ -0,0 +1,18 @@
+/** Upstream model and the revision the bundled asset was built from. */
+export declare const MODEL_ID = "minishlab/potion-base-8M";
+export declare const MODEL_REVISION = "bf8b056651a2c21b8d2565580b8569da283cab23";
+/** The model a provider stamp discloses. */
+export declare const MODEL_LABEL: string;
+export type StaticModel = Readonly<{
+    /** Model id plus the asset hash, so a changed table never reuses vectors. */
+    id: string;
+    dims: number;
+    encode(text: string): Float32Array;
+}>;
+/**
+ * A static embedding: the mean of the int8 table rows of a text's tokens,
+ * L2-normalised. Sums run in double precision in token order, so a text has
+ * the same vector on every platform and in every batch.
+ */
+export declare function loadStaticModel(assetsDirectory?: string): StaticModel;
+//# sourceMappingURL=model.d.ts.map

package/dist/model.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"model.d.ts","sourceRoot":"","sources":["../src/model.ts"],"names":[],"mappings":"AAOA,wEAAwE;AACxE,eAAO,MAAM,QAAQ,6BAA6B,CAAC;AACnD,eAAO,MAAM,cAAc,6CAA6C,CAAC;AACzE,4CAA4C;AAC5C,eAAO,MAAM,WAAW,QAAoD,CAAC;AAK7E,MAAM,MAAM,WAAW,GAAG,QAAQ,CAAC;IACjC,6EAA6E;IAC7E,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY,CAAC;CACpC,CAAC,CAAC;AAIH;;;;GAIG;AAEH,wBAAgB,eAAe,CAC7B,eAAe,SAAmB,GACjC,WAAW,CAmDb"}

package/dist/model.js

@@ -0,0 +1,56 @@
+import { createHash } from "node:crypto";
+import { readFileSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { WordPieceTokenizer } from "./tokenizer.js";
+/** Upstream model and the revision the bundled asset was built from. */
+export const MODEL_ID = "minishlab/potion-base-8M";
+export const MODEL_REVISION = "bf8b056651a2c21b8d2565580b8569da283cab23";
+/** The model a provider stamp discloses. */
+export const MODEL_LABEL = `${MODEL_ID}@${MODEL_REVISION.slice(0, 12)}/int8`;
+/** model2vec truncates each text to this many tokens. */
+const MAX_TOKENS = 512;
+const ASSETS_DIRECTORY = fileURLToPath(new URL("../assets/", import.meta.url));
+/**
+ * A static embedding: the mean of the int8 table rows of a text's tokens,
+ * L2-normalised. Sums run in double precision in token order, so a text has
+ * the same vector on every platform and in every batch.
+ */
+// implements REQ-kibi-plugin-search-hints
+export function loadStaticModel(assetsDirectory = ASSETS_DIRECTORY) {
+    const table = readFileSync(path.join(assetsDirectory, "potion-base-8M.int8.bin"));
+    if (table.toString("ascii", 0, 4) !== "KSHM" || table.readUInt32LE(4) !== 1)
+        throw new Error("search hints model asset has an unknown format");
+    const rows = table.readUInt32LE(8);
+    const dims = table.readUInt32LE(12);
+    const scales = Float64Array.from({ length: rows }, (_, row) => table.readFloatLE(16 + row * 4));
+    const weights = new Int8Array(table.buffer, table.byteOffset + 16 + rows * 4, rows * dims);
+    const vocabulary = readFileSync(path.join(assetsDirectory, "vocab.txt"), "utf8")
+        .trimEnd()
+        .split("\n");
+    if (vocabulary.length !== rows)
+        throw new Error(`search hints vocabulary has ${vocabulary.length} tokens for ${rows} table rows`);
+    const tokenizer = new WordPieceTokenizer(vocabulary);
+    const digest = createHash("sha256").update(table).digest("hex");
+    return {
+        id: `${MODEL_LABEL}-${digest.slice(0, 12)}`,
+        dims,
+        encode(text) {
+            const ids = tokenizer
+                .encode(text)
+                .filter((token) => token !== tokenizer.unkId)
+                .slice(0, MAX_TOKENS);
+            const sum = new Float64Array(dims);
+            for (const token of ids) {
+                const scale = scales[token];
+                weights
+                    .subarray(token * dims, (token + 1) * dims)
+                    .forEach((weight, col) => {
+                    sum[col] = sum[col] + weight * scale;
+                });
+            }
+            const norm = Math.sqrt(sum.reduce((total, value) => total + value * value, 0));
+            return Float32Array.from(sum, (value) => (norm === 0 ? 0 : value / norm));
+        },
+    };
+}

package/dist/tokenizer.d.ts

@@ -0,0 +1,17 @@
+/**
+ * BERT WordPiece tokenizer, matching the Hugging Face `tokenizers`
+ * BertNormalizer (clean text, CJK spacing, accent stripping, lowercase),
+ * BertPreTokenizer and WordPiece model that potion-base-8M ships with.
+ */
+export declare const UNK_TOKEN = "[UNK]";
+export declare function normalize(text: string): string;
+export declare function preTokenize(normalized: string): string[];
+export declare class WordPieceTokenizer {
+    private readonly ids;
+    readonly unkId: number;
+    constructor(vocabulary: readonly string[]);
+    private word;
+    /** Token ids without special tokens, as `add_special_tokens: false`. */
+    encode(text: string): number[];
+}
+//# sourceMappingURL=tokenizer.d.ts.map

package/dist/tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,eAAO,MAAM,SAAS,UAAU,CAAC;AAqCjC,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAW9C;AAGD,wBAAgB,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAiBxD;AAGD,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,QAAQ,CAAC,GAAG,CAA8B;IAClD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;gBAEX,UAAU,EAAE,SAAS,MAAM,EAAE;IAOzC,OAAO,CAAC,IAAI;IA8BZ,wEAAwE;IACxE,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;CAY/B"}

package/dist/tokenizer.js

@@ -0,0 +1,135 @@
+/**
+ * BERT WordPiece tokenizer, matching the Hugging Face `tokenizers`
+ * BertNormalizer (clean text, CJK spacing, accent stripping, lowercase),
+ * BertPreTokenizer and WordPiece model that potion-base-8M ships with.
+ */
+const MAX_INPUT_CHARS_PER_WORD = 100;
+const CONTINUING_PREFIX = "##";
+export const UNK_TOKEN = "[UNK]";
+/** Added tokens matched verbatim before normalization, as tokenizer.json declares. */
+const SPECIAL_TOKENS = ["[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]"];
+const SPECIAL_SPLIT = /(\[PAD\]|\[UNK\]|\[CLS\]|\[SEP\]|\[MASK\])/;
+// Unassigned code points (Cn) are kept, as the upstream normalizer does.
+const CONTROL = /[\p{Cc}\p{Cf}\p{Co}]/u;
+const CLEAN_WHITESPACE = /[\t\n\r\p{Zs}]/u;
+const WHITESPACE = /\p{White_Space}/u;
+// BERT also counts every ASCII symbol as punctuation, not only Unicode P.
+const PUNCTUATION = /[\p{P}$+<=>^`|~]/u;
+const NONSPACING_MARK = /\p{Mn}/gu;
+function isControl(char) {
+    if (char === "\t" || char === "\n" || char === "\r")
+        return false;
+    return CONTROL.test(char);
+}
+function isChineseChar(code) {
+    return ((code >= 0x4e00 && code <= 0x9fff) ||
+        (code >= 0x3400 && code <= 0x4dbf) ||
+        (code >= 0x20000 && code <= 0x2a6df) ||
+        (code >= 0x2a700 && code <= 0x2b73f) ||
+        (code >= 0x2b740 && code <= 0x2b81f) ||
+        // Upstream starts this range at 0x2B920, not Unicode's 0x2B820.
+        (code >= 0x2b920 && code <= 0x2ceaf) ||
+        (code >= 0xf900 && code <= 0xfaff) ||
+        (code >= 0x2f800 && code <= 0x2fa1f));
+}
+function isPunctuation(char) {
+    return PUNCTUATION.test(char);
+}
+// implements REQ-kibi-plugin-search-hints
+export function normalize(text) {
+    let cleaned = "";
+    for (const char of text) {
+        const code = char.codePointAt(0) ?? 0;
+        // NUL is a control character too.
+        if (code === 0xfffd || isControl(char))
+            continue;
+        if (CLEAN_WHITESPACE.test(char))
+            cleaned += " ";
+        else if (isChineseChar(code))
+            cleaned += ` ${char} `;
+        else
+            cleaned += char;
+    }
+    return cleaned.normalize("NFD").replace(NONSPACING_MARK, "").toLowerCase();
+}
+// implements REQ-kibi-plugin-search-hints
+export function preTokenize(normalized) {
+    const words = [];
+    let current = "";
+    for (const char of normalized) {
+        if (WHITESPACE.test(char)) {
+            if (current !== "")
+                words.push(current);
+            current = "";
+        }
+        else if (isPunctuation(char)) {
+            if (current !== "")
+                words.push(current);
+            words.push(char);
+            current = "";
+        }
+        else {
+            current += char;
+        }
+    }
+    if (current !== "")
+        words.push(current);
+    return words;
+}
+// implements REQ-kibi-plugin-search-hints
+export class WordPieceTokenizer {
+    ids;
+    unkId;
+    constructor(vocabulary) {
+        this.ids = new Map(vocabulary.map((token, id) => [token, id]));
+        const unk = this.ids.get(UNK_TOKEN);
+        if (unk === undefined)
+            throw new Error("vocabulary has no [UNK] token");
+        this.unkId = unk;
+    }
+    word(word, out) {
+        const chars = Array.from(word);
+        if (chars.length > MAX_INPUT_CHARS_PER_WORD) {
+            out.push(this.unkId);
+            return;
+        }
+        const pieces = [];
+        let start = 0;
+        while (start < chars.length) {
+            let end = chars.length;
+            let found;
+            // rationale: at start === end the piece is empty, and neither "" nor
+            // "##" is a vocabulary entry, so the loop ends the same way.
+            // Stryker disable next-line EqualityOperator
+            while (start < end) {
+                const piece = chars.slice(start, end).join("");
+                found = this.ids.get(start > 0 ? CONTINUING_PREFIX + piece : piece);
+                if (found !== undefined)
+                    break;
+                end--;
+            }
+            if (found === undefined) {
+                out.push(this.unkId);
+                return;
+            }
+            pieces.push(found);
+            start = end;
+        }
+        out.push(...pieces);
+    }
+    /** Token ids without special tokens, as `add_special_tokens: false`. */
+    encode(text) {
+        const out = [];
+        for (const part of text.split(SPECIAL_SPLIT)) {
+            const special = SPECIAL_TOKENS.includes(part)
+                ? this.ids.get(part)
+                : undefined;
+            if (special !== undefined)
+                out.push(special);
+            else
+                for (const word of preTokenize(normalize(part)))
+                    this.word(word, out);
+        }
+        return out;
+    }
+}

package/integrity.json

@@ -0,0 +1,128 @@
+{
+  "schemaVersion": "kibi.package-integrity.v1",
+  "package": "kibi-plugin-search-hints",
+  "version": "0.1.0",
+  "algorithm": "sha256",
+  "selfHash": {
+    "path": "integrity.json",
+    "excluded": true,
+    "reason": "A manifest cannot contain its own digest; the host approval digest covers this file."
+  },
+  "files": [
+    {
+      "path": "CHANGELOG.md",
+      "bytes": 1902,
+      "sha256": "0fae42f30add91b196f8209a207b912973cd5d586904bf56ed348b1c8e033cc2"
+    },
+    {
+      "path": "LICENSE.md",
+      "bytes": 34519,
+      "sha256": "068de0a1cda6e78feb18db2150be2f023dca2b157062257b0fb8de50a359b22c"
+    },
+    {
+      "path": "README.md",
+      "bytes": 1474,
+      "sha256": "0607469f6a6a35aa3eb6a5eecf7d7ebe4663f1f707dd8dcd8c292183372955e8"
+    },
+    {
+      "path": "SBOM.spdx.json",
+      "bytes": 2353,
+      "sha256": "236722d2eaf390065ab6f1bfba872610d99d48346ce635703a85add0a26bdedc"
+    },
+    {
+      "path": "THIRD_PARTY_NOTICES.md",
+      "bytes": 993,
+      "sha256": "3b0b169c9173957364e4b03aeab1ebe87efd488fb3b465c022027ce7adc8c331"
+    },
+    {
+      "path": "assets/potion-base-8M.int8.bin",
+      "bytes": 7677296,
+      "sha256": "02eb3b486a152a01c49055e109cce3e483fc3b49761f5343b5518c0cfc76f54f"
+    },
+    {
+      "path": "assets/vocab.txt",
+      "bytes": 219690,
+      "sha256": "1394523a67ddd404a825428018c0582a6998bcfa044ecbcbf1f4d71adb94c61c"
+    },
+    {
+      "path": "catalog.json",
+      "bytes": 900,
+      "sha256": "686f031bfbffe8593e6dc355d3a859c067e17f3f61376abfe4dc84d11b48f4a1"
+    },
+    {
+      "path": "dist/cache.d.ts",
+      "bytes": 1103,
+      "sha256": "adeca3b210197f3f1039eb82024d800e806744a2e248b0d193de3ac4e0faf749"
+    },
+    {
+      "path": "dist/cache.d.ts.map",
+      "bytes": 723,
+      "sha256": "e7fea8c72ed8220c1a029509c865dca37026fc15f3e112237b7f48dca6b2ad43"
+    },
+    {
+      "path": "dist/cache.js",
+      "bytes": 4326,
+      "sha256": "5c35a4e265c8a48a1adb573770e97ea1a0126e4fc4dd23d7f508f52cce29f450"
+    },
+    {
+      "path": "dist/index.d.ts",
+      "bytes": 1288,
+      "sha256": "73eb11ee7415166394fd4df7cee6c53b9e12f0e149b2681721a28a36053c33ec"
+    },
+    {
+      "path": "dist/index.d.ts.map",
+      "bytes": 720,
+      "sha256": "b36616a0cf872a9de7aec8660dbdae5d3b12b9a36ecc07b2d1c250188e6c8d4f"
+    },
+    {
+      "path": "dist/index.js",
+      "bytes": 2978,
+      "sha256": "6a5eff91a9796e7fcc24ad3797d83840119894909f9a34a1858a3892b99ac8f9"
+    },
+    {
+      "path": "dist/model.d.ts",
+      "bytes": 831,
+      "sha256": "a9bc35f94df9392d8bb7d3167ee6bc17da6a2176532735132f20704e78039ddc"
+    },
+    {
+      "path": "dist/model.d.ts.map",
+      "bytes": 451,
+      "sha256": "e36bf801454b83d4d76902dfd903f7019e7edc74a5b050e232bc299d1036f8fd"
+    },
+    {
+      "path": "dist/model.js",
+      "bytes": 2765,
+      "sha256": "13af1dc20ce42d374264c14953d40320c2da7b23bf413db4af993bf928251621"
+    },
+    {
+      "path": "dist/tokenizer.d.ts",
+      "bytes": 704,
+      "sha256": "d949a9244fae4f6a29d53d3a75c554d517c3b3fe684028d52ef26d3b0981ab21"
+    },
+    {
+      "path": "dist/tokenizer.d.ts.map",
+      "bytes": 476,
+      "sha256": "e44d6f2e0b5e626d8e87e9fc9747e7c1d5bba44f36909ca15a2e69950d516c0c"
+    },
+    {
+      "path": "dist/tokenizer.js",
+      "bytes": 4728,
+      "sha256": "e35b1200ada6df138c36d90a7c861242a1b7eb3a662dadf653bf102495a59cc8"
+    },
+    {
+      "path": "licenses/bge-base-en-v1.5-vocabulary-MIT.txt",
+      "bytes": 1065,
+      "sha256": "587a673933425dbc36ec61268d3b954051b2d3ef3c9b322ede357976055ffdd5"
+    },
+    {
+      "path": "licenses/potion-base-8M-MIT.txt",
+      "bytes": 1074,
+      "sha256": "b0214f148eceae739f916209e78cfe15712c4b810c8bc9d44b6c2d1d1616aab6"
+    },
+    {
+      "path": "package.json",
+      "bytes": 1320,
+      "sha256": "0963d85f6b5483292d0e0e7e3ce313c2790e860ac93506f555f29a65d86058d4"
+    }
+  ]
+}

package/licenses/bge-base-en-v1.5-vocabulary-MIT.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 staoxiao
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/licenses/potion-base-8M-MIT.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Thomas van Dongen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.