mini-semantic-search 0.0.0-stage → 0.2.0

package/dist/model.js

@@ -0,0 +1,143 @@
+import { decodePacked, unpack } from './format.js';
+import { DEFAULT_MODEL, loadBytes } from './load.js';
+import { wordPiece } from './tokenizer.js';
+/**
+ * A Model2Vec static embedding model: one vector per token, and a sentence's vector is the mean of its tokens'
+ * vectors, normalised to length 1. No network runs, so embedding takes microseconds. Long texts are cut as the
+ * Python library cuts them: to `maxTokens` × the median token length in characters, then to `maxTokens` tokens,
+ * before unknown tokens are left out.
+ *
+ * A packed model (2 or 4 bits) stays packed in memory: about 2 MB for the bundled one instead of 30 MB.
+ */
+export class StaticModel {
+    tokens;
+    dims;
+    unk;
+    /** Tokens read per text; Model2Vec's default is 512. */
+    maxTokens = 512;
+    vocab;
+    medianTokenLength;
+    longestPiece;
+    table;
+    packed;
+    constructor(data) {
+        const { tokens, dims } = data;
+        this.tokens = tokens;
+        this.dims = dims;
+        this.vocab = new Map(tokens.map((t, i) => [t, i]));
+        this.unk = this.vocab.get('[UNK]') ?? -1;
+        // Token lengths in characters, counted (they are short), for the median and the longest word piece.
+        const counts = [];
+        let longest = 0;
+        for (const t of tokens) {
+            const len = t.length === 1 || !/[\uD800-\uDFFF]/.test(t) ? t.length : Array.from(t).length;
+            counts[len] = (counts[len] ?? 0) + 1;
+            longest = Math.max(longest, len);
+        }
+        const nth = (k) => {
+            for (let len = 0, seen = 0;; len++)
+                if ((seen += counts[len] ?? 0) > k)
+                    return len;
+        };
+        const mid = tokens.length >> 1;
+        this.medianTokenLength = Math.trunc(tokens.length % 2 ? nth(mid) : (nth(mid - 1) + nth(mid)) / 2);
+        this.longestPiece = longest;
+        if ('vectors' in data) {
+            this.table = data.vectors;
+            this.packed = null;
+        }
+        else {
+            // Each code byte holds `perByte` values; `lookup` gives their levels for every possible byte.
+            const perByte = 8 / data.bits, mask = (1 << data.bits) - 1;
+            const lookup = new Float32Array(256 * perByte);
+            for (let b = 0; b < 256; b++)
+                for (let j = 0; j < perByte; j++)
+                    lookup[b * perByte + j] = data.levels[(b >> (j * data.bits)) & mask];
+            this.table = null;
+            this.packed = { ...data, lookup, perByte, rowBytes: (dims * data.bits) / 8 };
+        }
+    }
+    /** Loads a model file; by default the bundled potion-base-8M (256 dimensions at 2 bits, 2.2 MB). */
+    static async load(src = DEFAULT_MODEL) {
+        return StaticModel.fromBytes(await loadBytes(src));
+    }
+    static fromBytes(bytes) {
+        return new StaticModel(decodePacked(bytes));
+    }
+    /** The table of token vectors, tokens.length × dims, row by row (unpacked on first use for a packed model). */
+    get vectors() {
+        return (this.table ??= unpack(this.packed).vectors);
+    }
+    /** Token ids, as Hugging Face's tokenizer gives them without special tokens (unknown words included as UNK). */
+    tokenize(text) {
+        return wordPiece(this.vocab, this.unk, text, this.longestPiece);
+    }
+    /** Whether `token` (e.g. "voucher" or "##ing") is in the vocabulary. */
+    has(token) {
+        return this.vocab.has(token);
+    }
+    /** The text's vector: unit length, or all zeros if it has no known tokens. */
+    embed(text) {
+        const { dims, packed } = this;
+        const v = new Float32Array(dims);
+        const chars = this.maxTokens * this.medianTokenLength;
+        if (text.length > chars)
+            text = Array.from(text).slice(0, chars).join('');
+        const ids = this.tokenize(text);
+        const end = Math.min(ids.length, this.maxTokens);
+        if (packed) {
+            const { codes, rowScales, lookup, perByte, rowBytes, colScales } = packed;
+            for (let t = 0; t < end; t++) {
+                const id = ids[t];
+                if (id === this.unk)
+                    continue;
+                const r = rowScales[id];
+                if (perByte === 4) {
+                    for (let p = id * rowBytes, d = 0; d < dims; p++, d += 4) {
+                        const o = codes[p] << 2;
+                        v[d] += lookup[o] * r;
+                        v[d + 1] += lookup[o + 1] * r;
+                        v[d + 2] += lookup[o + 2] * r;
+                        v[d + 3] += lookup[o + 3] * r;
+                    }
+                }
+                else {
+                    for (let p = id * rowBytes, d = 0; d < dims; p++, d += 2) {
+                        const o = codes[p] << 1;
+                        v[d] += lookup[o] * r;
+                        v[d + 1] += lookup[o + 1] * r;
+                    }
+                }
+            }
+            for (let d = 0; d < dims; d++)
+                v[d] *= colScales[d];
+        }
+        else {
+            const table = this.table;
+            for (let t = 0; t < end; t++) {
+                const id = ids[t];
+                if (id === this.unk)
+                    continue;
+                for (let d = 0, o = id * dims; d < dims; d++)
+                    v[d] += table[o + d];
+            }
+        }
+        let norm = 0;
+        for (let d = 0; d < dims; d++)
+            norm += v[d] * v[d];
+        norm = Math.sqrt(norm) || 1;
+        for (let d = 0; d < dims; d++)
+            v[d] /= norm;
+        return v;
+    }
+    embedMany(texts) {
+        return texts.map((t) => this.embed(t));
+    }
+}
+/** Dot product: the cosine similarity of two unit vectors, such as `embed` returns (1 = same meaning). */
+export function similarity(a, b) {
+    let s = 0;
+    for (let i = 0; i < a.length; i++)
+        s += a[i] * b[i];
+    return s;
+}

package/dist/node.d.ts

@@ -0,0 +1,4 @@
+import { MiniSemanticSearch } from './index.js';
+export default MiniSemanticSearch;
+export * from './index.js';
+export { packModel, type PackOptions } from './pack.js';

package/dist/node.js

@@ -0,0 +1,8 @@
+// The Node entry point: everything in the browser one, plus loading from local paths and packing models.
+import { readFile } from 'node:fs/promises';
+import { setFileReader } from './load.js';
+import { MiniSemanticSearch } from './index.js';
+setFileReader(async (path) => new Uint8Array(await readFile(path)));
+export default MiniSemanticSearch;
+export * from './index.js';
+export { packModel } from './pack.js';

package/dist/pack.d.ts

@@ -0,0 +1,10 @@
+import { type EncodeOptions } from './format.js';
+export interface PackOptions extends EncodeOptions {
+    /** Keep only the leading dimensions (Model2Vec's are PCA components, so they hold the most variance). */
+    dims?: number;
+}
+/**
+ * Packs a Model2Vec model folder (model.safetensors and tokenizer.json, as on Hugging Face, e.g.
+ * minishlab/potion-base-8M) into the file `StaticModel.load` reads.
+ */
+export declare function packModel(dir: string, { dims, ...encode }?: PackOptions): Promise<Uint8Array>;

package/dist/pack.js

@@ -0,0 +1,37 @@
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { encodeModel } from './format.js';
+/**
+ * Packs a Model2Vec model folder (model.safetensors and tokenizer.json, as on Hugging Face, e.g.
+ * minishlab/potion-base-8M) into the file `StaticModel.load` reads.
+ */
+export async function packModel(dir, { dims, ...encode } = {}) {
+    const st = await readFile(join(dir, 'model.safetensors'));
+    const headerLen = Number(st.readBigUInt64LE(0));
+    const header = JSON.parse(st.subarray(8, 8 + headerLen).toString());
+    const others = Object.keys(header).filter((k) => k !== 'embeddings' && k !== '__metadata__');
+    if (others.length)
+        throw new Error(`unsupported tensors: ${others.join(', ')} (token weights or mappings)`);
+    const info = header.embeddings;
+    if (!info || info.dtype !== 'F32')
+        throw new Error('expected an F32 "embeddings" tensor');
+    const [n, full] = info.shape;
+    const keep = Math.min(dims ?? full, full);
+    const data = st.subarray(8 + headerLen + info.data_offsets[0], 8 + headerLen + info.data_offsets[1]);
+    const src = new Float32Array(data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength));
+    const tok = JSON.parse(await readFile(join(dir, 'tokenizer.json'), 'utf8'));
+    if (tok.model?.type !== 'WordPiece' || tok.normalizer?.type !== 'BertNormalizer' || tok.normalizer.lowercase === false) {
+        throw new Error('only lower-casing BERT (WordPiece) tokenizers are supported');
+    }
+    const tokens = [];
+    for (const [t, id] of Object.entries(tok.model.vocab))
+        tokens[id] = t;
+    for (const a of tok.added_tokens ?? [])
+        tokens[a.id] = a.content;
+    if (tokens.length !== n || tokens.includes(undefined))
+        throw new Error('vocabulary does not match the table');
+    const vectors = new Float32Array(n * keep);
+    for (let i = 0; i < n; i++)
+        vectors.set(src.subarray(i * full, i * full + keep), i * keep);
+    return encodeModel({ tokens, dims: keep, vectors }, encode);
+}

package/dist/search.d.ts

@@ -0,0 +1,84 @@
+import { type Source } from './load.js';
+import { StaticModel } from './model.js';
+type AnyDocument = any;
+type DocumentId = any;
+export interface SearchOptions {
+    /** Only these fields count (default: all of `fields`). */
+    fields?: string[];
+    /** Weights of fields against each other, e.g. `{ title: 2 }` (default 1 each). */
+    boost?: Record<string, number>;
+    /** Only results for which this is true. */
+    filter?: (result: SearchResult) => boolean;
+    /** Leave out results scoring under this (default: none). */
+    minScore?: number;
+}
+export interface Options<T = AnyDocument> {
+    /** The fields embedded and searched. */
+    fields: string[];
+    /** The fields returned with each result (default: none). */
+    storeFields?: string[];
+    /** The field holding each document's unique id (default "id"). */
+    idField?: string;
+    /** Reads a field from a document (default: `document[fieldName]`). Arrays are embedded item by item. */
+    extractField?: (document: T, fieldName: string) => unknown;
+    /** Defaults for `search`. */
+    searchOptions?: SearchOptions;
+    /** A loaded model, or where to load one from in `create` (default: the bundled potion-base-8M). */
+    model?: StaticModel | Source;
+}
+export type SearchResult = {
+    id: DocumentId;
+    /** About −0.1 to 1: how close in meaning the document is to the query. */
+    score: number;
+    [storedField: string]: unknown;
+};
+/**
+ * Semantic search with MiniSearch's API: documents are found by meaning rather than shared words, so "stop sending me
+ * newsletters" finds "How do I unsubscribe from your emails?". Each document is embedded once per field and once
+ * whole. It scores half its whole vector's cosine similarity to the query, plus half its fields' (weighted by
+ * `boost`; an array field counts its closest item). Searching takes no network and no server.
+ */
+export declare class MiniSemanticSearch<T = AnyDocument> {
+    readonly model: StaticModel;
+    private readonly options;
+    private readonly index;
+    private plan;
+    /** The weighting of the last search that had no plan: a plan is made when one is used twice running. */
+    private lastKey;
+    /** Loads the model (the bundled one unless `options.model` says otherwise), then makes the index. */
+    static create<T = AnyDocument>(options: Options<T>): Promise<MiniSemanticSearch<T>>;
+    /** For a model you have loaded already (`StaticModel.load()`); `create` loads it for you. */
+    constructor(options: Options<T> & {
+        model: StaticModel;
+    });
+    /** How many documents are indexed. */
+    get documentCount(): number;
+    /** Adds a document; it can be searched straight away. Embedding takes about 0.05 ms. */
+    add(document: T): void;
+    addAll(documents: readonly T[]): void;
+    /** Adds documents `chunkSize` at a time, yielding between chunks so a page stays responsive. */
+    addAllAsync(documents: readonly T[], { chunkSize }?: {
+        chunkSize?: number;
+    }): Promise<void>;
+    /** Removes a document, found by its id. */
+    remove(document: T): void;
+    /** Removes the document with this id. */
+    discard(id: DocumentId): void;
+    discardAll(ids: readonly DocumentId[]): void;
+    /** Removes the given documents, or every document if none are given. */
+    removeAll(documents?: readonly T[]): void;
+    /** Replaces the document that has the same id. */
+    replace(document: T): void;
+    has(id: DocumentId): boolean;
+    /** The document's `storeFields`, or undefined if it isn't indexed. */
+    getStoredFields(id: DocumentId): Record<string, unknown> | undefined;
+    /** Every document, best first, with its id, score and stored fields. */
+    search(query: string, searchOptions?: SearchOptions): SearchResult[];
+    /** The plan for this weighting: the current one, a new one if it was also the last search's, or else none. */
+    private planFor;
+    /** Adds the document's row to the plan. */
+    private place;
+    /** Takes the document's row out of the plan, moving the last row into its place. */
+    private unplace;
+}
+export {};

package/dist/search.js

@@ -0,0 +1,270 @@
+import { DEFAULT_MODEL } from './load.js';
+import { StaticModel } from './model.js';
+const pack8 = (v) => {
+    let max = 0;
+    for (let i = 0; i < v.length; i++)
+        max = Math.max(max, Math.abs(v[i]));
+    const s = max / 127 || 1, q = new Int8Array(v.length);
+    for (let i = 0; i < v.length; i++)
+        q[i] = Math.round(v[i] / s);
+    return { q, s };
+};
+/** q · row `o` of m. */
+const dot = (q, m, o) => {
+    let a = 0, b = 0, c = 0, d = 0, i = 0;
+    for (; i + 3 < q.length; i += 4) {
+        a += q[i] * m[o + i];
+        b += q[i + 1] * m[o + i + 1];
+        c += q[i + 2] * m[o + i + 2];
+        d += q[i + 3] * m[o + i + 3];
+    }
+    for (; i < q.length; i++)
+        a += q[i] * m[o + i];
+    return a + b + c + d;
+};
+const dot8 = (q, { q: v, s }) => {
+    let a = 0, b = 0, c = 0, d = 0, i = 0;
+    for (; i + 3 < q.length; i += 4) {
+        a += q[i] * v[i];
+        b += q[i + 1] * v[i + 1];
+        c += q[i + 2] * v[i + 2];
+        d += q[i + 3] * v[i + 3];
+    }
+    for (; i < q.length; i++)
+        a += q[i] * v[i];
+    return (a + b + c + d) * s;
+};
+/**
+ * A document's score as terms, each a weight times the best similarity among some vectors (a single-valued field has
+ * one): half the whole vector's similarity plus half the fields' similarities averaged by weight, or just the fields'
+ * average when only some fields count. Null if no field with a value counts.
+ */
+function terms(doc, { use, weights, subset }) {
+    let weight = 0;
+    for (let i = 0; i < doc.fields.length; i++)
+        if (use[i] && doc.fields[i].length)
+            weight += weights[i];
+    if (!weight)
+        return null;
+    // The whole-document vector covers every field, so it only counts when every field does.
+    const whole = subset || !doc.whole ? 0 : 0.5;
+    const out = whole ? [[whole, [doc.whole]]] : [];
+    for (let i = 0; i < doc.fields.length; i++) {
+        if (use[i] && doc.fields[i].length)
+            out.push([((1 - whole) * weights[i]) / weight, doc.fields[i]]);
+    }
+    return out;
+}
+const texts = (value) => {
+    if (value === undefined || value === null)
+        return [];
+    return (Array.isArray(value) ? value : [value]).map(String).filter((s) => s.trim());
+};
+/**
+ * Semantic search with MiniSearch's API: documents are found by meaning rather than shared words, so "stop sending me
+ * newsletters" finds "How do I unsubscribe from your emails?". Each document is embedded once per field and once
+ * whole. It scores half its whole vector's cosine similarity to the query, plus half its fields' (weighted by
+ * `boost`; an array field counts its closest item). Searching takes no network and no server.
+ */
+export class MiniSemanticSearch {
+    model;
+    options;
+    index = new Map();
+    plan = null;
+    /** The weighting of the last search that had no plan: a plan is made when one is used twice running. */
+    lastKey = '';
+    /** Loads the model (the bundled one unless `options.model` says otherwise), then makes the index. */
+    static async create(options) {
+        const { model = DEFAULT_MODEL } = options;
+        return new MiniSemanticSearch({ ...options, model: model instanceof StaticModel ? model : await StaticModel.load(model) });
+    }
+    /** For a model you have loaded already (`StaticModel.load()`); `create` loads it for you. */
+    constructor(options) {
+        const { model, fields, storeFields = [], idField = 'id', extractField = (d, f) => d[f], searchOptions = {} } = options;
+        if (!(model instanceof StaticModel))
+            throw new Error('MiniSemanticSearch: the constructor needs a loaded model; use MiniSemanticSearch.create to load one');
+        if (!fields?.length)
+            throw new Error('MiniSemanticSearch: option "fields" must list at least one field');
+        this.model = model;
+        this.options = { fields, storeFields, idField, extractField, searchOptions };
+    }
+    /** How many documents are indexed. */
+    get documentCount() {
+        return this.index.size;
+    }
+    /** Adds a document; it can be searched straight away. Embedding takes about 0.05 ms. */
+    add(document) {
+        const { fields, storeFields, idField, extractField } = this.options;
+        const id = extractField(document, idField);
+        if (id === undefined)
+            throw new Error(`MiniSemanticSearch: document does not have ID field "${idField}"`);
+        if (this.index.has(id))
+            throw new Error(`MiniSemanticSearch: duplicate ID ${id}`);
+        const values = fields.map((f) => texts(extractField(document, f)));
+        const all = values.flat().join(' ');
+        const doc = {
+            id,
+            fields: values.map((v) => v.map((s) => pack8(this.model.embed(s)))),
+            whole: all ? pack8(this.model.embed(all)) : null,
+            stored: Object.fromEntries(storeFields.map((f) => [f, extractField(document, f)])),
+            row: -1,
+        };
+        this.index.set(id, doc);
+        if (this.plan)
+            this.place(this.plan, doc);
+    }
+    addAll(documents) {
+        for (const d of documents)
+            this.add(d);
+    }
+    /** Adds documents `chunkSize` at a time, yielding between chunks so a page stays responsive. */
+    async addAllAsync(documents, { chunkSize = 100 } = {}) {
+        for (let i = 0; i < documents.length; i += chunkSize) {
+            this.addAll(documents.slice(i, i + chunkSize));
+            await new Promise((resolve) => setTimeout(resolve, 0));
+        }
+    }
+    /** Removes a document, found by its id. */
+    remove(document) {
+        this.discard(this.options.extractField(document, this.options.idField));
+    }
+    /** Removes the document with this id. */
+    discard(id) {
+        const doc = this.index.get(id);
+        if (!doc)
+            throw new Error(`MiniSemanticSearch: cannot discard document with ID ${id}: it is not in the index`);
+        this.unplace(doc);
+        this.index.delete(id);
+    }
+    discardAll(ids) {
+        for (const id of ids)
+            this.discard(id);
+    }
+    /** Removes the given documents, or every document if none are given. */
+    removeAll(documents) {
+        if (documents === undefined) {
+            this.index.clear();
+            this.plan = null;
+        }
+        else
+            for (const d of documents)
+                this.remove(d);
+    }
+    /** Replaces the document that has the same id. */
+    replace(document) {
+        this.remove(document);
+        this.add(document);
+    }
+    has(id) {
+        return this.index.has(id);
+    }
+    /** The document's `storeFields`, or undefined if it isn't indexed. */
+    getStoredFields(id) {
+        return this.index.get(id)?.stored;
+    }
+    /** Every document, best first, with its id, score and stored fields. */
+    search(query, searchOptions = {}) {
+        const { fields: only, boost = {}, filter, minScore = -Infinity } = { ...this.options.searchOptions, ...searchOptions };
+        const { fields } = this.options;
+        const use = fields.map((f) => !only || only.includes(f));
+        const weights = fields.map((f, i) => (use[i] ? boost[f] ?? 1 : 0));
+        const weighting = { key: JSON.stringify([use, weights]), use, weights, subset: use.some((u) => !u) };
+        const q = this.model.embed(query);
+        const results = [];
+        const emit = (doc, score) => {
+            if (score < minScore)
+                return;
+            const result = { id: doc.id, score, ...doc.stored }; // stored fields come last, as in MiniSearch
+            if (!filter || filter(result))
+                results.push(result);
+        };
+        const best = (items) => {
+            let b = -Infinity;
+            for (const v of items)
+                b = Math.max(b, dot8(q, v));
+            return b;
+        };
+        const plan = this.planFor(weighting);
+        if (plan) {
+            const { rows, docs, multi } = plan;
+            for (let r = 0, o = 0; r < docs.length; r++, o += q.length) {
+                let score = dot(q, rows, o);
+                const m = multi[r];
+                if (m)
+                    for (const [w, items] of m)
+                        score += w * best(items);
+                emit(docs[r], score);
+            }
+        }
+        else {
+            for (const doc of this.index.values()) {
+                const t = terms(doc, weighting);
+                if (!t)
+                    continue;
+                let score = 0;
+                for (const [w, items] of t)
+                    score += w * best(items);
+                emit(doc, score);
+            }
+        }
+        return results.sort((a, b) => b.score - a.score);
+    }
+    /** The plan for this weighting: the current one, a new one if it was also the last search's, or else none. */
+    planFor(weighting) {
+        if (this.plan?.key === weighting.key)
+            return this.plan;
+        if (this.lastKey !== weighting.key) {
+            this.lastKey = weighting.key;
+            return null;
+        }
+        const plan = { ...weighting, rows: new Float32Array(Math.max(this.index.size, 16) * this.model.dims), docs: [], multi: [] };
+        this.plan = plan;
+        for (const doc of this.index.values())
+            this.place(plan, doc);
+        return plan;
+    }
+    /** Adds the document's row to the plan. */
+    place(plan, doc) {
+        doc.row = -1;
+        const t = terms(doc, plan);
+        if (!t)
+            return;
+        const dims = this.model.dims, n = plan.docs.length;
+        if ((n + 1) * dims > plan.rows.length) {
+            const grown = new Float32Array(plan.rows.length * 2);
+            grown.set(plan.rows);
+            plan.rows = grown;
+        }
+        const row = plan.rows.subarray(n * dims, (n + 1) * dims).fill(0);
+        let multi = null;
+        for (const [w, items] of t) {
+            if (items.length > 1) {
+                (multi ??= []).push([w, items]);
+                continue;
+            }
+            const { q, s } = items[0], k = s * w;
+            for (let i = 0; i < dims; i++)
+                row[i] += q[i] * k;
+        }
+        plan.docs.push(doc);
+        plan.multi.push(multi);
+        doc.row = n;
+    }
+    /** Takes the document's row out of the plan, moving the last row into its place. */
+    unplace(doc) {
+        const plan = this.plan;
+        if (!plan || doc.row < 0)
+            return;
+        const dims = this.model.dims, last = plan.docs.length - 1;
+        if (doc.row !== last) {
+            plan.rows.copyWithin(doc.row * dims, last * dims, (last + 1) * dims);
+            const moved = plan.docs[last];
+            plan.docs[doc.row] = moved;
+            plan.multi[doc.row] = plan.multi[last];
+            moved.row = doc.row;
+        }
+        plan.docs.pop();
+        plan.multi.pop();
+        doc.row = -1;
+    }
+}

package/dist/tokenizer.d.ts

@@ -0,0 +1,9 @@
+/** Drops control characters, spaces out CJK characters, strips accents and lower-cases. */
+export declare function normalize(text: string): string;
+/** Splits normalised text into words and single punctuation characters. */
+export declare function preTokenize(text: string): string[];
+/**
+ * Token ids for text: each word as the longest known pieces, left to right, or `unk` if it can't be spelt.
+ * `longest` (the vocabulary's longest token, in characters) only saves looking up pieces too long to be known.
+ */
+export declare function wordPiece(vocab: Map<string, number>, unk: number, text: string, longest?: number): number[];

package/dist/tokenizer.js

@@ -0,0 +1,85 @@
+// BERT's tokenizer, as Hugging Face's `tokenizers` runs it for Model2Vec models (BertNormalizer with lowercasing,
+// BertPreTokenizer, WordPiece with "##" continuations). Checked token for token against it in the tests.
+const isPunct = (c) => {
+    const cp = c.codePointAt(0);
+    if (cp < 128)
+        return (cp >= 33 && cp <= 47) || (cp >= 58 && cp <= 64) || (cp >= 91 && cp <= 96) || (cp >= 123 && cp <= 126);
+    return /\p{P}/u.test(c);
+};
+const isCjk = (cp) => (cp >= 0x4e00 && cp <= 0x9fff) || (cp >= 0x3400 && cp <= 0x4dbf)
+    || (cp >= 0x20000 && cp <= 0x2a6df) || (cp >= 0x2a700 && cp <= 0x2b73f) || (cp >= 0x2b740 && cp <= 0x2b81f)
+    || (cp >= 0x2b820 && cp <= 0x2ceaf) || (cp >= 0xf900 && cp <= 0xfaff) || (cp >= 0x2f800 && cp <= 0x2fa1f);
+/** Drops control characters, spaces out CJK characters, strips accents and lower-cases. */
+export function normalize(text) {
+    // Printable ASCII, tabs and newlines: nothing to drop, space out or strip.
+    if (/^[\t\n\r\x20-\x7e]*$/.test(text))
+        return text.replace(/[\t\n\r]/g, ' ').toLowerCase();
+    let out = '';
+    for (const c of text) {
+        const cp = c.codePointAt(0);
+        if (cp === 0 || cp === 0xfffd || (/\p{C}/u.test(c) && !/[\t\n\r]/.test(c)))
+            continue;
+        if (/\s/u.test(c))
+            out += ' ';
+        else if (isCjk(cp))
+            out += ` ${c} `;
+        else
+            out += c;
+    }
+    return out.normalize('NFD').replace(/\p{Mn}/gu, '').toLowerCase();
+}
+/** Splits normalised text into words and single punctuation characters. */
+export function preTokenize(text) {
+    const words = [];
+    for (const chunk of text.split(' ')) {
+        let word = '';
+        for (const c of chunk) {
+            if (isPunct(c)) {
+                if (word)
+                    words.push(word);
+                words.push(c);
+                word = '';
+            }
+            else
+                word += c;
+        }
+        if (word)
+            words.push(word);
+    }
+    return words;
+}
+/**
+ * Token ids for text: each word as the longest known pieces, left to right, or `unk` if it can't be spelt.
+ * `longest` (the vocabulary's longest token, in characters) only saves looking up pieces too long to be known.
+ */
+export function wordPiece(vocab, unk, text, longest = Infinity) {
+    const ids = [];
+    for (const w of preTokenize(normalize(text))) {
+        const chars = Array.from(w);
+        if (chars.length > 100) {
+            ids.push(unk);
+            continue;
+        }
+        const piece = chars.length === w.length ? (s, e) => w.slice(s, e) : (s, e) => chars.slice(s, e).join('');
+        const pieces = [];
+        for (let start = 0; start < chars.length;) {
+            let end = Math.min(chars.length, start + longest);
+            let id;
+            for (; end > start; end--) {
+                id = vocab.get(start ? '##' + piece(start, end) : piece(start, end));
+                if (id !== undefined)
+                    break;
+            }
+            if (end === start) {
+                pieces.length = 0;
+                pieces.push(unk);
+                break;
+            }
+            pieces.push(id);
+            start = end;
+        }
+        for (const id of pieces)
+            ids.push(id);
+    }
+    return ids;
+}

package/models/README.md

@@ -0,0 +1,14 @@
+# Bundled data
+
+This file is not covered by the package's MIT licence for its code; it keeps its source's terms.
+
+## potion-base-8M.bin
+
+[minishlab/potion-base-8M](https://huggingface.co/minishlab/potion-base-8M) by MinishLab, a
+[Model2Vec](https://github.com/MinishLab/model2vec) model distilled from
+[BAAI/bge-base-en-v1.5](https://huggingface.co/BAAI/bge-base-en-v1.5), released under the MIT licence. Its
+vocabulary is bge-base-en-v1.5's (BERT's, uncased).
+
+Modified: repacked by `mini-semantic-search pack` at 2 bits per value (each value one of four fitted levels, times
+an f32 scale for its dimension and an f16 scale for its token), 2.2 MB instead of 30.5 MB. All 29,528 tokens and all
+256 dimensions are kept.