mini-semantic-search 0.0.0-stage → 0.2.0

package/LICENSE

@@ -0,0 +1,24 @@
+MIT License
+
+Copyright (c) 2026 ShakirasHips
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+This licence covers the code. The files in models/ come from others and keep
+their own licences: see models/README.md.

package/README.md

@@ -1,3 +1,213 @@
-# Temporary Holding Version
+# mini-semantic-search
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Semantic search with [MiniSearch](https://github.com/lucaong/minisearch)'s API. Documents are matched by meaning
+rather than shared words: "stop sending me newsletters" finds "How do I unsubscribe from your emails?", and "the mug
+came smashed" finds "My item arrived damaged or broken". It runs in browsers and Node with no dependencies, no
+server and no API calls, and the package is 2.2 MB with its model (2.0 MB gzipped).
+
+```ts
+import MiniSemanticSearch from 'mini-semantic-search';
+
+const search = await MiniSemanticSearch.create({
+  fields: ['title', 'text'],       // fields to embed and search
+  storeFields: ['title'],          // fields to return with results
+});
+
+search.addAll([
+  { id: 1, title: 'Returns', text: 'Start a return under Orders in your account.' },
+  { id: 2, title: 'Damaged items', text: 'Send us a photo of the damage and we will replace it.' },
+  { id: 3, title: 'Shipping times', text: 'Standard shipping takes 3 to 5 business days.' },
+]);
+
+search.search('the mug came smashed');
+// => [
+//   { id: 2, score: 0.29, title: 'Damaged items' },
+//   { id: 3, score: 0.16, title: 'Shipping times' },
+//   { id: 1, score: 0.02, title: 'Returns' },
+// ]
+```
+
+The embeddings come from [Model2Vec](https://github.com/MinishLab/model2vec), a static embedding model. It keeps one
+vector per token, and a text's vector is the average of its tokens' vectors. There is no neural network to run, so
+embedding a paragraph takes about 15 µs on a CPU.
+
+## Install
+
+```sh
+npm install mini-semantic-search
+```
+
+- **Node 18+:** `import MiniSemanticSearch from 'mini-semantic-search'`. The bundled model loads from the package.
+- **Bundlers (Vite, webpack 5, esbuild, Rollup):** the same import. The model is found with
+  `new URL('../models/…', import.meta.url)`, which bundlers copy as an asset. Or pass `model:` your own URL.
+- **Straight from a CDN, no build step:**
+
+  ```html
+  <script type="module">
+    import MiniSemanticSearch from 'https://cdn.jsdelivr.net/npm/mini-semantic-search@0.2.0/dist/index.js';
+    const search = await MiniSemanticSearch.create({ fields: ['title', 'text'] });
+  </script>
+  ```
+
+  The model (2.2 MB) then loads from the CDN too.
+
+## API
+
+It follows MiniSearch, so code written for one mostly works with the other. The one difference in setting it up is
+that the model has to load first, so you either `await MiniSemanticSearch.create(options)` or pass a loaded model
+to the constructor:
+
+```ts
+import MiniSemanticSearch, { StaticModel } from 'mini-semantic-search';
+
+const model = await StaticModel.load();  // load once, share between indexes
+const search = new MiniSemanticSearch({ model, fields: ['title', 'text'] });
+```
+
+| Option | Default | |
+|---|---|---|
+| `fields` | required | The fields embedded and searched. A field holding an array is embedded item by item, and its closest item counts. |
+| `storeFields` | `[]` | The fields returned with each result. |
+| `idField` | `'id'` | The field holding each document's unique id. |
+| `extractField` | `doc[field]` | `(document, fieldName) => value`, for nested or computed fields. |
+| `searchOptions` | `{}` | Defaults for `search`. |
+| `model` | bundled potion-base-8M | A `StaticModel`, or (in `create`) a URL, path or bytes of a packed model. |
+
+| Method | |
+|---|---|
+| `add(doc)`, `addAll(docs)` | Index documents. They can be searched straight away. A duplicate id throws. |
+| `addAllAsync(docs, { chunkSize: 100 })` | The same, yielding between chunks so a page stays responsive. |
+| `remove(doc)`, `discard(id)`, `discardAll(ids)` | Remove documents, by document or by id. |
+| `removeAll(docs?)` | Remove the given documents, or all of them. |
+| `replace(doc)` | Replace the document with the same id. |
+| `has(id)`, `getStoredFields(id)`, `documentCount` | What's indexed. |
+| `search(query, options?)` | Every document, best first, as `{ id, score, ...storeFields }`. |
+
+Search options:
+
+| Option | |
+|---|---|
+| `fields` | Search only these fields, e.g. `{ fields: ['title'] }`. |
+| `boost` | Weights of fields against each other, e.g. `{ boost: { title: 3 } }`. |
+| `filter` | `(result) => boolean`; the result includes its stored fields. |
+| `minScore` | Leave out results scoring under this. |
+
+**How documents are scored.** Each document is embedded once per field and once with all its fields together. Its
+score is half the whole document's cosine similarity to the query, plus half its fields' similarities averaged
+(weighted by `boost`). Scores run from about −0.1 to 1. A document always scores something, so every document comes
+back: `slice` the results, or set `minScore` to drop the ones too far off to count as a match. Where to set it
+depends on your documents' wording. Try your own off-topic questions and put it just above the scores they get.
+
+**How it differs from MiniSearch.** Results have no `terms`, `queryTerms` or `match`, because nothing is matched
+word for word. There's no `prefix`, `fuzzy`, `combineWith`, `autoSuggest` or query syntax, and no
+`toJSON`/`loadJSON`: indexing costs about 0.05 ms per document, so you can just index again on load.
+
+Indexing takes about 0.05 ms per document with three fields. A search over 10,000 documents takes about 4 ms, or
+about 13 ms when its `fields` or `boost` differ from the last search's. (Options used twice running get each
+document's vectors added up into one, so a search is one dot product per document.)
+
+## Embeddings
+
+```ts
+import { StaticModel, similarity } from 'mini-semantic-search';
+
+const model = await StaticModel.load();            // or StaticModel.load(urlOrPathOrBytes)
+const a = model.embed('how do I cancel my order');  // Float32Array(256), unit length
+const b = model.embed('stop my purchase before it ships');
+similarity(a, b);                                   // cosine similarity, 1 = same meaning
+model.embedMany(['refund', 'exchange']);           // several at once
+model.tokenize('Café');                             // token ids
+```
+
+This matches the Python library. Token ids are identical to Hugging Face's tokenizer, and with a full-precision
+model the vectors match `model2vec` to within 1e-6. The tests check the token ids and the bundled model's vectors
+against fixtures made with `scripts/make-fixture.py`.
+
+Any Model2Vec model with a lower-casing BERT tokenizer and no token weights can be packed (most `minishlab/potion-*`
+models). At 4 bits instead of 2, potion-base-8M is 4.1 MB and scores as the full-precision model does:
+
+```sh
+npx mini-semantic-search pack ./potion-base-8M ./potion-base-8M-4bit.bin --bits 4
+```
+
+```ts
+const search = await MiniSemanticSearch.create({ fields: ['title', 'text'], model: '/potion-base-8M-4bit.bin' });
+```
+
+## Command line
+
+```sh
+npx mini-semantic-search search docs.json "how do I get a refund" --fields title,text [--k 5]
+npx mini-semantic-search embed "some text"                  # the vector as JSON
+npx mini-semantic-search pack <model folder> out.bin [--bits 2|4|8|32] [--dims N]
+```
+
+## How well it works
+
+On the example FAQ in `examples/` (a made-up shop, 45 entries searched on `section`, `q` and `a`, and 135 casual
+test questions), with the right entry ranked first or in the top 3:
+
+| Model | Size | `boost` | First | Top 3 |
+|---|---|---|---|---|
+| **potion-base-8M at 2 bits (bundled)** | **2.2 MB** | none | **66.7%** | **83.7%** |
+| **potion-base-8M at 2 bits (bundled)** | **2.2 MB** | `{ q: 3 }` | **65.2%** | **85.9%** |
+| potion-base-8M at 4 bits | 4.1 MB | `{ q: 3 }` | 65.2% | 88.1% |
+| potion-base-8M, full precision | 30.5 MB | `{ q: 3 }` | 64.4% | 88.1% |
+| 0.1's model: 128 of the 256 dimensions at 4 bits | 2.2 MB | `{ q: 3 }` | 60.7% | 83.7% |
+
+The test questions were written with the entries in view, so expect real questions to do somewhat worse. The set
+is small: one question is 0.7%. Two public benchmarks are bigger. In [Banking77](https://huggingface.co/datasets/mteb/banking77),
+3,080 customer questions each count as right when the closest of 10,003 other questions has the same intent (of 77).
+In [SciFact](https://huggingface.co/datasets/mteb/scifact), 300 claims look for the abstracts about them among 5,183
+(nDCG@10). `scripts/eval.mjs` runs both:
+
+| Model | Size | Banking77 | SciFact |
+|---|---|---|---|
+| **potion-base-8M at 2 bits (bundled)** | **2.2 MB** | **88.3%** | **50.1** |
+| potion-base-8M at 4 bits | 4.1 MB | 88.3% | 50.4 |
+| potion-base-8M, full precision | 30.5 MB | 88.5% | 50.6 |
+| 0.1's model: 128 of the 256 dimensions at 4 bits | 2.2 MB | 87.3% | 45.9 |
+
+**How the model is packed.** Each value is one of four levels fitted to the values (Lloyd-Max), times a scale for
+its dimension and one for its token. The scale per dimension matters because Model2Vec's dimensions are PCA
+components whose spread falls about fivefold from the first to the last. At 2 bits a single vector is noisy (cosine
+about 0.95 to the full-precision one), but the noise is spread across 256 dimensions and mostly cancels out when
+vectors are compared, so it loses less than keeping half the dimensions at 4 bits. The model stays packed in memory,
+so it takes about 3 MB of it rather than 30 MB, and loads in about 6 ms.
+
+**What it can't do:**
+- Averaging tokens loses word order, so "can I add a shopper" and "can a shopper add me" look alike.
+- It only knows words its training text had. Brand names and jargon ("amex") need other phrasings, such as an
+  array field of the ways people ask.
+- Misspelt words break into word pieces that mean something else, so typos hurt.
+- English only.
+
+## Development
+
+```sh
+npm install
+npm test                       # builds, then runs the tests (Node 21+)
+sh scripts/smoke.sh            # packs the package, installs it in an empty project and uses it from there
+python scripts/make-fixture.py <potion-base-8M folder>   # rebuilds the reference fixture (needs model2vec)
+node scripts/eval.mjs <data folder> <model.bin> ...       # Banking77 and SciFact (see the script for the data)
+```
+
+The bundled model is rebuilt with
+`node dist/cli.js pack <potion-base-8M folder> models/potion-base-8M.bin --bits 2`.
+
+CI (`.github/workflows/ci.yml`) builds every push and pull request on Node 18, 20, 22 and 24, runs the tests (22
+and 24) and the install check (all four).
+
+To release, bump the version and push the tag: `npm version patch && git push --follow-tags`. The release workflow
+checks that the tag matches `package.json`, tests again, publishes to npm with provenance and creates a GitHub
+release. It publishes with npm's trusted publishing (the package's Settings → Trusted publishing: GitHub Actions,
+repository `ShakirasHips/model2vec-js`, workflow `release.yml`, environment `npm`), so no token is stored anywhere.
+
+## Credits and licence
+
+The code is MIT-licensed (`LICENSE`). The bundled model is
+[potion-base-8M](https://huggingface.co/minishlab/potion-base-8M) by MinishLab, MIT, repacked at 2 bits (see
+`models/README.md`). The API follows [MiniSearch](https://github.com/lucaong/minisearch) by Luca Ongaro.
+
+This package reimplements Model2Vec's inference and BERT's tokenizer in TypeScript; it contains no code from
+Model2Vec, BERT or MiniSearch.

package/dist/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/cli.js

@@ -0,0 +1,60 @@
+#!/usr/bin/env node
+import { readFile, writeFile } from 'node:fs/promises';
+import { MiniSemanticSearch, packModel, StaticModel } from './node.js';
+const HELP = `mini-semantic-search: semantic search with Model2Vec static embeddings
+
+  mini-semantic-search search <docs.json> "<query>" --fields title,text [--id id] [--k 5] [--model m.bin]
+      The documents closest in meaning to the query. docs.json is an array of objects.
+  mini-semantic-search embed "<text>" [--model m.bin]
+      The text's vector as JSON.
+  mini-semantic-search pack <model folder> <out.bin> [--bits 2|4|8|32] [--dims N] [--group 64]
+      Packs a Model2Vec model from Hugging Face (model.safetensors + tokenizer.json). --group is for 8 bits.
+
+--model defaults to the bundled potion-base-8M (256 dimensions at 2 bits).`;
+const argv = process.argv.slice(2);
+const flags = new Map();
+const args = [];
+for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a.startsWith('--'))
+        flags.set(a.slice(2), argv[++i]);
+    else
+        args.push(a);
+}
+const flag = (name) => flags.get(name);
+const num = (name, def) => (flags.has(name) ? Number(flag(name)) : def);
+const model = () => StaticModel.load(flag('model'));
+async function main() {
+    const [cmd, ...rest] = args;
+    if (cmd === 'search' && rest.length === 2 && flag('fields')) {
+        const fields = flag('fields').split(',');
+        const idField = flag('id') ?? 'id';
+        const docs = JSON.parse(await readFile(rest[0], 'utf8'));
+        const search = await MiniSemanticSearch.create({ fields, storeFields: fields, idField, model: flag('model') });
+        search.addAll(docs);
+        for (const r of search.search(rest[1]).slice(0, num('k', 5))) {
+            console.log(`${r.score.toFixed(3)}  ${String(r.id)}  ${String(r[fields[0]] ?? '')}`);
+        }
+    }
+    else if (cmd === 'embed' && rest.length === 1) {
+        console.log(JSON.stringify(Array.from((await model()).embed(rest[0]), (x) => Number(x.toFixed(6)))));
+    }
+    else if (cmd === 'pack' && rest.length === 2) {
+        const bits = num('bits', 4);
+        if (bits !== 2 && bits !== 4 && bits !== 8 && bits !== 32)
+            throw new Error('--bits must be 2, 4, 8 or 32');
+        const bytes = await packModel(rest[0], { bits, group: num('group', 64), dims: flags.has('dims') ? num('dims', 0) : undefined });
+        await writeFile(rest[1], bytes);
+        const m = StaticModel.fromBytes(bytes);
+        console.log(`${m.tokens.length} tokens × ${m.dims} dims at ${bits} bits -> ${rest[1]} (${(bytes.length / 1e6).toFixed(2)} MB)`);
+    }
+    else {
+        console.log(HELP);
+        if (cmd && cmd !== 'help' && cmd !== '--help')
+            process.exitCode = 1;
+    }
+}
+main().catch((err) => {
+    console.error(`mini-semantic-search: ${err.message}`);
+    process.exit(1);
+});

package/dist/format.d.ts

@@ -0,0 +1,32 @@
+export interface ModelData {
+    tokens: string[];
+    dims: number;
+    /** tokens.length × dims values, row by row. */
+    vectors: Float32Array;
+}
+/** A version 2 model as stored: value (row, d) = levels[code] × rowScales[row] × colScales[d]. */
+export interface PackedModelData {
+    tokens: string[];
+    dims: number;
+    bits: 2 | 4;
+    levels: Float32Array;
+    colScales: Float32Array;
+    rowScales: Float32Array;
+    /** dims × bits / 8 bytes per row. */
+    codes: Uint8Array;
+}
+export interface EncodeOptions {
+    /** 32, 8, 4 or 2 (default 4). 4 and 2 use a learned table of levels; 8 a scale per group. */
+    bits?: 32 | 8 | 4 | 2;
+    /** At 8 bits, the values sharing one scale (default 64); must divide the number of dimensions. */
+    group?: number;
+}
+export declare function fromF16(h: number): number;
+export declare function toF16(x: number): number;
+/** Reads a model file, leaving a version 2 model packed (as `StaticModel` keeps it). */
+export declare function decodePacked(bytes: Uint8Array): ModelData | PackedModelData;
+/** The full table of a packed model. */
+export declare function unpack({ tokens, dims, bits, levels, colScales, rowScales, codes }: PackedModelData): ModelData;
+/** Reads a model file into its full table of vectors. */
+export declare function decodeModel(bytes: Uint8Array): ModelData;
+export declare function encodeModel({ tokens, dims, vectors }: ModelData, options?: EncodeOptions): Uint8Array;

package/dist/format.js

@@ -0,0 +1,194 @@
+// The packed model file. Little-endian. It starts with seven u32s: magic "FAQM", version, vocabulary size, dimensions,
+// bits, group size (version 1 only, else 0) and the vocabulary's byte length; then the tokens joined by "\n", padded
+// to 4 bytes; then the vectors.
+//
+// Version 2 (2 or 4 bits): each value is a code into a learned table of 2^bits levels, scaled by its column's scale
+// and its row's scale. After the vocabulary come the levels (f32), the column scales (one f32 per dimension), the row
+// scales (one f16 per token, padded to 4 bytes) and the codes, each row in dims × bits / 8 bytes, low bits first.
+// Model2Vec's dimensions are PCA components whose spread falls about fivefold from the first to the last, so a scale
+// per column lets every dimension use all the levels, and a table fitted to the values (Lloyd-Max) puts the levels
+// where the values are.
+//
+// Version 1 (32, 8 or 4 bits), still read: at 32 bits plain f32; at 8 bits each value is an int8, and at 4 bits two
+// values share a byte (low nibble first, each nibble − 8); either way every `group` values share one f16 scale, and
+// all the scales come first, padded to 4 bytes.
+const MAGIC = 0x4d514146;
+export function fromF16(h) {
+    const e = (h >> 10) & 31, m = h & 1023, s = h & 0x8000 ? -1 : 1;
+    if (e === 0)
+        return s * m * 2 ** -24;
+    if (e === 31)
+        return m ? NaN : s * Infinity;
+    return s * (1 + m / 1024) * 2 ** (e - 15);
+}
+export function toF16(x) {
+    const u = new Uint32Array(new Float32Array([x]).buffer)[0];
+    const s = (u >> 16) & 0x8000, e = ((u >> 23) & 255) - 127 + 15;
+    let m = u & 0x7fffff;
+    if (e <= 0)
+        return s | Math.round((m | 0x800000) / 2 ** (14 - e));
+    if (e >= 31)
+        return s | 0x7c00;
+    m = Math.round(m / 8192);
+    return m === 1024 ? s | ((e + 1) << 10) : s | (e << 10) | m;
+}
+const pad4 = (n) => (n + 3) & ~3;
+/** Reads a model file, leaving a version 2 model packed (as `StaticModel` keeps it). */
+export function decodePacked(bytes) {
+    const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    if (bytes.byteLength < 28 || dv.getUint32(0, true) !== MAGIC)
+        throw new Error('not a model2vec-js model file');
+    const [, version, n, dims, bits, group, vocabLen] = Array.from({ length: 7 }, (_, i) => dv.getUint32(i * 4, true));
+    if (version !== 1 && version !== 2)
+        throw new Error(`unsupported model file version ${version}; a newer mini-semantic-search may read it`);
+    let off = 28;
+    const tokens = new TextDecoder().decode(bytes.subarray(off, off + vocabLen)).split('\n');
+    if (tokens.length !== n)
+        throw new Error('model file is corrupt: vocabulary size');
+    off += pad4(vocabLen);
+    const count = n * dims;
+    if (version === 2) {
+        if (bits !== 2 && bits !== 4)
+            throw new Error(`unsupported bits: ${bits}`);
+        const k = 1 << bits, rowBytes = (dims * bits) / 8;
+        if (bytes.byteLength < off + 4 * (k + dims) + pad4(n * 2) + n * rowBytes)
+            throw new Error('model file is corrupt: too short');
+        const f32 = (len) => {
+            const a = new Float32Array(len);
+            for (let i = 0; i < len; i++, off += 4)
+                a[i] = dv.getFloat32(off, true);
+            return a;
+        };
+        const levels = f32(k), colScales = f32(dims);
+        const rowScales = new Float32Array(n);
+        for (let i = 0; i < n; i++)
+            rowScales[i] = fromF16(dv.getUint16(off + i * 2, true));
+        off += pad4(n * 2);
+        return { tokens, dims, bits, levels, colScales, rowScales, codes: bytes.subarray(off, off + n * rowBytes) };
+    }
+    const vectors = new Float32Array(count);
+    if (bits === 32) {
+        // Copied byte for byte: a Node Buffer's `slice` is a view, so its `.buffer` can hold much more than this.
+        new Uint8Array(vectors.buffer).set(bytes.subarray(off, off + count * 4));
+    }
+    else if (bits === 8 || bits === 4) {
+        const groups = count / group;
+        const scales = new Float32Array(groups);
+        for (let g = 0; g < groups; g++)
+            scales[g] = fromF16(dv.getUint16(off + g * 2, true));
+        off += pad4(groups * 2);
+        for (let i = 0; i < count; i++) {
+            const q = bits === 8 ? (bytes[off + i] << 24) >> 24 : ((bytes[off + (i >> 1)] >> ((i & 1) * 4)) & 15) - 8;
+            vectors[i] = q * scales[(i / group) | 0];
+        }
+    }
+    else {
+        throw new Error(`unsupported bits: ${bits}`);
+    }
+    return { tokens, dims, vectors };
+}
+/** The full table of a packed model. */
+export function unpack({ tokens, dims, bits, levels, colScales, rowScales, codes }) {
+    const vectors = new Float32Array(tokens.length * dims), mask = (1 << bits) - 1;
+    for (let i = 0; i < vectors.length; i++) {
+        const bit = i * bits, code = (codes[bit >> 3] >> (bit & 7)) & mask;
+        vectors[i] = levels[code] * rowScales[(i / dims) | 0] * colScales[i % dims];
+    }
+    return { tokens, dims, vectors };
+}
+/** Reads a model file into its full table of vectors. */
+export function decodeModel(bytes) {
+    const data = decodePacked(bytes);
+    return 'vectors' in data ? data : unpack(data);
+}
+/** Lloyd-Max: the `k` levels that minimise the squared error of rounding `values` (sorted) to the nearest one. */
+function fitLevels(values, k, iterations = 30) {
+    const levels = Float32Array.from({ length: k }, (_, i) => values[Math.floor(((i + 0.5) / k) * values.length)]);
+    for (let it = 0; it < iterations; it++) {
+        const sum = new Float64Array(k), count = new Float64Array(k);
+        for (let i = 0, j = 0; i < values.length; i++) {
+            const x = values[i];
+            while (j < k - 1 && Math.abs(x - levels[j + 1]) <= Math.abs(x - levels[j]))
+                j++;
+            sum[j] += x;
+            count[j]++;
+        }
+        for (let j = 0; j < k; j++)
+            if (count[j])
+                levels[j] = sum[j] / count[j];
+    }
+    return levels;
+}
+function encodeBody(n, dims, vectors, { bits = 4, group = 64 }) {
+    if (bits === 32)
+        return new Uint8Array(new Float32Array(vectors).buffer);
+    if (bits === 8) {
+        if (dims % group)
+            throw new Error(`group (${group}) must divide dims (${dims})`);
+        const count = n * dims, groups = count / group, scaleBytes = pad4(groups * 2);
+        const body = new Uint8Array(scaleBytes + count), dv = new DataView(body.buffer);
+        for (let g = 0; g < groups; g++) {
+            let max = 0;
+            for (let i = g * group; i < (g + 1) * group; i++)
+                max = Math.max(max, Math.abs(vectors[i]));
+            const h = toF16(max / 127), scale = fromF16(h) || 1;
+            dv.setUint16(g * 2, h, true);
+            for (let i = g * group; i < (g + 1) * group; i++)
+                dv.setInt8(scaleBytes + i, Math.max(-127, Math.min(127, Math.round(vectors[i] / scale))));
+        }
+        return body;
+    }
+    if (bits !== 4 && bits !== 2)
+        throw new Error(`unsupported bits: ${bits}`);
+    if ((dims * bits) % 8)
+        throw new Error(`dims (${dims}) × bits (${bits}) must be a whole number of bytes`);
+    const k = 1 << bits, rowBytes = (dims * bits) / 8;
+    // Column scales, then row scales over the column-scaled values, as stored (f32, f16).
+    const col = new Float64Array(dims);
+    for (let i = 0; i < vectors.length; i++)
+        col[i % dims] += vectors[i] ** 2;
+    const colScales = Float32Array.from(col, (s) => Math.sqrt(s / n) || 1);
+    const scaled = Float32Array.from(vectors, (x, i) => x / colScales[i % dims]);
+    const rowHalves = new Uint16Array(n), rowScales = new Float32Array(n);
+    for (let r = 0; r < n; r++) {
+        let s = 0;
+        for (let d = 0; d < dims; d++)
+            s += scaled[r * dims + d] ** 2;
+        rowHalves[r] = toF16(Math.sqrt(s / dims));
+        rowScales[r] = fromF16(rowHalves[r]);
+        const div = rowScales[r] || 1;
+        for (let d = 0; d < dims; d++)
+            scaled[r * dims + d] /= div;
+    }
+    const levels = fitLevels(scaled.slice().sort(), k);
+    const bounds = levels.subarray(1).map((l, j) => (levels[j] + l) / 2);
+    const body = new Uint8Array(4 * (k + dims) + pad4(n * 2) + n * rowBytes), dv = new DataView(body.buffer);
+    let off = 0;
+    for (const x of [...levels, ...colScales])
+        dv.setFloat32((off += 4) - 4, x, true);
+    rowHalves.forEach((h, r) => dv.setUint16(off + r * 2, h, true));
+    off += pad4(n * 2);
+    for (let i = 0; i < scaled.length; i++) {
+        let code = 0;
+        while (code < k - 1 && scaled[i] > bounds[code])
+            code++;
+        body[off + ((i * bits) >> 3)] |= code << ((i * bits) & 7);
+    }
+    return body;
+}
+export function encodeModel({ tokens, dims, vectors }, options = {}) {
+    const n = tokens.length, bits = options.bits ?? 4;
+    if (vectors.length !== n * dims)
+        throw new Error('vectors must hold tokens.length × dims values');
+    if (tokens.some((t) => t.includes('\n')))
+        throw new Error('tokens must not contain newlines');
+    const vocab = new TextEncoder().encode(tokens.join('\n'));
+    const body = encodeBody(n, dims, vectors, { ...options, bits });
+    const version = bits === 4 || bits === 2 ? 2 : 1, group = bits === 8 ? options.group ?? 64 : 0;
+    const out = new Uint8Array(28 + pad4(vocab.length) + body.length);
+    const dv = new DataView(out.buffer);
+    [MAGIC, version, n, dims, bits, group, vocab.length].forEach((v, i) => dv.setUint32(i * 4, v, true));
+    out.set(vocab, 28);
+    out.set(body, 28 + pad4(vocab.length));
+    return out;
+}

package/dist/index.d.ts

@@ -0,0 +1,7 @@
+import { MiniSemanticSearch } from './search.js';
+export default MiniSemanticSearch;
+export { MiniSemanticSearch, type Options, type SearchOptions, type SearchResult } from './search.js';
+export { StaticModel, similarity } from './model.js';
+export { normalize, preTokenize, wordPiece } from './tokenizer.js';
+export { decodeModel, encodeModel, type ModelData, type EncodeOptions } from './format.js';
+export { loadBytes, DEFAULT_MODEL, type Source } from './load.js';

package/dist/index.js

@@ -0,0 +1,7 @@
+import { MiniSemanticSearch } from './search.js';
+export default MiniSemanticSearch;
+export { MiniSemanticSearch } from './search.js';
+export { StaticModel, similarity } from './model.js';
+export { normalize, preTokenize, wordPiece } from './tokenizer.js';
+export { decodeModel, encodeModel } from './format.js';
+export { loadBytes, DEFAULT_MODEL } from './load.js';

package/dist/load.d.ts

@@ -0,0 +1,9 @@
+/** Where to load a file from: a URL, a path (Node), or the bytes themselves. */
+export type Source = string | URL | ArrayBuffer | ArrayBufferView;
+type FileReader = (path: string | URL) => Promise<Uint8Array>;
+/** @internal Used by the Node entry point. */
+export declare function setFileReader(reader: FileReader): void;
+/** The bundled model: potion-base-8M, all 256 dimensions packed at 2 bits. */
+export declare const DEFAULT_MODEL: URL;
+export declare function loadBytes(src: Source): Promise<Uint8Array>;
+export {};

package/dist/load.js

@@ -0,0 +1,23 @@
+// Fetching files: URLs everywhere, and local paths in Node (the Node entry point registers a file reader, so the
+// browser build never imports node:fs).
+let readFile = null;
+/** @internal Used by the Node entry point. */
+export function setFileReader(reader) {
+    readFile = reader;
+}
+/** The bundled model: potion-base-8M, all 256 dimensions packed at 2 bits. */
+export const DEFAULT_MODEL = new URL('../models/potion-base-8M.bin', import.meta.url);
+export async function loadBytes(src) {
+    if (src instanceof ArrayBuffer)
+        return new Uint8Array(src);
+    if (ArrayBuffer.isView(src))
+        return new Uint8Array(src.buffer, src.byteOffset, src.byteLength);
+    const url = src instanceof URL ? src : null;
+    if (readFile && (url ? url.protocol === 'file:' : !/^[a-z][a-z0-9+.-]*:\/\//i.test(src))) {
+        return readFile(url ?? src);
+    }
+    const res = await fetch(src);
+    if (!res.ok)
+        throw new Error(`${String(src)}: HTTP ${res.status}`);
+    return new Uint8Array(await res.arrayBuffer());
+}

package/dist/model.d.ts

@@ -0,0 +1,37 @@
+import { type ModelData, type PackedModelData } from './format.js';
+import { type Source } from './load.js';
+/**
+ * A Model2Vec static embedding model: one vector per token, and a sentence's vector is the mean of its tokens'
+ * vectors, normalised to length 1. No network runs, so embedding takes microseconds. Long texts are cut as the
+ * Python library cuts them: to `maxTokens` × the median token length in characters, then to `maxTokens` tokens,
+ * before unknown tokens are left out.
+ *
+ * A packed model (2 or 4 bits) stays packed in memory: about 2 MB for the bundled one instead of 30 MB.
+ */
+export declare class StaticModel {
+    readonly tokens: string[];
+    readonly dims: number;
+    readonly unk: number;
+    /** Tokens read per text; Model2Vec's default is 512. */
+    maxTokens: number;
+    private readonly vocab;
+    private readonly medianTokenLength;
+    private readonly longestPiece;
+    private table;
+    private readonly packed;
+    constructor(data: ModelData | PackedModelData);
+    /** Loads a model file; by default the bundled potion-base-8M (256 dimensions at 2 bits, 2.2 MB). */
+    static load(src?: Source): Promise<StaticModel>;
+    static fromBytes(bytes: Uint8Array): StaticModel;
+    /** The table of token vectors, tokens.length × dims, row by row (unpacked on first use for a packed model). */
+    get vectors(): Float32Array;
+    /** Token ids, as Hugging Face's tokenizer gives them without special tokens (unknown words included as UNK). */
+    tokenize(text: string): number[];
+    /** Whether `token` (e.g. "voucher" or "##ing") is in the vocabulary. */
+    has(token: string): boolean;
+    /** The text's vector: unit length, or all zeros if it has no known tokens. */
+    embed(text: string): Float32Array;
+    embedMany(texts: string[]): Float32Array[];
+}
+/** Dot product: the cosine similarity of two unit vectors, such as `embed` returns (1 = same meaning). */
+export declare function similarity(a: Float32Array, b: Float32Array): number;