bm25-turbo-wasm 0.1.0

package/README.md

@@ -0,0 +1,138 @@
+# bm25-turbo
+
+The fastest BM25 full-text search engine, compiled to WebAssembly.
+
+BM25 Turbo is a Rust-native BM25 information retrieval engine that supports 5 scoring variants (Robertson, Lucene, ATIRE, BM25L, BM25+), 17-language tokenization with Snowball stemming, and compressed sparse column storage. This package brings the full engine to the browser and Node.js via WebAssembly.
+
+## Installation
+
+```bash
+npm install bm25-turbo
+```
+
+## Quick Start
+
+```javascript
+import init, { WasmBM25 } from 'bm25-turbo';
+
+// Initialize the WASM module (required once before use)
+await init();
+
+// Build an index from an array of documents
+const index = new WasmBM25([
+  "The quick brown fox jumps over the lazy dog",
+  "A fast red car drives on the highway",
+  "The brown dog sleeps in the sun",
+  "Quick foxes are surprisingly lazy animals",
+]);
+
+// Search for the top 2 results
+const results = index.search("quick brown fox", 2);
+console.log(results);
+// { doc_ids: [0, 3], scores: [1.82, 0.94] }
+```
+
+## Usage
+
+### Creating an Index
+
+The `WasmBM25` constructor accepts an array of document strings and optional parameters:
+
+```typescript
+const index = new WasmBM25(
+  documents,   // string[] — array of document texts
+  method?,     // string — scoring variant (default: "lucene")
+  k1?,         // number — term frequency saturation (default: 1.5)
+  b?,          // number — document length normalization (default: 0.75)
+);
+```
+
+**Supported methods:** `"robertson"`, `"lucene"`, `"atire"`, `"bm25l"`, `"bm25plus"`
+
+### Searching
+
+```typescript
+const results = index.search(query, k);
+// Returns: { doc_ids: number[], scores: number[] }
+```
+
+- `query` — search query string
+- `k` — maximum number of results to return (must be > 0)
+
+### Serialization
+
+Save an index to bytes for storage (e.g., IndexedDB, localStorage):
+
+```typescript
+// Serialize
+const bytes = index.to_bytes(); // Uint8Array
+
+// Deserialize
+const restored = WasmBM25.loadBytes(bytes);
+```
+
+### Index Statistics
+
+```typescript
+console.log(index.num_docs());    // number of documents
+console.log(index.vocab_size());  // number of unique terms
+console.log(index.stats());      // JSON string with full stats
+```
+
+## API Reference
+
+### `new WasmBM25(documents, method?, k1?, b?)`
+
+Construct a BM25 index from a corpus of documents.
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `documents` | `string[]` | (required) | Array of document texts to index |
+| `method` | `string` | `"lucene"` | BM25 scoring variant |
+| `k1` | `number` | `1.5` | Term frequency saturation parameter |
+| `b` | `number` | `0.75` | Document length normalization parameter |
+
+### `.search(query, k)`
+
+Search the index and return the top-k results.
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `query` | `string` | Search query text |
+| `k` | `number` | Maximum number of results (must be > 0) |
+
+**Returns:** `{ doc_ids: number[], scores: number[] }`
+
+### `.num_docs()`
+
+Returns the number of documents in the index.
+
+### `.vocab_size()`
+
+Returns the number of unique terms in the vocabulary.
+
+### `.stats()`
+
+Returns a JSON string with index statistics including `num_docs`, `vocab_size`, `method`, `k1`, `b`, `delta`, and `avg_doc_len`.
+
+### `.to_bytes()`
+
+Serializes the index to a `Uint8Array` for storage or transfer.
+
+### `WasmBM25.loadBytes(data)`
+
+Static method. Deserializes an index from bytes produced by `to_bytes()`.
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `data` | `Uint8Array` | Serialized index bytes |
+
+**Returns:** `WasmBM25`
+
+## Browser Support
+
+Requires a browser with WebAssembly support (all modern browsers). The package targets the `web` platform by default, producing ES module output suitable for `<script type="module">` or bundlers.
+
+## License
+
+MIT OR Apache-2.0

package/bm25_turbo_wasm.d.ts

@@ -0,0 +1,61 @@
+/* tslint:disable */
+/* eslint-disable */
+
+/**
+ * BM25 search engine for WebAssembly.
+ *
+ * Provides index construction, search, and serialization capabilities
+ * entirely in WASM linear memory. No filesystem access, no threading.
+ */
+export class WasmBM25 {
+    free(): void;
+    [Symbol.dispose](): void;
+    /**
+     * Load a pre-built index from bytes.
+     *
+     * Accepts bytes produced by `to_bytes()` or by the native Rust library's
+     * bincode serialization of the same format.
+     */
+    static loadBytes(data: Uint8Array): WasmBM25;
+    /**
+     * Build a BM25 index from an array of document strings.
+     *
+     * # Arguments
+     * * `documents` - JavaScript array of strings
+     * * `method` - BM25 variant: "robertson", "lucene", "atire", "bm25l", "bm25plus"
+     * * `k1` - Term frequency saturation parameter (default: 1.5)
+     * * `b` - Document length normalization parameter (default: 0.75)
+     */
+    constructor(documents: string[], method?: string | null, k1?: number | null, b?: number | null);
+    /**
+     * Return the number of documents in the index.
+     */
+    num_docs(): number;
+    /**
+     * Search the index and return the top-k results.
+     *
+     * Returns a JavaScript object with `doc_ids` (Uint32Array) and `scores` (Float32Array).
+     */
+    search(query: string, k: number): any;
+    /**
+     * Return index statistics as a JSON string.
+     */
+    stats(): string;
+    /**
+     * Serialize the index to bytes for storage/transfer.
+     *
+     * The returned bytes can be loaded back via `WasmBM25.load_bytes()`.
+     */
+    to_bytes(): Uint8Array;
+    /**
+     * Return the vocabulary size.
+     */
+    vocab_size(): number;
+}
+
+/**
+ * Called automatically when the WASM module is instantiated.
+ * Sets up `console_error_panic_hook` so that Rust panics produce
+ * readable stack traces in the browser console.
+ */
+export function init(): void;

package/bm25_turbo_wasm.js

@@ -0,0 +1,9 @@
+/* @ts-self-types="./bm25_turbo_wasm.d.ts" */
+
+import * as wasm from "./bm25_turbo_wasm_bg.wasm";
+import { __wbg_set_wasm } from "./bm25_turbo_wasm_bg.js";
+__wbg_set_wasm(wasm);
+wasm.__wbindgen_start();
+export {
+    WasmBM25, init
+} from "./bm25_turbo_wasm_bg.js";

package/bm25_turbo_wasm_bg.js

@@ -0,0 +1,353 @@
+/**
+ * BM25 search engine for WebAssembly.
+ *
+ * Provides index construction, search, and serialization capabilities
+ * entirely in WASM linear memory. No filesystem access, no threading.
+ */
+export class WasmBM25 {
+    static __wrap(ptr) {
+        ptr = ptr >>> 0;
+        const obj = Object.create(WasmBM25.prototype);
+        obj.__wbg_ptr = ptr;
+        WasmBM25Finalization.register(obj, obj.__wbg_ptr, obj);
+        return obj;
+    }
+    __destroy_into_raw() {
+        const ptr = this.__wbg_ptr;
+        this.__wbg_ptr = 0;
+        WasmBM25Finalization.unregister(this);
+        return ptr;
+    }
+    free() {
+        const ptr = this.__destroy_into_raw();
+        wasm.__wbg_wasmbm25_free(ptr, 0);
+    }
+    /**
+     * Load a pre-built index from bytes.
+     *
+     * Accepts bytes produced by `to_bytes()` or by the native Rust library's
+     * bincode serialization of the same format.
+     * @param {Uint8Array} data
+     * @returns {WasmBM25}
+     */
+    static loadBytes(data) {
+        const ptr0 = passArray8ToWasm0(data, wasm.__wbindgen_malloc);
+        const len0 = WASM_VECTOR_LEN;
+        const ret = wasm.wasmbm25_loadBytes(ptr0, len0);
+        if (ret[2]) {
+            throw takeFromExternrefTable0(ret[1]);
+        }
+        return WasmBM25.__wrap(ret[0]);
+    }
+    /**
+     * Build a BM25 index from an array of document strings.
+     *
+     * # Arguments
+     * * `documents` - JavaScript array of strings
+     * * `method` - BM25 variant: "robertson", "lucene", "atire", "bm25l", "bm25plus"
+     * * `k1` - Term frequency saturation parameter (default: 1.5)
+     * * `b` - Document length normalization parameter (default: 0.75)
+     * @param {string[]} documents
+     * @param {string | null} [method]
+     * @param {number | null} [k1]
+     * @param {number | null} [b]
+     */
+    constructor(documents, method, k1, b) {
+        const ptr0 = passArrayJsValueToWasm0(documents, wasm.__wbindgen_malloc);
+        const len0 = WASM_VECTOR_LEN;
+        var ptr1 = isLikeNone(method) ? 0 : passStringToWasm0(method, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        var len1 = WASM_VECTOR_LEN;
+        const ret = wasm.wasmbm25_new(ptr0, len0, ptr1, len1, isLikeNone(k1) ? 0x100000001 : Math.fround(k1), isLikeNone(b) ? 0x100000001 : Math.fround(b));
+        if (ret[2]) {
+            throw takeFromExternrefTable0(ret[1]);
+        }
+        this.__wbg_ptr = ret[0] >>> 0;
+        WasmBM25Finalization.register(this, this.__wbg_ptr, this);
+        return this;
+    }
+    /**
+     * Return the number of documents in the index.
+     * @returns {number}
+     */
+    num_docs() {
+        const ret = wasm.wasmbm25_num_docs(this.__wbg_ptr);
+        return ret >>> 0;
+    }
+    /**
+     * Search the index and return the top-k results.
+     *
+     * Returns a JavaScript object with `doc_ids` (Uint32Array) and `scores` (Float32Array).
+     * @param {string} query
+     * @param {number} k
+     * @returns {any}
+     */
+    search(query, k) {
+        const ptr0 = passStringToWasm0(query, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        const len0 = WASM_VECTOR_LEN;
+        const ret = wasm.wasmbm25_search(this.__wbg_ptr, ptr0, len0, k);
+        if (ret[2]) {
+            throw takeFromExternrefTable0(ret[1]);
+        }
+        return takeFromExternrefTable0(ret[0]);
+    }
+    /**
+     * Return index statistics as a JSON string.
+     * @returns {string}
+     */
+    stats() {
+        let deferred1_0;
+        let deferred1_1;
+        try {
+            const ret = wasm.wasmbm25_stats(this.__wbg_ptr);
+            deferred1_0 = ret[0];
+            deferred1_1 = ret[1];
+            return getStringFromWasm0(ret[0], ret[1]);
+        } finally {
+            wasm.__wbindgen_free(deferred1_0, deferred1_1, 1);
+        }
+    }
+    /**
+     * Serialize the index to bytes for storage/transfer.
+     *
+     * The returned bytes can be loaded back via `WasmBM25.load_bytes()`.
+     * @returns {Uint8Array}
+     */
+    to_bytes() {
+        const ret = wasm.wasmbm25_to_bytes(this.__wbg_ptr);
+        if (ret[3]) {
+            throw takeFromExternrefTable0(ret[2]);
+        }
+        var v1 = getArrayU8FromWasm0(ret[0], ret[1]).slice();
+        wasm.__wbindgen_free(ret[0], ret[1] * 1, 1);
+        return v1;
+    }
+    /**
+     * Return the vocabulary size.
+     * @returns {number}
+     */
+    vocab_size() {
+        const ret = wasm.wasmbm25_vocab_size(this.__wbg_ptr);
+        return ret >>> 0;
+    }
+}
+if (Symbol.dispose) WasmBM25.prototype[Symbol.dispose] = WasmBM25.prototype.free;
+
+/**
+ * Called automatically when the WASM module is instantiated.
+ * Sets up `console_error_panic_hook` so that Rust panics produce
+ * readable stack traces in the browser console.
+ */
+export function init() {
+    wasm.init();
+}
+export function __wbg_Error_83742b46f01ce22d(arg0, arg1) {
+    const ret = Error(getStringFromWasm0(arg0, arg1));
+    return ret;
+}
+export function __wbg_String_8564e559799eccda(arg0, arg1) {
+    const ret = String(arg1);
+    const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+    const len1 = WASM_VECTOR_LEN;
+    getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true);
+    getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true);
+}
+export function __wbg___wbindgen_string_get_395e606bd0ee4427(arg0, arg1) {
+    const obj = arg1;
+    const ret = typeof(obj) === 'string' ? obj : undefined;
+    var ptr1 = isLikeNone(ret) ? 0 : passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+    var len1 = WASM_VECTOR_LEN;
+    getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true);
+    getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true);
+}
+export function __wbg___wbindgen_throw_6ddd609b62940d55(arg0, arg1) {
+    throw new Error(getStringFromWasm0(arg0, arg1));
+}
+export function __wbg_error_a6fa202b58aa1cd3(arg0, arg1) {
+    let deferred0_0;
+    let deferred0_1;
+    try {
+        deferred0_0 = arg0;
+        deferred0_1 = arg1;
+        console.error(getStringFromWasm0(arg0, arg1));
+    } finally {
+        wasm.__wbindgen_free(deferred0_0, deferred0_1, 1);
+    }
+}
+export function __wbg_new_227d7c05414eb861() {
+    const ret = new Error();
+    return ret;
+}
+export function __wbg_new_a70fbab9066b301f() {
+    const ret = new Array();
+    return ret;
+}
+export function __wbg_new_ab79df5bd7c26067() {
+    const ret = new Object();
+    return ret;
+}
+export function __wbg_set_282384002438957f(arg0, arg1, arg2) {
+    arg0[arg1 >>> 0] = arg2;
+}
+export function __wbg_set_6be42768c690e380(arg0, arg1, arg2) {
+    arg0[arg1] = arg2;
+}
+export function __wbg_stack_3b0d974bbf31e44f(arg0, arg1) {
+    const ret = arg1.stack;
+    const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+    const len1 = WASM_VECTOR_LEN;
+    getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true);
+    getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true);
+}
+export function __wbindgen_cast_0000000000000001(arg0) {
+    // Cast intrinsic for `F64 -> Externref`.
+    const ret = arg0;
+    return ret;
+}
+export function __wbindgen_cast_0000000000000002(arg0, arg1) {
+    // Cast intrinsic for `Ref(String) -> Externref`.
+    const ret = getStringFromWasm0(arg0, arg1);
+    return ret;
+}
+export function __wbindgen_init_externref_table() {
+    const table = wasm.__wbindgen_externrefs;
+    const offset = table.grow(4);
+    table.set(0, undefined);
+    table.set(offset + 0, undefined);
+    table.set(offset + 1, null);
+    table.set(offset + 2, true);
+    table.set(offset + 3, false);
+}
+const WasmBM25Finalization = (typeof FinalizationRegistry === 'undefined')
+    ? { register: () => {}, unregister: () => {} }
+    : new FinalizationRegistry(ptr => wasm.__wbg_wasmbm25_free(ptr >>> 0, 1));
+
+function addToExternrefTable0(obj) {
+    const idx = wasm.__externref_table_alloc();
+    wasm.__wbindgen_externrefs.set(idx, obj);
+    return idx;
+}
+
+function getArrayU8FromWasm0(ptr, len) {
+    ptr = ptr >>> 0;
+    return getUint8ArrayMemory0().subarray(ptr / 1, ptr / 1 + len);
+}
+
+let cachedDataViewMemory0 = null;
+function getDataViewMemory0() {
+    if (cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)) {
+        cachedDataViewMemory0 = new DataView(wasm.memory.buffer);
+    }
+    return cachedDataViewMemory0;
+}
+
+function getStringFromWasm0(ptr, len) {
+    ptr = ptr >>> 0;
+    return decodeText(ptr, len);
+}
+
+let cachedUint8ArrayMemory0 = null;
+function getUint8ArrayMemory0() {
+    if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) {
+        cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer);
+    }
+    return cachedUint8ArrayMemory0;
+}
+
+function isLikeNone(x) {
+    return x === undefined || x === null;
+}
+
+function passArray8ToWasm0(arg, malloc) {
+    const ptr = malloc(arg.length * 1, 1) >>> 0;
+    getUint8ArrayMemory0().set(arg, ptr / 1);
+    WASM_VECTOR_LEN = arg.length;
+    return ptr;
+}
+
+function passArrayJsValueToWasm0(array, malloc) {
+    const ptr = malloc(array.length * 4, 4) >>> 0;
+    for (let i = 0; i < array.length; i++) {
+        const add = addToExternrefTable0(array[i]);
+        getDataViewMemory0().setUint32(ptr + 4 * i, add, true);
+    }
+    WASM_VECTOR_LEN = array.length;
+    return ptr;
+}
+
+function passStringToWasm0(arg, malloc, realloc) {
+    if (realloc === undefined) {
+        const buf = cachedTextEncoder.encode(arg);
+        const ptr = malloc(buf.length, 1) >>> 0;
+        getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf);
+        WASM_VECTOR_LEN = buf.length;
+        return ptr;
+    }
+
+    let len = arg.length;
+    let ptr = malloc(len, 1) >>> 0;
+
+    const mem = getUint8ArrayMemory0();
+
+    let offset = 0;
+
+    for (; offset < len; offset++) {
+        const code = arg.charCodeAt(offset);
+        if (code > 0x7F) break;
+        mem[ptr + offset] = code;
+    }
+    if (offset !== len) {
+        if (offset !== 0) {
+            arg = arg.slice(offset);
+        }
+        ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0;
+        const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len);
+        const ret = cachedTextEncoder.encodeInto(arg, view);
+
+        offset += ret.written;
+        ptr = realloc(ptr, len, offset, 1) >>> 0;
+    }
+
+    WASM_VECTOR_LEN = offset;
+    return ptr;
+}
+
+function takeFromExternrefTable0(idx) {
+    const value = wasm.__wbindgen_externrefs.get(idx);
+    wasm.__externref_table_dealloc(idx);
+    return value;
+}
+
+let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
+cachedTextDecoder.decode();
+const MAX_SAFARI_DECODE_BYTES = 2146435072;
+let numBytesDecoded = 0;
+function decodeText(ptr, len) {
+    numBytesDecoded += len;
+    if (numBytesDecoded >= MAX_SAFARI_DECODE_BYTES) {
+        cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
+        cachedTextDecoder.decode();
+        numBytesDecoded = len;
+    }
+    return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len));
+}
+
+const cachedTextEncoder = new TextEncoder();
+
+if (!('encodeInto' in cachedTextEncoder)) {
+    cachedTextEncoder.encodeInto = function (arg, view) {
+        const buf = cachedTextEncoder.encode(arg);
+        view.set(buf);
+        return {
+            read: arg.length,
+            written: buf.length
+        };
+    };
+}
+
+let WASM_VECTOR_LEN = 0;
+
+
+let wasm;
+export function __wbg_set_wasm(val) {
+    wasm = val;
+}

package/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "bm25-turbo-wasm",
+  "type": "module",
+  "description": "BM25 Turbo compiled to WebAssembly — the fastest BM25 search engine for the browser",
+  "version": "0.1.0",
+  "license": "MIT OR Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/TheSauceSuite/BM25-Turbo-Rust-Python-WASM-CLI-"
+  },
+  "files": [
+    "bm25_turbo_wasm_bg.wasm",
+    "bm25_turbo_wasm.js",
+    "bm25_turbo_wasm_bg.js",
+    "bm25_turbo_wasm.d.ts"
+  ],
+  "main": "bm25_turbo_wasm.js",
+  "types": "bm25_turbo_wasm.d.ts",
+  "sideEffects": [
+    "./bm25_turbo_wasm.js",
+    "./snippets/*"
+  ]
+}