lucivy-wasm 0.2.0 → 0.3.1

package/README.md

@@ -1,116 +1,127 @@
-# @lucivy/wasm
+# lucivy-wasm
 
-Fast BM25 full-text search for browsers — WASM build of Lucivy with OPFS persistence. Runs in a Web Worker.
+Fast BM25 full-text search for browsers — WASM build of Lucivy with **threading** (emscripten pthreads), OPFS persistence, and LUCE snapshot support. Runs in a Web Worker.
+
+> **v0.3.0 breaking change**: This package now uses the emscripten build with threading support. The API is now worker-based (async). See below for migration.
 
 ## Install
 
 ```bash
-npm install @lucivy/wasm
+npm install lucivy-wasm
 ```
 
+## Requirements
+
+- **COOP/COEP headers** required for `SharedArrayBuffer` (threading):
+  - `Cross-Origin-Opener-Policy: same-origin`
+  - `Cross-Origin-Embedder-Policy: require-corp`
+- For GitHub Pages or environments without header control, use [coi-serviceworker](https://github.com/nicojuicy/coi-serviceworker).
+
 ## Quick start
 
 ```javascript
-import init, { Index } from '@lucivy/wasm';
+import { Lucivy } from 'lucivy-wasm';
 
-await init();
+const lucivy = new Lucivy(new URL('lucivy-wasm/worker', import.meta.url));
+await lucivy.ready;
 
-const index = new Index('/my_index', JSON.stringify([
+const index = await lucivy.create('/my-index', [
     { name: 'title', type: 'text' },
     { name: 'body', type: 'text' },
-]));
+], 'english');
 
-index.add(1, JSON.stringify({ title: 'Rust Programming', body: 'Systems programming with memory safety' }));
-index.add(2, JSON.stringify({ title: 'Python Guide', body: 'Data science and web development' }));
-index.commit();
+await index.add(1, { title: 'Rust Programming', body: 'Systems programming with memory safety' });
+await index.add(2, { title: 'Python Guide', body: 'Data science and web development' });
+await index.commit();
 
-const results = JSON.parse(index.search(JSON.stringify('programming'), 10, true));
+const results = await index.search('programming');
 for (const r of results) {
-    console.log(r.doc_id, r.score, r.highlights);
+    console.log(r.docId, r.score);
 }
+
+// Cleanup
+lucivy.terminate();
 ```
 
 ## API
 
-### Create / open
+### Lucivy (main class)
 
 ```javascript
+import { Lucivy } from 'lucivy-wasm';
+
+const lucivy = new Lucivy('./path/to/lucivy-worker.js');
+await lucivy.ready;
+
 // Create a new index
-const index = new Index('/my_index', JSON.stringify([
+const index = await lucivy.create('/my-index', [
     { name: 'title', type: 'text' },
-    { name: 'body',  type: 'text' },
-    { name: 'tag',   type: 'keyword' },
-]));
+    { name: 'body', type: 'text' },
+], 'english');
+
+// Open an existing index from OPFS
+const index2 = await lucivy.open('/my-index');
 
-// Open from OPFS files (Map of filename → Uint8Array)
-const index2 = Index.open('/my_index', filesMap);
+// Import from a LUCE snapshot
+const index3 = await lucivy.importSnapshot(snapshotBlob, '/restored');
+
+// Terminate the worker (frees all WASM memory)
+lucivy.terminate();
 ```
 
-### Add / update / delete
+### LucivyIndex
 
 ```javascript
-index.add(1, JSON.stringify({ title: 'Hello', body: 'World' }));
-index.addMany(JSON.stringify([
+// Add / update / delete
+await index.add(1, { title: 'Hello', body: 'World' });
+await index.addMany([
     { docId: 2, title: 'Foo', body: 'Bar' },
     { docId: 3, title: 'Baz', body: 'Qux' },
-]));
-index.update(1, JSON.stringify({ title: 'Updated', body: 'Content' }));
-index.remove(2);
-index.commit();
-```
+]);
+await index.update(1, { title: 'Updated', body: 'Content' });
+await index.remove(2);
+await index.commit();
 
-### Search
-
-```javascript
-// String query (contains_split across all text fields)
-const results = JSON.parse(index.search(JSON.stringify('rust programming'), 10));
+// Search (BM25)
+const results = await index.search('rust programming');
 
-// Structured query
-const results2 = JSON.parse(index.search(JSON.stringify({
-    type: 'contains', field: 'body', value: 'program'
-}), 10, true));
+// Structured query with highlights
+const results2 = await index.search(
+    { type: 'contains', field: 'body', value: 'program' },
+    { highlights: true }
+);
 
 // Pre-filtered by doc IDs
-const results3 = JSON.parse(index.searchFiltered(
-    JSON.stringify('programming'), 10, new Uint32Array([1, 3]), true
-));
-```
-
-### OPFS persistence
-
-```javascript
-// Get dirty files (modified since last export) for incremental OPFS sync
-const dirty = index.exportDirtyFiles(); // Map<string, Uint8Array>
+const results3 = await index.searchFiltered('programming', [1, 3], { highlights: true });
 
-// Get all files
-const all = index.exportAllFiles(); // Map<string, Uint8Array>
+// Metadata
+const count = await index.numDocs();
+const schema = await index.schema();
 ```
 
-### Snapshots (export / import)
+### Snapshots
 
 ```javascript
 // Export to Uint8Array (.luce format)
-const snapshot = index.exportSnapshot();
+const snapshot = await index.exportSnapshot();
 
 // Import from Uint8Array
-const restored = Index.importSnapshot(snapshot, '/restored_index');
+const restored = await lucivy.importSnapshot(snapshot, '/restored');
 ```
 
-### Properties
+### Cleanup
 
 ```javascript
-index.numDocs     // number of documents
-index.path        // index path
-index.schemaJson  // JSON string of field definitions
+await index.close();    // Remove from tracking (OPFS files kept)
+await index.destroy();  // Remove from tracking + delete OPFS files
+lucivy.terminate();     // Kill worker, free all WASM memory
 ```
 
-## OPFS + Web Worker pattern
+> **Note**: Always call `lucivy.terminate()` when done. Individual `close()`/`destroy()` are instant and non-blocking. Actual WASM memory is reclaimed on `terminate()`.
 
-This package is designed to run in a Web Worker with OPFS for persistence:
+## Supported stemmers
 
-1. On startup: read files from OPFS → `Index.open(path, filesMap)`
-2. After mutations + commit: `index.exportDirtyFiles()` → write to OPFS
-3. For backup/transfer: `index.exportSnapshot()` → portable `.luce` blob
+`english`, `french`, `german`, `spanish`, `italian`, `portuguese`, `dutch`, `russian`
 
 ## License
 

package/js/lucivy-worker.js

@@ -0,0 +1,363 @@
+// lucivy-worker.js — Web Worker that runs lucivy-emscripten with OPFS persistence.
+//
+// Usage from main thread:
+//   const worker = new Worker('lucivy-worker.js', { type: 'module' });
+//   worker.postMessage({ type: 'init', id: 1 });
+//
+// Or use lucivy.js for a Promise-based API.
+
+// The emscripten module must be in ../pkg/lucivy.js relative to this worker.
+// We use importScripts-compatible dynamic import.
+let Module = null;
+
+const indexes = new Map(); // path -> ctx pointer
+
+// ── Helpers ──────────────────────────────────────────────────────────────────
+
+function callStr(fn, ...args) {
+    const types = args.map(a => typeof a === 'number' ? 'number' : 'string');
+    const ptr = Module.ccall(fn, 'number', types, args);
+    return Module.UTF8ToString(ptr);
+}
+
+function checkResult(res) {
+    if (res.startsWith('{')) {
+        const parsed = JSON.parse(res);
+        if (parsed.error) throw new Error(parsed.error);
+        return parsed;
+    }
+    if (res !== 'ok') throw new Error(res);
+    return res;
+}
+
+function getCtx(path) {
+    const ctx = indexes.get(path);
+    if (!ctx) throw new Error(`No index open at path: ${path}`);
+    return ctx;
+}
+
+// ── Base64 decode (for file export from Rust) ────────────────────────────────
+
+function base64ToUint8Array(b64) {
+    const bin = atob(b64);
+    const arr = new Uint8Array(bin.length);
+    for (let i = 0; i < bin.length; i++) arr[i] = bin.charCodeAt(i);
+    return arr;
+}
+
+// ── OPFS helpers ─────────────────────────────────────────────────────────────
+
+async function getOpfsDir(path) {
+    const root = await navigator.storage.getDirectory();
+    const parts = path.replace(/^\/+/, '').split('/').filter(Boolean);
+    let dir = root;
+    for (const part of parts) {
+        dir = await dir.getDirectoryHandle(part, { create: true });
+    }
+    return dir;
+}
+
+async function readAllFiles(path) {
+    const files = new Map();
+    try {
+        const dir = await getOpfsDir(path);
+        for await (const [name, handle] of dir) {
+            if (handle.kind === 'file') {
+                const file = await handle.getFile();
+                const buffer = await file.arrayBuffer();
+                files.set(name, new Uint8Array(buffer));
+            }
+        }
+    } catch (e) {
+        // Directory doesn't exist yet — return empty map.
+    }
+    return files;
+}
+
+async function writeFiles(path, modified, deleted) {
+    const dir = await getOpfsDir(path);
+
+    for (const [name, data] of modified) {
+        const fileHandle = await dir.getFileHandle(name, { create: true });
+        const writable = await fileHandle.createWritable();
+        await writable.write(data);
+        await writable.close();
+    }
+
+    for (const name of deleted) {
+        try {
+            await dir.removeEntry(name);
+        } catch (e) {
+            // File may already be gone.
+        }
+    }
+}
+
+async function removeAllFiles(path) {
+    try {
+        const root = await navigator.storage.getDirectory();
+        const parts = path.replace(/^\/+/, '').split('/').filter(Boolean);
+        if (parts.length > 0) {
+            await root.removeEntry(parts[0], { recursive: true });
+        }
+    } catch (e) {
+        // Directory doesn't exist.
+    }
+}
+
+// ── Sync dirty files from emscripten index to OPFS ──────────────────────────
+
+async function syncDirtyToOpfs(path, ctx) {
+    const dirtyJson = callStr('lucivy_export_dirty', ctx);
+    const dirty = JSON.parse(dirtyJson);
+
+    const modified = (dirty.modified || []).map(([name, b64]) => [name, base64ToUint8Array(b64)]);
+    const deleted = dirty.deleted || [];
+
+    if (modified.length > 0 || deleted.length > 0) {
+        await writeFiles(path, modified, deleted);
+    }
+}
+
+// ── Message handler ──────────────────────────────────────────────────────────
+
+self.onmessage = async (e) => {
+    const { type, id, ...args } = e.data;
+
+    try {
+        if (!Module && type !== 'init') {
+            throw new Error('Module not initialized — send {type: "init"} first');
+        }
+
+        let result;
+
+        switch (type) {
+            case 'init': {
+                // Dynamic import of the emscripten module.
+                // The worker URL determines the base for relative imports.
+                const { default: createLucivy } = await import('../pkg/lucivy.js');
+                Module = await createLucivy();
+                result = true;
+                break;
+            }
+
+            case 'create': {
+                const { path, fields, stemmer } = args;
+                const fieldsJson = typeof fields === 'string' ? fields : JSON.stringify(fields);
+                const ctx = Module.ccall('lucivy_create', 'number',
+                    ['string', 'string', 'string'],
+                    [path, fieldsJson, stemmer || '']);
+                if (!ctx) throw new Error('lucivy_create returned null');
+                indexes.set(path, ctx);
+
+                // Export all files to OPFS for initial persistence (best-effort).
+                try {
+                    const allJson = callStr('lucivy_export_all', ctx);
+                    const allFiles = JSON.parse(allJson);
+                    const modified = allFiles.map(([name, b64]) => [name, base64ToUint8Array(b64)]);
+                    await writeFiles(path, modified, []);
+                } catch (e) {
+                    console.warn('[lucivy-worker] OPFS initial sync skipped:', e.message);
+                }
+
+                result = { path, numDocs: Module.ccall('lucivy_num_docs', 'number', ['number'], [ctx]) };
+                break;
+            }
+
+            case 'open': {
+                const { path } = args;
+                const files = await readAllFiles(path);
+                if (files.size === 0) {
+                    throw new Error(`No index found at OPFS path: ${path}`);
+                }
+
+                // Two-phase open: begin → import files → finish.
+                const openCtx = Module.ccall('lucivy_open_begin', 'number', ['string'], [path]);
+                if (!openCtx) throw new Error('lucivy_open_begin returned null');
+
+                for (const [name, data] of files) {
+                    // Allocate memory and copy file data.
+                    const ptr = Module._malloc(data.length);
+                    Module.HEAPU8.set(data, ptr);
+                    // Write file name as UTF8.
+                    const nameBytes = Module.lengthBytesUTF8(name) + 1;
+                    const namePtr = Module._malloc(nameBytes);
+                    Module.stringToUTF8(name, namePtr, nameBytes);
+                    Module.ccall('lucivy_import_file', null,
+                        ['number', 'number', 'number', 'number'],
+                        [openCtx, namePtr, ptr, data.length]);
+                    Module._free(namePtr);
+                    Module._free(ptr);
+                }
+
+                const ctx = Module.ccall('lucivy_open_finish', 'number', ['number'], [openCtx]);
+                if (!ctx) throw new Error('lucivy_open_finish returned null');
+                indexes.set(path, ctx);
+
+                result = { path, numDocs: Module.ccall('lucivy_num_docs', 'number', ['number'], [ctx]) };
+                break;
+            }
+
+            case 'add': {
+                const ctx = getCtx(args.path);
+                const fieldsJson = typeof args.fields === 'string'
+                    ? args.fields : JSON.stringify(args.fields);
+                const res = callStr('lucivy_add', ctx, args.docId, fieldsJson);
+                checkResult(res);
+                result = true;
+                break;
+            }
+
+            case 'addMany': {
+                const ctx = getCtx(args.path);
+                const docsJson = typeof args.docs === 'string'
+                    ? args.docs : JSON.stringify(args.docs);
+                const res = callStr('lucivy_add_many', ctx, docsJson);
+                checkResult(res);
+                result = true;
+                break;
+            }
+
+            case 'remove': {
+                const ctx = getCtx(args.path);
+                const res = callStr('lucivy_remove', ctx, args.docId);
+                checkResult(res);
+                result = true;
+                break;
+            }
+
+            case 'update': {
+                const ctx = getCtx(args.path);
+                const fieldsJson = typeof args.fields === 'string'
+                    ? args.fields : JSON.stringify(args.fields);
+                const res = callStr('lucivy_update', ctx, args.docId, fieldsJson);
+                checkResult(res);
+                result = true;
+                break;
+            }
+
+            case 'commit': {
+                const ctx = getCtx(args.path);
+                const res = callStr('lucivy_commit', ctx);
+                checkResult(res);
+
+                // Sync dirty files to OPFS (best-effort — may not be available in all contexts).
+                try {
+                    await syncDirtyToOpfs(args.path, ctx);
+                } catch (e) {
+                    console.warn('[lucivy-worker] OPFS sync skipped:', e.message);
+                }
+
+                result = { numDocs: Module.ccall('lucivy_num_docs', 'number', ['number'], [ctx]) };
+                break;
+            }
+
+            case 'rollback': {
+                const ctx = getCtx(args.path);
+                const res = callStr('lucivy_rollback', ctx);
+                checkResult(res);
+                result = true;
+                break;
+            }
+
+            case 'search': {
+                const ctx = getCtx(args.path);
+                const queryJson = typeof args.query === 'string' && !args.query.startsWith('{')
+                    ? JSON.stringify(args.query)
+                    : (typeof args.query === 'object' ? JSON.stringify(args.query) : args.query);
+                const json = callStr('lucivy_search', ctx, queryJson, args.limit || 10, args.highlights ? 1 : 0);
+                result = JSON.parse(json);
+                if (result.error) throw new Error(result.error);
+                break;
+            }
+
+            case 'searchFiltered': {
+                const ctx = getCtx(args.path);
+                const queryJson = typeof args.query === 'string' && !args.query.startsWith('{')
+                    ? JSON.stringify(args.query)
+                    : (typeof args.query === 'object' ? JSON.stringify(args.query) : args.query);
+
+                const ids = new Uint32Array(args.allowedIds);
+                const idsPtr = Module._malloc(ids.byteLength);
+                Module.HEAPU8.set(new Uint8Array(ids.buffer), idsPtr);
+
+                const resPtr = Module.ccall('lucivy_search_filtered', 'number',
+                    ['number', 'string', 'number', 'number', 'number', 'number'],
+                    [ctx, queryJson, args.limit || 10, idsPtr, ids.length, args.highlights ? 1 : 0]);
+                const json = Module.UTF8ToString(resPtr);
+                Module._free(idsPtr);
+
+                result = JSON.parse(json);
+                if (result.error) throw new Error(result.error);
+                break;
+            }
+
+            case 'close': {
+                // Remove from tracking. The Rust context is intentionally NOT destroyed
+                // here because lucivy_destroy can deadlock under emscripten pthreads
+                // (IndexWriter drop joins threads via the event loop).
+                // All WASM memory is fully reclaimed when the worker is terminated.
+                indexes.delete(args.path);
+                result = true;
+                break;
+            }
+
+            case 'destroy': {
+                indexes.delete(args.path);
+                // OPFS cleanup is best-effort (may hang if storage API unavailable)
+                removeAllFiles(args.path).catch(() => {});
+                result = true;
+                break;
+            }
+
+            case 'exportSnapshot': {
+                const ctx = getCtx(args.path);
+                const lenPtr = Module._malloc(4);
+                const dataPtr = Module.ccall('lucivy_export_snapshot', 'number',
+                    ['number', 'number'], [ctx, lenPtr]);
+                if (!dataPtr) {
+                    Module._free(lenPtr);
+                    throw new Error('export failed — index may have uncommitted changes');
+                }
+                const len = Module.getValue(lenPtr, 'i32');
+                Module._free(lenPtr);
+                result = Module.HEAPU8.slice(dataPtr, dataPtr + len);
+                break;
+            }
+
+            case 'importSnapshot': {
+                const { data, path } = args;
+                const dataArr = data instanceof Uint8Array ? data : new Uint8Array(data);
+                const ptr = Module._malloc(dataArr.length);
+                Module.HEAPU8.set(dataArr, ptr);
+                const ctx = Module.ccall('lucivy_import_snapshot', 'number',
+                    ['number', 'number', 'string'], [ptr, dataArr.length, path]);
+                Module._free(ptr);
+                if (!ctx) throw new Error('import_snapshot failed — invalid snapshot data');
+                indexes.set(path, ctx);
+                result = { path, numDocs: Module.ccall('lucivy_num_docs', 'number', ['number'], [ctx]) };
+                break;
+            }
+
+            case 'numDocs': {
+                const ctx = getCtx(args.path);
+                result = Module.ccall('lucivy_num_docs', 'number', ['number'], [ctx]);
+                break;
+            }
+
+            case 'schema': {
+                const ctx = getCtx(args.path);
+                const json = callStr('lucivy_schema_json', ctx);
+                result = json ? JSON.parse(json) : null;
+                break;
+            }
+
+            default:
+                throw new Error(`Unknown message type: ${type}`);
+        }
+
+        self.postMessage({ id, result });
+    } catch (err) {
+        self.postMessage({ id, error: err.message || String(err) });
+    }
+};

package/js/lucivy.d.ts

@@ -0,0 +1,92 @@
+/** Field definition for creating an index. */
+export interface FieldDef {
+    name: string;
+    type: 'text' | 'string' | 'u64' | 'i64' | 'f64';
+}
+
+/** Search query — either a plain string or a structured query object. */
+export type SearchQuery = string | {
+    type: 'contains' | 'contains_split' | 'term' | 'boolean';
+    field?: string;
+    value?: string;
+    should?: SearchQuery[];
+    must?: SearchQuery[];
+    must_not?: SearchQuery[];
+};
+
+/** Search options. */
+export interface SearchOptions {
+    limit?: number;
+    highlights?: boolean;
+}
+
+/** A single search result. */
+export interface SearchResult {
+    docId: number;
+    score: number;
+    highlights?: Record<string, [number, number][]>;
+}
+
+/** Main-thread Promise API for lucivy-emscripten. */
+export declare class Lucivy {
+    /** Resolves when the WASM module is loaded and ready. */
+    readonly ready: Promise<boolean>;
+
+    constructor(workerUrl: string);
+
+    /** Create a new index at the given path with the specified fields. */
+    create(path: string, fields: FieldDef[], stemmer?: string): Promise<LucivyIndex>;
+
+    /** Open an existing index from OPFS. */
+    open(path: string): Promise<LucivyIndex>;
+
+    /** Import an index from a LUCE snapshot blob. */
+    importSnapshot(data: Uint8Array, path: string): Promise<LucivyIndex>;
+
+    /** Terminate the worker. */
+    terminate(): void;
+}
+
+/** Handle to an open lucivy index. All operations go through the worker. */
+export declare class LucivyIndex {
+    readonly path: string;
+
+    /** Add a document with the given ID and field values. */
+    add(docId: number, fields: Record<string, string | number>): Promise<boolean>;
+
+    /** Add multiple documents at once. Each doc must have a `docId` key. */
+    addMany(docs: Array<Record<string, string | number> & { docId: number }>): Promise<boolean>;
+
+    /** Remove a document by ID. */
+    remove(docId: number): Promise<boolean>;
+
+    /** Update a document (remove + add). */
+    update(docId: number, fields: Record<string, string | number>): Promise<boolean>;
+
+    /** Commit pending changes and sync to OPFS. */
+    commit(): Promise<{ numDocs: number }>;
+
+    /** Rollback uncommitted changes. */
+    rollback(): Promise<boolean>;
+
+    /** Search the index. */
+    search(query: SearchQuery, options?: SearchOptions): Promise<SearchResult[]>;
+
+    /** Search with an allowed document ID filter. */
+    searchFiltered(query: SearchQuery, allowedIds: number[], options?: SearchOptions): Promise<SearchResult[]>;
+
+    /** Get the number of indexed documents. */
+    numDocs(): Promise<number>;
+
+    /** Get the index schema. */
+    schema(): Promise<FieldDef[] | null>;
+
+    /** Export the index as a LUCE snapshot blob (Uint8Array). */
+    exportSnapshot(): Promise<Uint8Array>;
+
+    /** Close the index (keep OPFS files). */
+    close(): Promise<boolean>;
+
+    /** Close the index and delete OPFS files. */
+    destroy(): Promise<boolean>;
+}