lucivy-wasm 3.0.6 → 3.0.8

package/README.md

@@ -1,4 +1,4 @@
-# lucivy-wasm 3.0.6
+# lucivy-wasm 3.0.8
 
 Fast BM25 full-text search for browsers — WASM build with **threading** (emscripten pthreads), OPFS persistence, and snapshot/delta sync support. Runs in a Web Worker.
 

package/js/lucivy-worker.js

@@ -586,6 +586,62 @@ self.onmessage = async (e) => {
                 break;
             }
 
+            // ── Incremental sync (LUCIDS) ───────────────────────────────
+            // The three entry points below were compiled into the wasm and
+            // listed in EXPORTED_FUNCTIONS from the start, but nothing above
+            // them called into them, so a browser client could only ever take
+            // a whole snapshot. Syncing a growing server index to a browser is
+            // the case this exists for, and a full snapshot each time is what
+            // makes it impractical.
+
+            case 'shardVersions': {
+                // What this client already holds, per shard — hand it to the
+                // server so it can send back only what moved.
+                const ctx = getCtx(args.path);
+                const json = await callStr('lucivy_shard_versions', ctx);
+                const parsed = json ? JSON.parse(json) : [];
+                // The C side reports failure as {"error": "..."}, which is a
+                // perfectly valid parse — returning it as data would hand the
+                // caller an object where it expects a list.
+                if (!Array.isArray(parsed)) throw new Error(parsed.error || 'shard_versions failed');
+                result = parsed;
+                break;
+            }
+
+            case 'exportShardedDelta': {
+                const ctx = getCtx(args.path);
+                const versions = JSON.stringify(args.clientVersions || []);
+                const lenPtr = Module._malloc(4);
+                const dataPtr = await Module.ccall('lucivy_export_sharded_delta', 'number',
+                    ['number', 'string', 'number'], [ctx, versions, lenPtr], { async: true });
+                if (!dataPtr) {
+                    Module._free(lenPtr);
+                    throw new Error('export_sharded_delta failed — check the client versions shape');
+                }
+                const len = Module.getValue(lenPtr, 'i32');
+                Module._free(lenPtr);
+                // slice(), not subarray(): the wasm heap moves when it grows,
+                // and a view into it would silently point elsewhere afterwards.
+                result = Module.HEAPU8.slice(dataPtr, dataPtr + len);
+                break;
+            }
+
+            case 'applyShardedDelta': {
+                const ctx = getCtx(args.path);
+                const data = args.data instanceof Uint8Array ? args.data : new Uint8Array(args.data);
+                const ptr = Module._malloc(data.length);
+                Module.HEAPU8.set(data, ptr);
+                let res;
+                try {
+                    res = await callStr('lucivy_apply_sharded_delta', ctx, ptr, data.length);
+                } finally {
+                    Module._free(ptr);
+                }
+                checkResult(res);
+                result = true;
+                break;
+            }
+
             case 'numDocs': {
                 const ctx = getCtx(args.path);
                 result = await Module.ccall('lucivy_num_docs', 'number', ['number'], [ctx], { async: true });

package/js/lucivy.d.ts

@@ -374,6 +374,20 @@ export class LucivyIndex {
   numDocs(): Promise<number>;
   schema(): Promise<IndexConfig>;
   exportSnapshot(): Promise<Uint8Array>;
+  /** What this index holds per shard — hand it to whoever exports the delta. */
+  shardVersions(): Promise<ShardVersion[]>;
+  /** A LUCIDS delta holding only the shards that moved since `clientVersions`. */
+  exportShardedDelta(clientVersions?: ShardVersion[]): Promise<Uint8Array>;
+  /** Apply a LUCIDS delta onto this index, in place. */
+  applyShardedDelta(data: Uint8Array | ArrayBuffer): Promise<boolean>;
   close(): Promise<void>;
   destroy(): Promise<void>;
 }
+
+/** One shard's identity and the version of it a client holds. */
+export interface ShardVersion {
+  shard_id: number;
+  version: string;
+  /** The segments that version is made of — what the exporter diffs against. */
+  segment_ids: string[];
+}

package/js/lucivy.js

@@ -229,6 +229,44 @@ export class LucivyIndex {
         return this._lucivy._call('exportSnapshot', { path: this.path });
     }
 
+    /**
+     * What this index already holds, one entry per shard.
+     *
+     * Send it to whoever exports the delta: it is what lets them return only
+     * the shards that moved instead of the whole index.
+     *
+     * @returns {Promise<Array<{shard: number, version: string}>>}
+     */
+    shardVersions() {
+        return this._lucivy._call('shardVersions', { path: this.path });
+    }
+
+    /**
+     * A LUCIDS delta holding only what changed since `clientVersions`.
+     *
+     * @param {Array} clientVersions what the other side holds, from its own
+     *        `shardVersions()`. An empty array asks for everything.
+     * @returns {Promise<Uint8Array>}
+     */
+    exportShardedDelta(clientVersions = []) {
+        return this._lucivy._call('exportShardedDelta', { path: this.path, clientVersions });
+    }
+
+    /**
+     * Apply a LUCIDS delta onto this index, in place.
+     *
+     * The counterpart of `exportShardedDelta`: after it resolves, this index
+     * holds what the exporting side held when it produced the delta. Applying
+     * a delta built against different versions than this index actually has is
+     * rejected rather than half-applied.
+     *
+     * @param {Uint8Array|ArrayBuffer} data the delta blob
+     * @returns {Promise<boolean>}
+     */
+    applyShardedDelta(data) {
+        return this._lucivy._call('applyShardedDelta', { path: this.path, data });
+    }
+
     close() {
         return this._lucivy._call('close', { path: this.path });
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lucivy-wasm",
-  "version": "3.0.6",
+  "version": "3.0.8",
   "type": "module",
   "description": "Fast BM25 full-text search for browsers — WASM build of Lucivy with threading, OPFS persistence, and LUCE snapshot support",
   "license": "MIT",