hermes-wasm 1.8.29 → 1.8.30

package/README.md

@@ -5,7 +5,7 @@ WebAssembly bindings for the [Hermes](https://github.com/SpaceFrontiers/hermes)
 ## Features
 
 - **Local indexing** — create indexes, add documents, commit, and search entirely in WASM
-- **IndexedDB persistence** — indexes survive page refreshes via per-file IDB records
+- **Pluggable persistence** — bring your own storage (IDB, encrypted, OPFS) via simple JS interface
 - **Remote search** — load pre-built indexes over HTTP with slice caching
 - **IPFS support** — load indexes from IPFS via JavaScript fetch callbacks
 - **15+ language stemmers** — English, German, French, Spanish, Russian, Arabic, and more
@@ -59,21 +59,53 @@ const doc = await index.getDocument(
 // { title: "Rust Programming", body: "Rust is a systems language.", views: 1500 }
 ```
 
-### Persistent Index (IndexedDB)
+### Persistent Index (Custom Storage)
+
+Bring your own storage backend — IndexedDB, encrypted storage, OPFS, or anything async:
+
+```ts
+// Your storage must implement this interface:
+interface IFilesStorage {
+  write(name: string, buffer: ArrayBuffer): Promise<void>;
+  get(name: string): Promise<ArrayBuffer | null>;
+  delete(names: string[]): Promise<void>;
+  list(): Promise<string[]>;
+}
+```
 
 ```js
-// Create — auto-saves changed files to IDB on each commit
-const index = await LocalIndex.createPersistent("my-index", schema);
+// Create with custom storage — auto-saves changed files on each commit
+const index = await LocalIndex.withStorage(myStorage, schema);
 await index.addDocuments(docs);
-await index.commit(); // only new segment files written to IDB
+await index.commit(); // only new segment files written to storage
 
-// Later, on page reload:
-const index = await LocalIndex.open("my-index");
+// Later (page reload) — same call reopens if storage has files
+const index = await LocalIndex.withStorage(myStorage, schema);
 const results = await index.search("rust", 10); // works immediately
+```
+
+A simple IndexedDB implementation:
+
+```js
+class IdbStorage {
+  constructor(name) {
+    this.prefix = `idx:${name}:`;
+  }
+  async write(name, buffer) {
+    /* idb put this.prefix + name → buffer */
+  }
+  async get(name) {
+    /* idb get this.prefix + name */
+  }
+  async delete(names) {
+    /* idb delete each this.prefix + name */
+  }
+  async list() {
+    /* idb getAllKeys matching this.prefix, strip prefix */
+  }
+}
 
-// Management
-await LocalIndex.exists("my-index"); // true
-await LocalIndex.deleteIndex("my-index");
+const index = await LocalIndex.withStorage(new IdbStorage("articles"), schema);
 ```
 
 ### Remote Index (HTTP)
@@ -99,22 +131,19 @@ const doc = await index.get_document(
 
 ### `LocalIndex`
 
-| Method                                     | Description                             |
-| ------------------------------------------ | --------------------------------------- |
-| `LocalIndex.create(sdl)`                   | Create in-memory index from SDL schema  |
-| `LocalIndex.createPersistent(name, sdl)`   | Create IndexedDB-backed index           |
-| `LocalIndex.open(name)`                    | Open existing persistent index from IDB |
-| `LocalIndex.deleteIndex(name)`             | Delete persistent index from IDB        |
-| `LocalIndex.exists(name)`                  | Check if persistent index exists in IDB |
-| `index.addDocument(json)`                  | Add a single document                   |
-| `index.addDocuments(jsonArray)`            | Add multiple documents, returns count   |
-| `index.commit()`                           | Commit pending docs (builds segments)   |
-| `index.search(query, limit)`               | Search with BM25 ranking                |
-| `index.searchOffset(query, limit, offset)` | Search with pagination                  |
-| `index.getDocument(segmentId, docId)`      | Retrieve stored document                |
-| `index.numDocs()`                          | Count of committed documents            |
-| `index.pendingDocs()`                      | Count of uncommitted documents          |
-| `index.fieldNames()`                       | List of field names                     |
+| Method                                     | Description                                        |
+| ------------------------------------------ | -------------------------------------------------- |
+| `LocalIndex.create(sdl)`                   | Create in-memory index from SDL schema             |
+| `LocalIndex.withStorage(storage, sdl)`     | Create or open index with pluggable storage        |
+| `index.addDocument(json)`                  | Add a single document                              |
+| `index.addDocuments(jsonArray)`            | Add multiple documents, returns count              |
+| `index.commit()`                           | Commit pending docs, sync to storage if configured |
+| `index.search(query, limit)`               | Search with BM25 ranking                           |
+| `index.searchOffset(query, limit, offset)` | Search with pagination                             |
+| `index.getDocument(segmentId, docId)`      | Retrieve stored document                           |
+| `index.numDocs()`                          | Count of committed documents                       |
+| `index.pendingDocs()`                      | Count of uncommitted documents                     |
+| `index.fieldNames()`                       | List of field names                                |
 
 ### `RemoteIndex`
 
@@ -223,7 +252,8 @@ Browser JS
     ├── LocalIndex (create/index/search in WASM)
     │       └── WasmIndexWriter → SegmentBuilder → RamDirectory
     │                                                   │
-    │                                         [per-file IndexedDB records]
+    │                                         [pluggable IFilesStorage]
+    │                                         (IDB, encrypted, OPFS, ...)
     │
     ├── RemoteIndex (HTTP range requests)
     │       └── Searcher → SliceCachingDirectory → HttpDirectory

package/hermes_wasm.d.ts

@@ -133,12 +133,10 @@ export class IpfsIndex {
  *
  * ```js
  * // In-memory (ephemeral)
- * const index = await LocalIndex.create("index articles { field title: text<en_stem> [indexed, stored] }");
+ * const index = await LocalIndex.create("index articles { ... }");
  *
- * // Persistent (IndexedDB-backed)
- * const index = await LocalIndex.createPersistent("my-index", "index articles { ... }");
- * // ... later, on page reload:
- * const index = await LocalIndex.open("my-index");
+ * // With pluggable storage (IDB, encrypted, remote, etc.)
+ * const index = await LocalIndex.withStorage(myStorage, "index articles { ... }");
  * ```
  */
 export class LocalIndex {
@@ -156,31 +154,15 @@ export class LocalIndex {
     /**
      * Commit pending documents — builds segments and updates metadata.
      *
-     * For persistent indexes, only writes new/changed files to IndexedDB.
+     * For persistent indexes, only writes new/changed files to storage.
      */
     commit(): Promise<boolean>;
     /**
      * Create a new in-memory index from an SDL schema string.
      *
-     * Data is lost on page refresh. Use `createPersistent()` for IndexedDB-backed storage.
+     * Data is lost on page refresh. Use `withStorage()` for persistence.
      */
     static create(schema_sdl: string): Promise<LocalIndex>;
-    /**
-     * Create a new persistent index backed by IndexedDB.
-     *
-     * Each file in the index gets its own IDB record — commits only write
-     * new/changed files, not the entire index.
-     * Use `LocalIndex.open(name)` to reopen it after page refresh.
-     */
-    static createPersistent(name: string, schema_sdl: string): Promise<LocalIndex>;
-    /**
-     * Delete a persistent index from IndexedDB.
-     */
-    static deleteIndex(name: string): Promise<void>;
-    /**
-     * Check if a persistent index exists in IndexedDB.
-     */
-    static exists(name: string): Promise<boolean>;
     /**
      * Get field names from the schema.
      */
@@ -193,12 +175,6 @@ export class LocalIndex {
      * Number of indexed documents (across all committed segments).
      */
     numDocs(): number;
-    /**
-     * Open an existing persistent index from IndexedDB.
-     *
-     * Loads the file manifest and all index files into a RamDirectory.
-     */
-    static open(name: string): Promise<LocalIndex>;
     /**
      * Number of documents pending (not yet committed).
      */
@@ -213,6 +189,23 @@ export class LocalIndex {
      * Search with offset for pagination.
      */
     searchOffset(query_str: string, limit: number, offset: number): Promise<any>;
+    /**
+     * Create or open an index with a pluggable storage backend.
+     *
+     * If the storage already contains index files, the index is reopened.
+     * Otherwise a new index is created from the SDL schema.
+     *
+     * The storage object must implement:
+     * ```ts
+     * interface IFilesStorage {
+     *     write(name: string, buffer: ArrayBuffer): Promise<void>;
+     *     get(name: string): Promise<ArrayBuffer | null>;
+     *     delete(names: string[]): Promise<void>;
+     *     list(): Promise<string[]>;
+     * }
+     * ```
+     */
+    static withStorage(storage: any, schema_sdl: string): Promise<LocalIndex>;
 }
 
 /**
@@ -382,8 +375,6 @@ export interface InitOutput {
     readonly remoteindex_search: (a: number, b: number, c: number, d: number) => any;
     readonly remoteindex_search_offset: (a: number, b: number, c: number, d: number, e: number) => any;
     readonly remoteindex_with_cache_size: (a: number, b: number, c: number) => number;
-    readonly init: () => void;
-    readonly setup_logging: () => void;
     readonly remoteindex_num_docs: (a: number) => number;
     readonly remoteindex_num_segments: (a: number) => number;
     readonly __wbg_localindex_free: (a: number, b: number) => void;
@@ -391,16 +382,15 @@ export interface InitOutput {
     readonly localindex_addDocuments: (a: number, b: any) => any;
     readonly localindex_commit: (a: number) => any;
     readonly localindex_create: (a: number, b: number) => any;
-    readonly localindex_createPersistent: (a: number, b: number, c: number, d: number) => any;
-    readonly localindex_deleteIndex: (a: number, b: number) => any;
-    readonly localindex_exists: (a: number, b: number) => any;
     readonly localindex_fieldNames: (a: number) => any;
     readonly localindex_getDocument: (a: number, b: number, c: number, d: number) => any;
     readonly localindex_numDocs: (a: number) => number;
-    readonly localindex_open: (a: number, b: number) => any;
     readonly localindex_pendingDocs: (a: number) => number;
     readonly localindex_search: (a: number, b: number, c: number, d: number) => any;
     readonly localindex_searchOffset: (a: number, b: number, c: number, d: number, e: number) => any;
+    readonly localindex_withStorage: (a: any, b: number, c: number) => any;
+    readonly init: () => void;
+    readonly setup_logging: () => void;
     readonly rust_zstd_wasm_shim_calloc: (a: number, b: number) => number;
     readonly rust_zstd_wasm_shim_free: (a: number) => void;
     readonly rust_zstd_wasm_shim_malloc: (a: number) => number;

package/hermes_wasm.js

@@ -300,12 +300,10 @@ if (Symbol.dispose) IpfsIndex.prototype[Symbol.dispose] = IpfsIndex.prototype.fr
  *
  * ```js
  * // In-memory (ephemeral)
- * const index = await LocalIndex.create("index articles { field title: text<en_stem> [indexed, stored] }");
+ * const index = await LocalIndex.create("index articles { ... }");
  *
- * // Persistent (IndexedDB-backed)
- * const index = await LocalIndex.createPersistent("my-index", "index articles { ... }");
- * // ... later, on page reload:
- * const index = await LocalIndex.open("my-index");
+ * // With pluggable storage (IDB, encrypted, remote, etc.)
+ * const index = await LocalIndex.withStorage(myStorage, "index articles { ... }");
  * ```
  */
 export class LocalIndex {
@@ -347,7 +345,7 @@ export class LocalIndex {
     /**
      * Commit pending documents — builds segments and updates metadata.
      *
-     * For persistent indexes, only writes new/changed files to IndexedDB.
+     * For persistent indexes, only writes new/changed files to storage.
      * @returns {Promise<boolean>}
      */
     commit() {
@@ -357,7 +355,7 @@ export class LocalIndex {
     /**
      * Create a new in-memory index from an SDL schema string.
      *
-     * Data is lost on page refresh. Use `createPersistent()` for IndexedDB-backed storage.
+     * Data is lost on page refresh. Use `withStorage()` for persistence.
      * @param {string} schema_sdl
      * @returns {Promise<LocalIndex>}
      */
@@ -367,46 +365,6 @@ export class LocalIndex {
         const ret = wasm.localindex_create(ptr0, len0);
         return ret;
     }
-    /**
-     * Create a new persistent index backed by IndexedDB.
-     *
-     * Each file in the index gets its own IDB record — commits only write
-     * new/changed files, not the entire index.
-     * Use `LocalIndex.open(name)` to reopen it after page refresh.
-     * @param {string} name
-     * @param {string} schema_sdl
-     * @returns {Promise<LocalIndex>}
-     */
-    static createPersistent(name, schema_sdl) {
-        const ptr0 = passStringToWasm0(name, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
-        const len0 = WASM_VECTOR_LEN;
-        const ptr1 = passStringToWasm0(schema_sdl, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
-        const len1 = WASM_VECTOR_LEN;
-        const ret = wasm.localindex_createPersistent(ptr0, len0, ptr1, len1);
-        return ret;
-    }
-    /**
-     * Delete a persistent index from IndexedDB.
-     * @param {string} name
-     * @returns {Promise<void>}
-     */
-    static deleteIndex(name) {
-        const ptr0 = passStringToWasm0(name, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
-        const len0 = WASM_VECTOR_LEN;
-        const ret = wasm.localindex_deleteIndex(ptr0, len0);
-        return ret;
-    }
-    /**
-     * Check if a persistent index exists in IndexedDB.
-     * @param {string} name
-     * @returns {Promise<boolean>}
-     */
-    static exists(name) {
-        const ptr0 = passStringToWasm0(name, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
-        const len0 = WASM_VECTOR_LEN;
-        const ret = wasm.localindex_exists(ptr0, len0);
-        return ret;
-    }
     /**
      * Get field names from the schema.
      * @returns {any}
@@ -435,19 +393,6 @@ export class LocalIndex {
         const ret = wasm.localindex_numDocs(this.__wbg_ptr);
         return ret >>> 0;
     }
-    /**
-     * Open an existing persistent index from IndexedDB.
-     *
-     * Loads the file manifest and all index files into a RamDirectory.
-     * @param {string} name
-     * @returns {Promise<LocalIndex>}
-     */
-    static open(name) {
-        const ptr0 = passStringToWasm0(name, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
-        const len0 = WASM_VECTOR_LEN;
-        const ret = wasm.localindex_open(ptr0, len0);
-        return ret;
-    }
     /**
      * Number of documents pending (not yet committed).
      * @returns {number}
@@ -483,6 +428,31 @@ export class LocalIndex {
         const ret = wasm.localindex_searchOffset(this.__wbg_ptr, ptr0, len0, limit, offset);
         return ret;
     }
+    /**
+     * Create or open an index with a pluggable storage backend.
+     *
+     * If the storage already contains index files, the index is reopened.
+     * Otherwise a new index is created from the SDL schema.
+     *
+     * The storage object must implement:
+     * ```ts
+     * interface IFilesStorage {
+     *     write(name: string, buffer: ArrayBuffer): Promise<void>;
+     *     get(name: string): Promise<ArrayBuffer | null>;
+     *     delete(names: string[]): Promise<void>;
+     *     list(): Promise<string[]>;
+     * }
+     * ```
+     * @param {any} storage
+     * @param {string} schema_sdl
+     * @returns {Promise<LocalIndex>}
+     */
+    static withStorage(storage, schema_sdl) {
+        const ptr0 = passStringToWasm0(schema_sdl, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        const len0 = WASM_VECTOR_LEN;
+        const ret = wasm.localindex_withStorage(storage, ptr0, len0);
+        return ret;
+    }
 }
 if (Symbol.dispose) LocalIndex.prototype[Symbol.dispose] = LocalIndex.prototype.free;
 
@@ -831,10 +801,18 @@ function __wbg_get_imports() {
             const ret = arg0.arrayBuffer();
             return ret;
         }, arguments); },
+        __wbg_buffer_60b8043cd926067d: function(arg0) {
+            const ret = arg0.buffer;
+            return ret;
+        },
         __wbg_call_2d781c1f4d5c0ef8: function() { return handleError(function (arg0, arg1, arg2) {
             const ret = arg0.call(arg1, arg2);
             return ret;
         }, arguments); },
+        __wbg_call_dcc2662fa17a72cf: function() { return handleError(function (arg0, arg1, arg2, arg3) {
+            const ret = arg0.call(arg1, arg2, arg3);
+            return ret;
+        }, arguments); },
         __wbg_call_e133b57c9155d22c: function() { return handleError(function (arg0, arg1) {
             const ret = arg0.call(arg1);
             return ret;
@@ -1130,6 +1108,10 @@ function __wbg_get_imports() {
         __wbg_prototypesetcall_d62e5099504357e6: function(arg0, arg1, arg2) {
             Uint8Array.prototype.set.call(getArrayU8FromWasm0(arg0, arg1), arg2);
         },
+        __wbg_push_e87b0e732085a946: function(arg0, arg1) {
+            const ret = arg0.push(arg1);
+            return ret;
+        },
         __wbg_put_f1673d719f93ce22: function() { return handleError(function (arg0, arg1, arg2) {
             const ret = arg0.put(arg1, arg2);
             return ret;
@@ -1262,22 +1244,22 @@ function __wbg_get_imports() {
             console.warn(arg0);
         },
         __wbindgen_cast_0000000000000001: function(arg0, arg1) {
-            // Cast intrinsic for `Closure(Closure { dtor_idx: 289, function: Function { arguments: [NamedExternref("Event")], shim_idx: 290, ret: Unit, inner_ret: Some(Unit) }, mutable: true }) -> Externref`.
+            // Cast intrinsic for `Closure(Closure { dtor_idx: 243, function: Function { arguments: [NamedExternref("Event")], shim_idx: 244, ret: Unit, inner_ret: Some(Unit) }, mutable: true }) -> Externref`.
             const ret = makeMutClosure(arg0, arg1, wasm.wasm_bindgen_efede8c867e8014b___closure__destroy___dyn_core_5858575f5ab61d4b___ops__function__FnMut__web_sys_108d82c43c7519e2___features__gen_IdbVersionChangeEvent__IdbVersionChangeEvent____Output_______, wasm_bindgen_efede8c867e8014b___convert__closures_____invoke___web_sys_108d82c43c7519e2___features__gen_IdbVersionChangeEvent__IdbVersionChangeEvent______true_);
             return ret;
         },
         __wbindgen_cast_0000000000000002: function(arg0, arg1) {
-            // Cast intrinsic for `Closure(Closure { dtor_idx: 289, function: Function { arguments: [NamedExternref("IDBVersionChangeEvent")], shim_idx: 290, ret: Unit, inner_ret: Some(Unit) }, mutable: true }) -> Externref`.
+            // Cast intrinsic for `Closure(Closure { dtor_idx: 243, function: Function { arguments: [NamedExternref("IDBVersionChangeEvent")], shim_idx: 244, ret: Unit, inner_ret: Some(Unit) }, mutable: true }) -> Externref`.
             const ret = makeMutClosure(arg0, arg1, wasm.wasm_bindgen_efede8c867e8014b___closure__destroy___dyn_core_5858575f5ab61d4b___ops__function__FnMut__web_sys_108d82c43c7519e2___features__gen_IdbVersionChangeEvent__IdbVersionChangeEvent____Output_______, wasm_bindgen_efede8c867e8014b___convert__closures_____invoke___web_sys_108d82c43c7519e2___features__gen_IdbVersionChangeEvent__IdbVersionChangeEvent______true__1);
             return ret;
         },
         __wbindgen_cast_0000000000000003: function(arg0, arg1) {
-            // Cast intrinsic for `Closure(Closure { dtor_idx: 762, function: Function { arguments: [], shim_idx: 763, ret: Unit, inner_ret: Some(Unit) }, mutable: true }) -> Externref`.
+            // Cast intrinsic for `Closure(Closure { dtor_idx: 741, function: Function { arguments: [], shim_idx: 742, ret: Unit, inner_ret: Some(Unit) }, mutable: true }) -> Externref`.
             const ret = makeMutClosure(arg0, arg1, wasm.wasm_bindgen_efede8c867e8014b___closure__destroy___dyn_core_5858575f5ab61d4b___ops__function__FnMut_____Output_______, wasm_bindgen_efede8c867e8014b___convert__closures_____invoke_______true_);
             return ret;
         },
         __wbindgen_cast_0000000000000004: function(arg0, arg1) {
-            // Cast intrinsic for `Closure(Closure { dtor_idx: 792, function: Function { arguments: [Externref], shim_idx: 793, ret: Result(Unit), inner_ret: Some(Result(Unit)) }, mutable: true }) -> Externref`.
+            // Cast intrinsic for `Closure(Closure { dtor_idx: 771, function: Function { arguments: [Externref], shim_idx: 772, ret: Result(Unit), inner_ret: Some(Result(Unit)) }, mutable: true }) -> Externref`.
             const ret = makeMutClosure(arg0, arg1, wasm.wasm_bindgen_efede8c867e8014b___closure__destroy___dyn_core_5858575f5ab61d4b___ops__function__FnMut__wasm_bindgen_efede8c867e8014b___JsValue____Output___core_5858575f5ab61d4b___result__Result_____wasm_bindgen_efede8c867e8014b___JsError___, wasm_bindgen_efede8c867e8014b___convert__closures_____invoke___wasm_bindgen_efede8c867e8014b___JsValue__core_5858575f5ab61d4b___result__Result_____wasm_bindgen_efede8c867e8014b___JsError___true_);
             return ret;
         },

package/package.json

@@ -5,7 +5,7 @@
     "izihawa"
   ],
   "description": "WASM bindings for Hermes search engine",
-  "version": "1.8.29",
+  "version": "1.8.30",
   "license": "MIT",
   "repository": {
     "type": "git",