lucivy 0.2.1 → 2.0.0

package/README.md

@@ -1,7 +1,17 @@
-# lucivy
+# lucivy v2
 
 Fast BM25 full-text search for Node.js — with substring matching, fuzzy search, regex, and highlights. Powered by Rust via napi-rs.
 
+[**Try the live playground**](https://l-defraiteur.github.io/lucivy/) — runs entirely in your browser via WASM.
+
+### What's new in v2
+
+- **SFX-only engine** — all queries route through the Suffix FST, no legacy code paths
+- **Distributed search** — `exportStats` / `mergeStats` / `searchWithGlobalStats`
+- **Incremental sync** — LUCIDS sharded delta export/apply
+- **Correct BM25 cross-shard** — identical scores whether 1 shard or 4
+- **5 bindings** — Python, Node.js, C++, WASM, Rust
+
 ## Install
 
 ```bash
@@ -13,16 +23,16 @@ npm install lucivy
 ```javascript
 const { Index } = require('lucivy');
 
-const index = Index.create('./my_index', [
-    { name: 'title', type: 'text' },
-    { name: 'body', type: 'text' },
+const index = Index.create('/tmp/my_index', [
+    { name: 'title', type: 'text', stored: true },
+    { name: 'body', type: 'text', stored: true },
 ]);
 
 index.add(1, { title: 'Rust Programming', body: 'Systems programming with memory safety' });
 index.add(2, { title: 'Python Guide', body: 'Data science and web development' });
 index.commit();
 
-let results = index.search('programming', { highlights: true });
+const results = index.search('programming', { highlights: true });
 for (const r of results) {
     console.log(r.docId, r.score, r.highlights);
 }
@@ -33,24 +43,31 @@ for (const r of results) {
 ### Create / open
 
 ```javascript
-const index = Index.create('./my_index', [
-    { name: 'title', type: 'text' },
-    { name: 'body',  type: 'text' },
-    { name: 'tag',   type: 'keyword' },
-    { name: 'year',  type: 'i64', indexed: true, fast: true },
+const index = Index.create('/tmp/my_index', [
+    { name: 'title', type: 'text', stored: true },
+    { name: 'body',  type: 'text', stored: true },
+    { name: 'score', type: 'f64', fast: true },
 ]);
 
-const index2 = Index.open('./my_index');
+// Sharded (4 shards)
+const sharded = Index.create('/tmp/sharded', [...], 4);
+
+// Open existing
+const index2 = Index.open('/tmp/my_index');
 ```
 
+Field types: `"text"` (full-text, tokenized), `"u64"`, `"i64"`, `"f64"`, `"bool"`, `"date"`.
+
 ### Add / update / delete
 
 ```javascript
-index.add(1, { title: 'Hello', body: 'World' });
+index.add(1, { title: 'Hello', body: 'World', score: 3.14 });
+
 index.addMany([
     { docId: 2, title: 'Foo', body: 'Bar' },
     { docId: 3, title: 'Baz', body: 'Qux' },
 ]);
+
 index.update(1, { title: 'Updated', body: 'Content' });
 index.delete(2);
 index.commit();
@@ -62,7 +79,7 @@ index.commit();
 // String query — each word searched across all text fields (contains_split)
 let results = index.search('rust async programming');
 
-// Options
+// Options: limit, highlights, allowedIds, fields
 results = index.search('rust', { limit: 20, highlights: true, allowedIds: [1, 3] });
 
 // Retrieve stored field values with results
@@ -74,26 +91,33 @@ for (const r of results) {
 
 #### contains — substring, fuzzy, regex (cross-token)
 
-Searches **stored text**, not individual tokens. Handles multi-word phrases, substrings, typos, and regex across token boundaries.
+All substring queries are cross-token: they match across token boundaries.
 
 ```javascript
-// Substring — matches "programming", "programmer", etc.
+// Substring — matches "programming", "programmer", "getProgramHandle", etc.
 index.search({ type: 'contains', field: 'body', value: 'program' });
 
-// Multi-word phrase
-index.search({ type: 'contains', field: 'body', value: 'memory safety' });
+// Fuzzy substring (Levenshtein distance)
+index.search({ type: 'contains', field: 'body', value: 'mutx', distance: 1 });
+
+// Regex substring — cross-token regex matching
+index.search({ type: 'contains', field: 'body', value: 'lock.*mutex', regex: true });
+
+// Prefix / startsWith — match must start at token boundary (SI=0)
+index.search({ type: 'startsWith', field: 'body', value: 'prog' });
 
-// Fuzzy (catches typos, distance=1 by default)
-index.search({ type: 'contains', field: 'body', value: 'programing languag', distance: 1 });
+// Exact whole-token match
+index.search({ type: 'term', field: 'body', value: 'lock' });
 
-// Regex on stored text
-index.search({ type: 'contains', field: 'body', value: 'program.*language', regex: true });
+// Phrase — adjacent tokens in order
+index.search({ type: 'phrase', field: 'body', value: 'mutex lock' });
 ```
 
-#### contains_split — one word = one contains query, OR'd together
+#### contains_split — multi-word search
+
+Split on whitespace, each word becomes a `contains` query, combined with boolean OR.
 
 ```javascript
-// "rust safety" → contains("rust") OR contains("safety") on body
 index.search({ type: 'contains_split', field: 'body', value: 'rust safety' });
 
 // With fuzzy distance — each word gets fuzzy tolerance
@@ -117,16 +141,31 @@ index.search({
 });
 ```
 
-#### keyword / range — for non-text fields
+#### Filtering
+
+Filter on non-text fields (combined with AND):
 
 ```javascript
-index.search({ type: 'keyword', field: 'tag', value: 'rust' });
 index.search({
-    type: 'contains', field: 'body', value: 'programming',
-    filters: [{ field: 'year', op: 'gte', value: 2023 }],
+    type: 'contains', field: 'body', value: 'lock',
+    filters: [
+        { field: 'category', op: 'eq', value: 'kernel' },
+        { field: 'score', op: 'gte', value: 0.5 },
+        { field: 'status', op: 'in', value: ['active', 'review'] },
+    ]
 });
 ```
 
+Filter ops: `eq`, `ne`, `lt`, `lte`, `gt`, `gte`, `in`, `not_in`, `between`, `starts_with`, `contains`.
+
+Pre-filter by document ID (fast, bitmap-based):
+
+```javascript
+index.search({ type: 'contains', field: 'body', value: 'lock' }, { allowedIds: [1, 2, 3] });
+```
+
+> **Note:** napi-rs converts snake_case to camelCase — use `allowedIds`, `docId`, `numDocs`, etc. in JavaScript.
+
 ### Snapshots (export / import)
 
 ```javascript
@@ -143,12 +182,54 @@ const restored = Index.importSnapshotFrom('./backup.luce', './restored_index');
 const restored2 = Index.importSnapshot(buf, './restored_index');
 ```
 
+### Delta sync (incremental)
+
+Sync only the segments that changed since the client's last version.
+
+```javascript
+// Get current shard versions
+const versions = index.shardVersions;
+
+// Export delta (only changed segments)
+const delta = index.exportShardedDelta(clientVersions);
+
+// Apply delta on the client side
+clientIndex.applyShardedDelta(delta);
+```
+
+### Distributed search
+
+Run BM25 search across multiple machines with correct IDF.
+
+```javascript
+const { mergeStats } = require('lucivy');
+
+const queryJson = '{"type":"contains","field":"body","value":"mutex"}';
+
+// 1. Each node exports its local BM25 stats
+const statsA = nodeA.exportStats(queryJson);  // JSON string
+const statsB = nodeB.exportStats(queryJson);  // JSON string
+
+// 2. Coordinator merges stats from all nodes
+const merged = mergeStats([statsA, statsB]);
+
+// 3. Each node searches with global stats (correct IDF across all nodes)
+const resultsA = nodeA.searchWithGlobalStats(queryJson, merged, 10);
+const resultsB = nodeB.searchWithGlobalStats(queryJson, merged, 10);
+
+// 4. Coordinator merges top-K results by score
+const all = [...resultsA, ...resultsB].sort((a, b) => b.score - a.score).slice(0, 10);
+```
+
 ### Properties
 
 ```javascript
-index.numDocs   // number of documents
-index.path      // index directory path
-index.schema    // array of field definitions
+index.numDocs      // number of documents (getter)
+index.numShards    // number of shards (getter)
+index.path         // index directory path (getter)
+index.schema       // array of {name, type} objects (getter)
+index.shardVersions // per-shard version info for delta sync (getter)
+index.close()      // flush + release writer lock
 ```
 
 ## License

package/index.d.ts

@@ -3,6 +3,11 @@
 
 /* auto-generated by NAPI-RS */
 
+export interface ShardVersion {
+  shardId: number
+  version: string
+  segmentIds: Array<string>
+}
 export interface SearchResult {
   docId: number
   score: number
@@ -23,48 +28,226 @@ export interface SearchOptions {
   fields?: boolean
 }
 export declare class Index {
-  /** Create a new index at the given path. */
-  static create(path: string, fields: Array<FieldDef>, stemmer?: string | undefined | null): Index
-  /** Open an existing index at the given path. */
+  /**
+   * Create a new index at the given path.
+   *
+   * @param path - Directory path for the index files.
+   * @param fields - Field definitions: `[{name: "body", type: "text", stored: true}]`.
+   *   Types: `"text"` (full-text), `"u64"`, `"i64"`, `"f64"`, `"bool"`, `"date"`.
+   * @param shards - Number of shards (default 1). More shards = faster search on large datasets.
+   */
+  static create(path: string, fields: Array<FieldDef>, shards?: number | undefined | null): Index
+  /**
+   * Open an existing index at the given path.
+   *
+   * Reads the persisted schema and segment metadata from disk.
+   * The index must have been previously created with `Index.create()`.
+   *
+   * @param path - Directory path of the existing index (same path used in `create()`).
+   * @returns An `Index` ready for search, add, delete, etc.
+   */
   static open(path: string): Index
-  /** Add a document. `fields` is an object with field names as keys. */
+  /**
+   * Add a document.
+   *
+   * @param docId - Unique document ID (_node_id).
+   * @param fields - Object with field names as keys: `{title: "Hello", body: "World", score: 3.14}`
+   */
   add(docId: number, fields: Record<string, any>): void
   /**
    * Add multiple documents at once.
-   * Each element must have a `docId` key and field values.
+   *
+   * Each element must have a `docId` (or `doc_id`) key plus field values.
+   *
+   * @param docs - Array of objects: `[{docId: 1, title: "Hello"}, {docId: 2, title: "World"}]`.
    */
   addMany(docs: Array<Record<string, any>>): void
-  /** Delete a document by doc_id. */
+  /**
+   * Delete a document by its `_node_id`.
+   *
+   * The deletion is staged in memory. Call `commit()` or run a search
+   * (which auto-commits via lazy commit) to make it visible.
+   *
+   * @param docId - The `_node_id` of the document to delete.
+   */
   delete(docId: number): void
-  /** Update a document (delete + re-add). */
+  /**
+   * Update a document (delete old + re-add with new fields).
+   *
+   * @param docId - The `_node_id` of the document to update.
+   * @param fields - New field values: `{title: "Updated", body: "New content"}`.
+   */
   update(docId: number, fields: Record<string, any>): void
-  /** Commit pending changes (makes added/deleted docs visible to searches). */
+  /**
+   * Commit pending changes to disk, making them visible to subsequent searches.
+   *
+   * Lucivy uses lazy commit: if you search without calling `commit()`,
+   * uncommitted changes are auto-flushed before the search executes.
+   * Call `commit()` explicitly when you need to control the commit point
+   * (e.g., after a batch of adds/deletes).
+   */
   commit(): void
-  /** Rollback pending changes. */
-  rollback(): void
+  /**
+   * Flush any pending writes and release the writer lock.
+   *
+   * After `close()`, the index data remains on disk and can be re-opened
+   * with `Index.open()`. No further mutations are allowed on this instance.
+   */
+  close(): void
   /**
    * Search the index.
-   * `query` can be a string (contains_split on all text fields) or an object (QueryConfig).
+   *
+   * @param query - String or query object.
+   *
+   * **String**: `"hello world"` — auto contains_split across all text fields.
+   *
+   * **Query object** (all substring queries are cross-token):
+   * - `{type: "contains", field: "body", value: "lock"}` — substring match
+   * - `{type: "contains", field: "body", value: "lock", distance: 1}` — fuzzy substring
+   * - `{type: "contains", field: "body", value: "lock.*init", regex: true}` — regex substring
+   * - `{type: "startsWith", field: "body", value: "lock"}` — token prefix
+   * - `{type: "contains_split", field: "body", value: "struct device"}` — words OR'd
+   * - `{type: "term", field: "body", value: "lock"}` — exact whole-token
+   * - `{type: "fuzzy", field: "body", value: "schdule", distance: 1}` — alias for contains+distance
+   * - `{type: "phrase", field: "body", value: "mutex lock"}` — adjacent tokens
+   * - `{type: "regex", field: "body", pattern: "sched[a-z]+"}` — regex on tokens
+   * - `{type: "boolean", must: [...], should: [...], must_not: [...]}` — boolean combo
+   * - `{type: "disjunction_max", queries: [...], tie_breaker: 0.1}` — best score
+   * - `{type: "more_like_this", field: "body", value: "sample text", min_doc_frequency: 1}` — similarity
+   *
+   * **Filtering:**
+   * - `allowed_ids` in options: pre-filter by _node_id (fast, bitmap-based)
+   * - `filters` key in query: filter on non-text fields (AND'd with search):
+   *   ```json
+   *   {type: "contains", field: "body", value: "lock",
+   *    filters: [
+   *      {field: "category", op: "eq", value: "kernel"},
+   *      {field: "score", op: "gte", value: 0.5},
+   *      {field: "status", op: "in", value: ["active", "review"]}
+   *    ]}
+   *   ```
+   *   Ops: `eq`, `ne`, `lt`, `lte`, `gt`, `gte`, `in`, `not_in`, `between`, `starts_with`, `contains`.
+   *   Composite: `must`, `should`, `must_not` with nested `clauses`.
+   *
+   * @param options - `{limit?: number, highlights?: boolean, fields?: boolean, allowed_ids?: number[]}`
    */
   search(query: any, options?: SearchOptions | undefined | null): Array<SearchResult>
-  /** Number of documents in the index. */
+  /**
+   * Number of documents in the index (getter, access as `index.numDocs`).
+   *
+   * @returns Total document count across all shards.
+   */
   get numDocs(): number
-  /** Index path. */
+  /**
+   * Number of shards (getter, access as `index.numShards`).
+   *
+   * @returns Shard count (1 for single-shard indexes).
+   */
+  get numShards(): number
+  /**
+   * Index directory path (getter, access as `index.path`).
+   *
+   * @returns The directory path where the index files are stored.
+   */
   get path(): string
   /**
-   * Export this index as a LUCE snapshot (Buffer).
-   * Throws if there are uncommitted changes.
+   * Export this index as a LUCE snapshot.
+   *
+   * Returns the full index content (all shards, schema, segments) as a
+   * binary blob. Restore later with `Index.importSnapshot()`.
+   *
+   * @returns `Buffer` containing the LUCE snapshot bytes.
    */
   exportSnapshot(): Buffer
-  /** Export this index as a LUCE snapshot to a file. */
+  /**
+   * Export this index as a LUCE snapshot directly to a file.
+   *
+   * @param path - Destination file path (typically ending in `.luce`).
+   */
   exportSnapshotTo(path: string): void
   /**
    * Import an index from a LUCE snapshot (Buffer).
-   * The snapshot must contain exactly one index.
+   *
+   * Restores a full index from a binary blob previously created by
+   * `exportSnapshot()`. The index files are written to `destPath`.
+   *
+   * @param data - Raw LUCE snapshot bytes (Buffer).
+   * @param destPath - Directory to write the restored index into.
+   *   Defaults to `"/tmp/lucivy_import"`.
+   * @returns A new `Index` instance ready for search.
    */
   static importSnapshot(data: Buffer, destPath?: string | undefined | null): Index
-  /** Import an index from a LUCE snapshot file. */
+  /**
+   * Import an index from a LUCE snapshot file (.luce).
+   *
+   * Convenience wrapper that reads the file then calls `importSnapshot()`.
+   *
+   * @param path - Path to the `.luce` snapshot file.
+   * @param destPath - Directory to write the restored index into.
+   *   Defaults to `"/tmp/lucivy_import"`.
+   * @returns A new `Index` instance ready for search.
+   */
   static importSnapshotFrom(path: string, destPath?: string | undefined | null): Index
-  /** Schema as a list of field definitions. */
+  /**
+   * Schema as a list of field definitions (getter, access as `index.schema`).
+   *
+   * @returns `Array<{name: string, type: string}>` for each user-defined field.
+   */
   get schema(): Array<FieldDef>
+  /**
+   * Per-shard version info for delta sync (getter, access as `index.shardVersions`).
+   *
+   * Returns the current version and segment IDs for each shard.
+   * Pass this to a remote server's `exportShardedDelta()` to
+   * receive only the segments that changed since your last sync.
+   *
+   * @returns `Array<{shardId: number, version: string, segmentIds: string[]}>`.
+   */
+  get shardVersions(): Array<ShardVersion>
+  /**
+   * Export a sharded delta (LUCIDS blob) containing only segments that
+   * changed since the client's known versions.
+   *
+   * Used for incremental sync: the client sends its `shardVersions`,
+   * the server computes and returns only the diff.
+   *
+   * @param clientVersions - Array of `{shardId, version, segmentIds}`,
+   *   typically obtained from the client's `shardVersions` getter.
+   * @returns `Buffer` containing the LUCIDS binary delta blob.
+   */
+  exportShardedDelta(clientVersions: Array<ShardVersion>): Buffer
+  /**
+   * Apply a sharded delta (LUCIDS blob) to this index.
+   *
+   * Merges the delta's segments into the local index, bringing it
+   * up to date with the server. Only modified shards are touched.
+   *
+   * @param data - LUCIDS binary blob from `exportShardedDelta()`.
+   */
+  applyShardedDelta(data: Buffer): void
+  /**
+   * Export BM25 statistics for a query (for distributed search).
+   *
+   * In a distributed setup, each node exports its local BM25 stats.
+   * A coordinator merges them into global stats and sends them back
+   * for scoring with `searchWithGlobalStats()`.
+   *
+   * @param queryJson - JSON string of QueryConfig (same format as `search()` object).
+   * @returns JSON string of `ExportableStats` (document frequencies, doc counts).
+   */
+  exportStats(queryJson: string): string
+  /**
+   * Search using externally-provided global BM25 stats (distributed mode).
+   *
+   * Scores are computed using the merged global stats instead of local-only
+   * stats, ensuring consistent ranking across nodes.
+   *
+   * @param queryJson - JSON string of QueryConfig.
+   * @param globalStatsJson - JSON string of merged `ExportableStats`
+   *   from all nodes (obtained by merging `exportStats()` outputs).
+   * @param limit - Maximum number of results (default 10).
+   * @param highlights - If true, return highlight byte offsets per field.
+   * @returns `Array<SearchResult>` scored with global BM25 statistics.
+   */
+  searchWithGlobalStats(queryJson: string, globalStatsJson: string, limit?: number | undefined | null, highlights?: boolean | undefined | null): Array<SearchResult>
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lucivy",
-  "version": "0.2.1",
+  "version": "2.0.0",
   "description": "Fast BM25 full-text search with substring matching, fuzzy search, and regex — powered by Rust",
   "main": "index.js",
   "types": "index.d.ts",
@@ -33,7 +33,7 @@
     "README.md"
   ],
   "devDependencies": {
-    "@napi-rs/cli": "^2.18.0"
+    "@napi-rs/cli": "^2.18.4"
   },
   "scripts": {
     "build": "napi build --release",