@delali/narsil 0.1.5 → 0.1.7

package/README.md

@@ -23,9 +23,9 @@ Requires Node.js >= 22.
 ## Quick start
 
 ```ts
-import { Narsil } from '@delali/narsil'
+import { createNarsil } from '@delali/narsil'
 
-const narsil = new Narsil()
+const narsil = await createNarsil()
 
 await narsil.createIndex('products', {
   schema: {
@@ -59,6 +59,72 @@ const results = await narsil.query('products', {
 })
 ```
 
+## Configuration
+
+`createNarsil` accepts an optional `NarsilConfig` object. All fields are optional.
+
+```ts
+import { createNarsil } from '@delali/narsil'
+
+const narsil = await createNarsil({
+  persistence: myAdapter,
+  workers: { enabled: true, count: 4 },
+  flush: { interval: 5000, mutationThreshold: 100 },
+})
+```
+
+### NarsilConfig
+
+| Field | Type | Description |
+| --- | --- | --- |
+| `persistence` | `PersistenceAdapter` | Storage backend for durable indexes (filesystem, IndexedDB, or custom) |
+| `invalidation` | `InvalidationAdapter` | Cross-instance cache coordination adapter for multi-process or multi-tab setups |
+| `plugins` | `NarsilPlugin[]` | Lifecycle hooks for insert, remove, update, search, and index events |
+| `idGenerator` | `() => string` | Custom function for generating document IDs (defaults to UUID v7) |
+| `workers` | `WorkerConfig` | Worker thread configuration for parallel search |
+| `flush` | `FlushConfig` | Controls when dirty partitions persist to storage |
+| `eagerLoad` | `boolean` | When `true`, loads all persisted data into memory at creation time |
+| `embedding` | `EmbeddingAdapter` | Default adapter for auto-embedding text fields into vectors (see [Embedding adapters](#embedding-adapters)) |
+
+### WorkerConfig
+
+| Field | Type | Default | Description |
+| --- | --- | --- | --- |
+| `enabled` | `boolean` | `false` | Enables worker thread pool for search |
+| `count` | `number` | CPU count | Number of worker threads to spawn |
+| `promotionThreshold` | `number` | - | Document count per index that triggers auto-promotion to workers |
+| `totalPromotionThreshold` | `number` | - | Total document count across all indexes that triggers auto-promotion |
+
+### FlushConfig
+
+| Field | Type | Default | Description |
+| --- | --- | --- | --- |
+| `interval` | `number` | - | Milliseconds between persistence flushes |
+| `mutationThreshold` | `number` | - | Number of mutations that triggers a flush |
+
+### Tokenizer cache
+
+The stemmer normalization cache auto-sizes based on the runtime environment. On Node.js it reads container memory limits (via `process.constrainedMemory()`), in browsers it checks `navigator.deviceMemory`, and falls back to a sensible default elsewhere.
+
+This cache is process-global, shared across all Narsil instances in the same process. You can override the size by calling `configureNormalizationCache` once at startup, before creating any instances:
+
+```ts
+import { configureNormalizationCache, createNarsil } from '@delali/narsil'
+
+configureNormalizationCache(500_000)
+
+const narsil = await createNarsil()
+```
+
+The value clamps to a floor of 50,000 and a ceiling of 2,000,000 entries. Invalid values (NaN, Infinity, negative numbers, zero) throw a `NarsilError` with code `CONFIG_INVALID`.
+
+Four utility functions are available for cache management:
+
+- `configureNormalizationCache(maxSize)` - set the maximum cache size
+- `clearNormalizationCache()` - drop all cached entries (reclaim memory after one-off indexing)
+- `resetNormalizationCache()` - clear the cache and reset the size to the auto-detected default
+- `getNormalizationCacheSize()` - return the current number of cached entries (useful for monitoring)
+
 ## Features
 
 ### Full-text search
@@ -146,6 +212,68 @@ const results = await narsil.query('products', {
 // hit.highlights.title.snippet => '<mark>Mechanical</mark> Keyboard'
 ```
 
+### Embedding adapters
+
+Auto-embed text fields into vectors on insert and query. Pass an `EmbeddingAdapter` at the instance level (shared default) or per-index. When a document is inserted without a pre-computed vector, Narsil concatenates the configured source text fields and calls the adapter to generate the embedding. On query, pass `text` instead of `value` to auto-embed the search phrase.
+
+```ts
+import { createNarsil } from '@delali/narsil'
+import { createOpenAIEmbedding } from '@delali/narsil/embeddings/openai'
+
+const narsil = await createNarsil({
+  embedding: createOpenAIEmbedding({
+    apiKey: process.env.OPENAI_API_KEY,
+    model: 'text-embedding-3-small',
+    dimensions: 1536,
+  }),
+})
+
+await narsil.createIndex('articles', {
+  schema: {
+    title: 'string',
+    body: 'string',
+    embedding: 'vector[1536]',
+  },
+  embedding: {
+    fields: {
+      embedding: ['title', 'body'],
+    },
+  },
+})
+
+await narsil.insert('articles', {
+  title: 'Distributed search engines',
+  body: 'Partitioning data across shards improves throughput...',
+})
+
+const results = await narsil.query('articles', {
+  mode: 'vector',
+  vector: { field: 'embedding', text: 'how do search engines scale?' },
+})
+```
+
+Two adapters ship with the project:
+
+| Adapter | Package | Dependencies |
+| --- | --- | --- |
+| OpenAI | `@delali/narsil/embeddings/openai` | None (uses `fetch`) |
+| Transformers.js | `@delali/narsil-embeddings-transformers` | `@huggingface/transformers` (peer) |
+
+The OpenAI adapter includes retry logic with exponential backoff and jitter, batch chunking (up to 2,048 inputs per request), and configurable timeouts. The Transformers.js adapter runs models locally with lazy pipeline initialization, supports WebGPU/WASM/CPU backends, and handles asymmetric models through document/query prefixes.
+
+Build a custom adapter by implementing the `EmbeddingAdapter` interface:
+
+```ts
+interface EmbeddingAdapter {
+  embed(input: string, purpose: 'document' | 'query', signal?: AbortSignal): Promise<Float32Array>
+  embedBatch?(inputs: string[], purpose: 'document' | 'query', signal?: AbortSignal): Promise<Float32Array[]>
+  readonly dimensions: number
+  shutdown?(): Promise<void>
+}
+```
+
+If `embedBatch` is not provided, Narsil falls back to parallel `embed()` calls with a concurrency limit of 8.
+
 ### Term suggestions
 
 Get autocomplete suggestions from the index's term dictionary. The API tokenizes the input, extracts the last word, and matches it against stored terms across all partitions. Results are ranked by document frequency.

package/dist/adapters/filesystem.d.ts

@@ -1,4 +1,4 @@
-import { P as PersistenceAdapter } from '../adapters-XODT2F3h.js';
+import { P as PersistenceAdapter } from '../adapters-BbRkoiu9.js';
 
 interface FilesystemPersistenceConfig {
     directory: string;

package/dist/adapters/filesystem.mjs

@@ -1,4 +1,4 @@
-import { NarsilError, ErrorCodes } from '../chunk-7GKPWGCN.mjs';
+import { NarsilError, ErrorCodes } from '../chunk-R2C2EQGO.mjs';
 
 // src/persistence/filesystem.ts
 var cachedFs = null;

package/dist/adapters/indexeddb.d.ts

@@ -1,4 +1,4 @@
-import { P as PersistenceAdapter } from '../adapters-XODT2F3h.js';
+import { P as PersistenceAdapter } from '../adapters-BbRkoiu9.js';
 
 interface IndexedDBPersistenceConfig {
     dbName?: string;

package/dist/adapters/indexeddb.mjs

@@ -1,4 +1,4 @@
-import { NarsilError, ErrorCodes } from '../chunk-7GKPWGCN.mjs';
+import { NarsilError, ErrorCodes } from '../chunk-R2C2EQGO.mjs';
 
 // src/persistence/indexeddb.ts
 var DEFAULT_DB_NAME = "narsil";

package/dist/adapters/memory.d.ts

@@ -1,4 +1,4 @@
-import { P as PersistenceAdapter } from '../adapters-XODT2F3h.js';
+import { P as PersistenceAdapter } from '../adapters-BbRkoiu9.js';
 
 declare function createMemoryPersistence(): PersistenceAdapter;
 

package/dist/{adapters-XODT2F3h.d.ts → adapters-BbRkoiu9.d.ts}

@@ -26,5 +26,11 @@ interface PartitionStatistics {
     docFrequencies: Record<string, number>;
     totalFieldLengths: Record<string, number>;
 }
+interface EmbeddingAdapter {
+    embed(input: string, purpose: 'document' | 'query', signal?: AbortSignal): Promise<Float32Array>;
+    embedBatch?(inputs: string[], purpose: 'document' | 'query', signal?: AbortSignal): Promise<Float32Array[]>;
+    readonly dimensions: number;
+    shutdown?(): Promise<void>;
+}
 
-export type { InvalidationAdapter as I, PersistenceAdapter as P, InvalidationEvent as a, PartitionStatistics as b };
+export type { EmbeddingAdapter as E, InvalidationAdapter as I, PersistenceAdapter as P, InvalidationEvent as a, PartitionStatistics as b };

package/dist/{chunk-WYC635GI.mjs → chunk-I24SGLFS.mjs}

@@ -53,7 +53,6 @@ var RE_STEP4_SUFFIXES = /^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|
 var RE_STEP4_STION = /^(.+?)(s|t)(ion)$/;
 var RE_TRAILING_E = /^(.+?)e$/;
 var RE_TRAILING_LL = /ll$/;
-var RE_LAST_CHAR = /.$/;
 function hasMeasureGt0(stem) {
   return RE_MEASURE_GT_0.test(stem);
 }
@@ -61,14 +60,14 @@ function hasMeasureGt1(stem) {
   return RE_MEASURE_GT_1.test(stem);
 }
 function stripPlurals(word) {
-  if (RE_PLURAL_SS_IES.test(word)) return word.replace(RE_PLURAL_SS_IES, "$1$2");
-  if (RE_PLURAL_S.test(word)) return word.replace(RE_PLURAL_S, "$1$2");
+  if (RE_PLURAL_SS_IES.test(word)) return word.slice(0, -2);
+  if (RE_PLURAL_S.test(word)) return word.slice(0, -1);
   return word;
 }
 function stripVerbSuffixes(word) {
   let match = RE_SUFFIX_EED.exec(word);
   if (match) {
-    return hasMeasureGt0(match[1]) ? word.replace(RE_LAST_CHAR, "") : word;
+    return hasMeasureGt0(match[1]) ? word.slice(0, -1) : word;
   }
   match = RE_SUFFIX_ED_ING.exec(word);
   if (match) {
@@ -76,7 +75,7 @@ function stripVerbSuffixes(word) {
     if (!RE_HAS_VOWEL.test(base)) return word;
     word = base;
     if (RE_SUFFIX_AT_BL_IZ.test(word)) return `${word}e`;
-    if (RE_DOUBLE_CONSONANT.test(word)) return word.replace(RE_LAST_CHAR, "");
+    if (RE_DOUBLE_CONSONANT.test(word)) return word.slice(0, -1);
     if (RE_CONSONANT_CVC.test(word)) return `${word}e`;
   }
   return word;
@@ -122,7 +121,7 @@ function tidyTrailingE(word) {
 }
 function collapseDoubleLl(word) {
   if (RE_TRAILING_LL.test(word) && hasMeasureGt1(word)) {
-    return word.replace(RE_LAST_CHAR, "");
+    return word.slice(0, -1);
   }
   return word;
 }