@delali/narsil 0.1.15 → 0.2.0

package/dist/languages/wolof.mjs

@@ -0,0 +1,79 @@
+// src/languages/wolof.ts
+var stopWords = /* @__PURE__ */ new Set([
+  "ak",
+  "am",
+  "ba",
+  "ban",
+  "beneen",
+  "benn",
+  "bi",
+  "bii",
+  "boobu",
+  "bu",
+  "ca",
+  "ci",
+  "dafa",
+  "dina",
+  "du",
+  "fa",
+  "fi",
+  "fu",
+  "gi",
+  "it",
+  "itam",
+  "ji",
+  "ki",
+  "kon",
+  "ku",
+  "la",
+  "l\xE9pp",
+  "li",
+  "lool",
+  "lu",
+  "ma",
+  "man",
+  "mi",
+  "moom",
+  "mu",
+  "na",
+  "naa",
+  "na\xF1u",
+  "ndax",
+  "ndaxte",
+  "ne",
+  "\xF1\xE9pp",
+  "nga",
+  "ngeen",
+  "ni",
+  "\xF1i",
+  "\xF1oom",
+  "\xF1u",
+  "nun",
+  "rekk",
+  "sa",
+  "sama",
+  "seen",
+  "si",
+  "su",
+  "sunu",
+  "te",
+  "waaye",
+  "walla",
+  "wi",
+  "yeen",
+  "yeneen",
+  "y\xE9pp",
+  "yi",
+  "yii",
+  "yooyu",
+  "yow"
+]);
+var wolof = {
+  name: "wolof",
+  revision: "1",
+  stemmer: null,
+  stopWords,
+  tokenizer: { splitPattern: /[^\p{Script=Latin}0-9]+/giu }
+};
+
+export { wolof };

package/dist/languages/xhosa.d.ts

@@ -0,0 +1,5 @@
+import { L as LanguageModule } from '../language-CmJBHf1j.js';
+
+declare const xhosa: LanguageModule;
+
+export { xhosa };

package/dist/languages/xhosa.mjs

@@ -0,0 +1,95 @@
+// src/languages/xhosa.ts
+var stopWords = /* @__PURE__ */ new Set([
+  "aba",
+  "abanye",
+  "abo",
+  "apho",
+  "bakhe",
+  "bona",
+  "bonke",
+  "eli",
+  "elo",
+  "emva",
+  "enye",
+  "esi",
+  "eso",
+  "ezi",
+  "ezinye",
+  "ezo",
+  "kakhulu",
+  "kamva",
+  "kancinci",
+  "kanye",
+  "ke",
+  "kodwa",
+  "konke",
+  "kuba",
+  "kunye",
+  "kwaye",
+  "la",
+  "lakhe",
+  "le",
+  "leyo",
+  "lo",
+  "lonke",
+  "lowo",
+  "malunga",
+  "mna",
+  "na",
+  "nangona",
+  "ngaphakathi",
+  "ngaphandle",
+  "ngenxa",
+  "ngoku",
+  "ngokuba",
+  "nina",
+  "nje",
+  "okanye",
+  "oko",
+  "oku",
+  "olo",
+  "olu",
+  "omnye",
+  "phakathi",
+  "phambi",
+  "phambili",
+  "phantsi",
+  "phezulu",
+  "sakhe",
+  "sonke",
+  "thina",
+  "ukuba",
+  "ukuze",
+  "wabo",
+  "wakhe",
+  "wakho",
+  "wam",
+  "wena",
+  "wenu",
+  "wethu",
+  "xa",
+  "yabo",
+  "yakhe",
+  "yakho",
+  "yam",
+  "yena",
+  "yenu",
+  "yethu",
+  "yonke",
+  "zabo",
+  "zakhe",
+  "zakho",
+  "zam",
+  "zenu",
+  "zethu",
+  "zonke"
+]);
+var xhosa = {
+  name: "xhosa",
+  revision: "1",
+  stemmer: null,
+  stopWords,
+  tokenizer: { splitPattern: /[^a-z0-9-]+/gi }
+};
+
+export { xhosa };

package/dist/languages/yoruba.d.ts

@@ -1,4 +1,4 @@
-import { L as LanguageModule } from '../language-ONQ3iVDf.js';
+import { L as LanguageModule } from '../language-CmJBHf1j.js';
 
 declare const yoruba: LanguageModule;
 

package/dist/languages/yoruba.mjs

@@ -91,6 +91,7 @@ var stopWords = /* @__PURE__ */ new Set([
 ]);
 var yoruba = {
   name: "yoruba",
+  revision: "1",
   stemmer: null,
   stopWords,
   tokenizer: {

package/dist/languages/zulu.d.ts

@@ -1,4 +1,4 @@
-import { L as LanguageModule } from '../language-ONQ3iVDf.js';
+import { L as LanguageModule } from '../language-CmJBHf1j.js';
 
 declare const zulu: LanguageModule;
 

package/dist/languages/zulu.mjs

@@ -101,10 +101,11 @@ var stopWords = /* @__PURE__ */ new Set([
 ]);
 var zulu = {
   name: "zulu",
+  revision: "1",
   stemmer: null,
   stopWords,
   tokenizer: {
-    splitPattern: /[^a-zA-Z0-9]+/gi,
+    splitPattern: /[^a-zA-Z0-9-]+/gi,
     normalizeDiacritics: false,
     minTokenLength: 1
   }

package/dist/narsil-CMZRPDlQ.d.ts

@@ -0,0 +1,159 @@
+import { I as IndexConfig, A as AnyDocument, a as InsertOptions, P as PartitionConfig } from './schema-BnvLvjRp.js';
+import { k as IndexInfo, l as IndexStats, p as PartitionStatsResult, B as BatchResult, Q as QueryParams, t as QueryResult, q as PreflightResult, x as SuggestParams, y as SuggestResult, M as MemoryStats, V as VectorMaintenanceResult, N as NarsilConfig } from './config-P27AtSn4.js';
+import { E as EmbeddingAdapter } from './adapters-Bo2JQg8b.js';
+
+type NarsilEventMap = {
+    persistenceError: {
+        indexName: string;
+        partitionId: number;
+        error: Error;
+        retriesExhausted: boolean;
+    };
+    workerCrash: {
+        workerId: number;
+        indexNames: string[];
+        error: Error;
+    };
+    workerPromote: {
+        workerCount: number;
+        reason: string;
+    };
+    workerPromoteFailure: {
+        reason: string;
+        error: Error;
+        retryable: boolean;
+    };
+    partitionRebalance: {
+        indexName: string;
+        oldCount: number;
+        newCount: number;
+    };
+    partitionWatermark: {
+        indexName: string;
+        documentCount: number;
+        capacity: number;
+        partitionCount: number;
+    };
+    analysisRebuild: {
+        indexName: string;
+        status: 'started' | 'completed' | 'failed';
+        partitionsRebuilt: number;
+        partitionCount: number;
+        error?: Error;
+    };
+    durabilityError: {
+        error: Error;
+    };
+    invalidationError: {
+        error: Error;
+    };
+};
+
+interface Narsil {
+    /**
+     * Creates an index you can insert documents into and query.
+     *
+     * The schema sets the type of each field, decides which fields you can
+     * filter and sort on, and controls how the engine tokenises text. The
+     * `language` option picks the analyser that stems those text fields.
+     *
+     * @param name - Every later call uses this name to reach the index.
+     * @param config - This carries the schema, and the language, partitioning,
+     * and scoring settings.
+     */
+    createIndex(name: string, config: IndexConfig): Promise<void>;
+    /** Registers an adapter under a name that index configs can reference and
+     * durability metadata can persist; re-registering rebinds referencing indexes. */
+    registerEmbeddingAdapter(name: string, adapter: EmbeddingAdapter): void;
+    dropIndex(name: string): Promise<void>;
+    listIndexes(): IndexInfo[];
+    getStats(indexName: string): IndexStats;
+    getPartitionStats(indexName: string): PartitionStatsResult[];
+    /**
+     * Adds one document to an index and returns the id it is stored under.
+     *
+     * The id comes from the `docId` argument when you pass one, otherwise from
+     * the document's own `id` field, and otherwise the engine generates it. Use
+     * {@link Narsil.insertBatch} for a large load, so that one bad document
+     * cannot abandon the rest.
+     *
+     * @param indexName - The index that receives the document.
+     * @param document - Its fields must match the types the schema declares.
+     * @param docId - Pass an id to control it yourself, or omit it and read the
+     * returned value.
+     * @returns The id the document is stored under.
+     */
+    insert(indexName: string, document: AnyDocument, docId?: string, options?: InsertOptions): Promise<string>;
+    insertBatch(indexName: string, documents: AnyDocument[], options?: InsertOptions): Promise<BatchResult>;
+    remove(indexName: string, docId: string): Promise<void>;
+    removeBatch(indexName: string, docIds: string[]): Promise<BatchResult>;
+    update(indexName: string, docId: string, document: AnyDocument): Promise<void>;
+    updateBatch(indexName: string, updates: Array<{
+        docId: string;
+        document: AnyDocument;
+    }>): Promise<BatchResult>;
+    get(indexName: string, docId: string): Promise<AnyDocument | undefined>;
+    getMultiple(indexName: string, docIds: string[]): Promise<Map<string, AnyDocument>>;
+    has(indexName: string, docId: string): Promise<boolean>;
+    countDocuments(indexName: string): Promise<number>;
+    /**
+     * Runs a search against an index and returns the ranked hits.
+     *
+     * One call covers keyword search, vector search, and a hybrid of the two,
+     * chosen by the `mode` parameter. BM25 ranks keyword results by default.
+     * Every hit carries its score and its document, along with highlights when
+     * you ask for them.
+     *
+     * @param indexName - The index the search runs against.
+     * @param params - These set the search term, filters, field boosts, sorting,
+     * facets, highlighting, and the result limit.
+     * @returns The hits, the total count, the elapsed time, and any facets or
+     * groups the query asked for.
+     */
+    query<T = AnyDocument>(indexName: string, params: QueryParams): Promise<QueryResult<T>>;
+    preflight(indexName: string, params: QueryParams): Promise<PreflightResult>;
+    suggest(indexName: string, params: SuggestParams): Promise<SuggestResult>;
+    /**
+     * Rebuilds an index's terms from the documents it already stores, and
+     * resolves once every partition carries terms the current language module
+     * produced.
+     *
+     * An index whose stored analysis revision differs from the one its language
+     * module carries answers text queries from terms an earlier analysis wrote,
+     * and every such result reports `analysisStale`. Call this after the engine
+     * reports a stale index to configuration that declined an automatic rebuild.
+     * Calling it for an index whose terms are current does nothing.
+     *
+     * @param indexName - The index whose terms are rebuilt.
+     */
+    rebuildAnalysis(indexName: string): Promise<void>;
+    snapshot(indexName: string): Promise<Uint8Array>;
+    restore(indexName: string, data: Uint8Array): Promise<void>;
+    checkpoint(indexName: string): Promise<void>;
+    clear(indexName: string): Promise<void>;
+    rebalance(indexName: string, targetPartitionCount: number): Promise<void>;
+    updatePartitionConfig(indexName: string, config: Partial<PartitionConfig>): Promise<void>;
+    getMemoryStats(): Promise<MemoryStats>;
+    compactVectors(indexName: string, fieldName?: string): Promise<void>;
+    optimizeVectors(indexName: string, fieldName?: string): Promise<void>;
+    vectorMaintenanceStatus(indexName: string): VectorMaintenanceResult[];
+    on<K extends keyof NarsilEventMap>(event: K, handler: (payload: NarsilEventMap[K]) => void): void;
+    off<K extends keyof NarsilEventMap>(event: K, handler: (payload: NarsilEventMap[K]) => void): void;
+    shutdown(): Promise<void>;
+}
+/**
+ * Creates a search engine that runs inside your process, with no server to
+ * provision.
+ *
+ * The engine holds every index you create, so an application usually keeps one
+ * instance for its lifetime. When you configure persistence, this recovers the
+ * indexes already on disk before it resolves, so the engine is ready to query
+ * as soon as you have it.
+ *
+ * @param config - This carries the persistence, partitioning, and worker
+ * invalidation settings. Omit it for an engine that keeps everything in memory.
+ * @returns The engine, ready for {@link Narsil.createIndex} to run against.
+ */
+declare function createNarsil(config?: NarsilConfig): Promise<Narsil>;
+
+export { type Narsil as N, type NarsilEventMap as a, createNarsil as c };

package/dist/persistence/durability/checkpoint-worker.d.ts

@@ -1,3 +1,6 @@
+import { P as PartitionConfig, c as ScoringMode, V as VectorIndexConfig } from '../../schema-BnvLvjRp.js';
+import '../../adapters-Bo2JQg8b.js';
+
 interface IndexMetadata {
     indexName: string;
     schema: Record<string, string>;
@@ -17,6 +20,16 @@ interface IndexMetadata {
     embedding?: IndexEmbeddingMetadata;
     /** Persisted so recovered indexes keep collecting surface forms. */
     surfaceForms?: boolean;
+    analysisRevision?: string;
+    tokenizer?: string;
+    stopWords?: string;
+    stopWordList?: string[];
+    partitionLimits?: PartitionConfig;
+    defaultScoring?: ScoringMode;
+    trackPositions?: boolean;
+    strict?: boolean;
+    required?: string[];
+    vectorPromotion?: VectorIndexConfig;
 }
 /** Persisted so recovery can restore field mappings and rebind the adapter by
  * name; the name is absent when the index was configured with a bare instance. */

package/dist/persistence/durability/checkpoint-worker.mjs

@@ -1,10 +1,11 @@
-import { createDurableDirectory, writeSegmentedCheckpoint } from '../../chunk-MEW7HD75.mjs';
-import '../../chunk-HMO6OMZ3.mjs';
+import { createDurableDirectory, writeSegmentedCheckpoint } from '../../chunk-XAKNSWIK.mjs';
+import '../../chunk-S6IAXOBA.mjs';
+import '../../chunk-TGP5M5R7.mjs';
 import '../../chunk-S7RJJMHI.mjs';
 import '../../chunk-KMMJMDUA.mjs';
-import '../../chunk-TW6BXTOL.mjs';
-import '../../chunk-ALB35PRQ.mjs';
-import '../../chunk-HRABJLPB.mjs';
+import '../../chunk-U2BKYEC7.mjs';
+import '../../chunk-M2EDG55Y.mjs';
+import '../../chunk-BKAI2WXI.mjs';
 
 // src/persistence/durability/rebuild.ts
 async function rebuildSnapshotFromDurable(root, metadata, targets, compactionThreshold) {

package/dist/{schema-DCgkO1j4.d.ts → schema-BnvLvjRp.d.ts}

@@ -24,14 +24,15 @@ interface EmbeddingFieldConfig {
     adapter?: EmbeddingAdapter | string;
     fields: Record<string, string | string[]>;
 }
+type StopWordOverride = Set<string> | ((defaults: Set<string>) => Set<string>);
 interface IndexConfig {
     schema: SchemaDefinition;
     language?: string;
     partitions?: PartitionConfig;
     defaultScoring?: ScoringMode;
     bm25?: BM25Params;
-    stopWords?: Set<string> | ((defaults: Set<string>) => Set<string>);
-    tokenizer?: CustomTokenizer;
+    stopWords?: StopWordOverride | string;
+    tokenizer?: CustomTokenizer | string;
     trackPositions?: boolean;
     vectorPromotion?: VectorIndexConfig;
     strict?: boolean;
@@ -56,10 +57,11 @@ interface CustomTokenizer {
 interface PartitionConfig {
     maxDocsPerPartition?: number;
     maxPartitions?: number;
+    watermark?: number;
 }
 type ScoringMode = 'local' | 'dfs' | 'broadcast';
 interface InsertOptions {
     skipClone?: boolean;
 }
 
-export type { AnyDocument as A, BM25Params as B, CustomTokenizer as C, EmbeddingFieldConfig as E, FieldType as F, IndexConfig as I, PartitionConfig as P, SchemaDefinition as S, InsertOptions as a, ScoringMode as b };
+export type { AnyDocument as A, BM25Params as B, CustomTokenizer as C, EmbeddingFieldConfig as E, FieldType as F, IndexConfig as I, PartitionConfig as P, StopWordOverride as S, VectorIndexConfig as V, InsertOptions as a, SchemaDefinition as b, ScoringMode as c };

package/dist/server.d.ts

@@ -1,7 +1,7 @@
-import { N as Narsil } from './narsil-D-84Nvm4.js';
+import { N as Narsil } from './narsil-CMZRPDlQ.js';
 import { E as EmbeddingAdapter } from './adapters-Bo2JQg8b.js';
-import './config-CojEPOgS.js';
-import './schema-DCgkO1j4.js';
+import './schema-BnvLvjRp.js';
+import './config-P27AtSn4.js';
 
 /**
  * Context handed to {@link OnRequestHook} for every inbound request, captured
@@ -163,6 +163,7 @@ interface HttpIndexConfig {
     partitions?: {
         maxDocsPerPartition?: number;
         maxPartitions?: number;
+        watermark?: number;
     };
     defaultScoring?: 'local' | 'dfs' | 'broadcast';
     bm25?: {

package/dist/server.mjs

@@ -1,4 +1,4 @@
-import { NarsilError, ErrorCodes } from './chunk-ALB35PRQ.mjs';
+import { NarsilError, ErrorCodes } from './chunk-M2EDG55Y.mjs';
 import { randomUUID } from 'crypto';
 
 // src/server/cors.ts
@@ -81,7 +81,18 @@ var STATUS_BY_CODE = {
   QUERY_ROUTING_FAILED: 503,
   QUERY_PARTIAL_FAILURE: 503,
   QUERY_NODE_TIMEOUT: 503,
-  QUERY_NO_ACTIVE_REPLICA: 503
+  QUERY_NO_ACTIVE_REPLICA: 503,
+  REPLICATION_ENTRY_INVALID: 503,
+  REPLICATION_INSYNC_REMOVAL_FAILED: 503,
+  REPLICATION_ROLLBACK_FAILED: 503,
+  REPLICATION_LOG_FULL: 503,
+  REPLICATION_ENTRY_CORRUPT: 503,
+  REPLICATION_SNAPSHOT_CORRUPT: 503,
+  REPLICATION_TERM_MISMATCH: 503,
+  REPLICATION_SYNC_FAILED: 503,
+  PARTITION_NOT_PRIMARY: 503,
+  PARTITION_UNASSIGNED: 503,
+  INSUFFICIENT_REPLICAS: 503
 };
 function httpStatusForNarsilError(code) {
   return STATUS_BY_CODE[code] ?? 500;

package/dist/{types-DHX6zGeD.d.ts → types-BRTcBgAp.d.ts}

@@ -1,4 +1,4 @@
-import { S as SchemaDefinition } from './schema-DCgkO1j4.js';
+import { b as SchemaDefinition } from './schema-BnvLvjRp.js';
 
 type NodeRole = 'data' | 'coordinator' | 'controller';
 type PartitionState = 'UNASSIGNED' | 'INITIALISING' | 'ACTIVE' | 'MIGRATING' | 'DECOMMISSIONING';

package/dist/workers/entry.mjs

@@ -1,4 +1,4 @@
-import { isValidWorkerAction } from '../chunk-HRABJLPB.mjs';
+import { isValidWorkerAction } from '../chunk-BKAI2WXI.mjs';
 
 // src/workers/worker-entry.ts
 function startWorker() {
@@ -15,7 +15,7 @@ async function setup() {
   } catch {
     parentPort = null;
   }
-  const { createDirectExecutor } = await import('../direct-executor-E52FFGDE.mjs');
+  const { createDirectExecutor } = await import('../direct-executor-2FV3MEDD.mjs');
   const executor = createDirectExecutor();
   function buildErrorResponse(requestId, code, message) {
     return { type: "error", requestId, code, message };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@delali/narsil",
-  "version": "0.1.15",
+  "version": "0.2.0",
   "type": "module",
   "description": "A distributed full-text search engine with cross-language portable binary format.",
   "author": "Delali (https://sondelali.com)",
@@ -109,7 +109,8 @@
   },
   "files": [
     "dist/",
-    "LICENSE"
+    "LICENSE",
+    "NOTICE"
   ],
   "dependencies": {
     "@msgpack/msgpack": "3.1.1"
@@ -122,6 +123,7 @@
     "fflate": "0.8.3",
     "selfsigned": "5.5.0",
     "tsup": "8.5.1",
+    "tsx": "4.21.0",
     "typescript": "5.9.3",
     "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.58.0",
     "vitest": "4.0.18",
@@ -149,10 +151,15 @@
     "test:recall": "vitest run src/__tests__/vector/recall.test.ts",
     "test:embeddings": "NARSIL_TEST_EMBEDDINGS=1 vitest run embedding-e2e",
     "bench": "vitest bench",
+    "stemmers": "tsx snowball/build.ts",
+    "revisions": "tsx scripts/analysis-revisions.ts",
+    "revisions:write": "tsx scripts/analysis-revisions.ts --write",
+    "lookalikes": "tsx scripts/lookalike-counts.ts",
+    "lookalikes:collect": "tsx scripts/lookalike-counts.ts --collect",
     "lint": "biome check src/",
     "typecheck:src": "tsc --noEmit",
     "typecheck:tests": "tsc --noEmit -p tsconfig.test.json",
-    "typecheck:tooling": "tsc --noEmit -p scripts/tsconfig.json && tsc --noEmit -p benchmarks/tsconfig.json && tsc --noEmit -p wasm/tsconfig.json",
+    "typecheck:tooling": "tsc --noEmit -p scripts/tsconfig.json && tsc --noEmit -p benchmarks/tsconfig.json && tsc --noEmit -p wasm/tsconfig.json && tsc --noEmit -p snowball/tsconfig.json",
     "typecheck": "pnpm run typecheck:src && pnpm run typecheck:tests && pnpm run typecheck:tooling"
   }
 }