@delali/narsil 0.1.15 → 0.2.1-next.25

package/dist/index.d.ts

@@ -1,19 +1,193 @@
-export { A as ArrayFilter, B as BatchResult, C as ComparisonFilter, E as ErrorCode, a as ErrorCodes, F as FacetConfig, b as FacetResult, c as FilterExpression, d as FlushConfig, G as GeoPolygonFilter, e as GeoRadiusFilter, f as GroupConfig, g as GroupReducer, h as GroupResult, H as HighlightConfig, i as HighlightMatch, j as Hit, k as HybridConfig, I as IndexContext, l as IndexInfo, m as IndexStats, n as InsertContext, M as MemoryStats, N as NarsilConfig, o as NarsilError, p as NarsilPlugin, P as PartitionContext, q as PartitionStatsResult, r as PreflightResult, s as PresenceFilter, t as ProcessMemoryReport, Q as QueryParams, u as QueryResult, R as RemoveContext, S as ScoreComponents, v as SearchContext, w as SearchMode, x as StringFilter, y as SuggestParams, z as SuggestResult, T as TermMatchPolicy, U as UpdateContext, V as VectorMaintenanceResult, D as VectorQueryConfig, W as WorkerConfig, J as WorkerContext } from './config-CojEPOgS.js';
-import { L as LanguageModule } from './language-ONQ3iVDf.js';
-export { N as Narsil, a as NarsilEventMap, c as createNarsil } from './narsil-D-84Nvm4.js';
-export { E as EmbeddingAdapter, I as InvalidationAdapter, a as InvalidationEvent, P as PartitionStatistics, b as PersistenceAdapter } from './adapters-Bo2JQg8b.js';
-export { A as AnyDocument, B as BM25Params, C as CustomTokenizer, E as EmbeddingFieldConfig, F as FieldType, I as IndexConfig, a as InsertOptions, P as PartitionConfig, S as SchemaDefinition, b as ScoringMode } from './schema-DCgkO1j4.js';
+import { S as StopWordOverride, C as CustomTokenizer } from './schema-CE6S_aAr.js';
+export { A as AnyDocument, B as BM25Params, E as EmbeddingFieldConfig, F as FieldType, I as IndexConfig, a as InsertOptions, P as PartitionConfig, b as SchemaDefinition, c as ScoringMode, V as VectorIndexConfig, d as VectorQuantizationMode } from './schema-CE6S_aAr.js';
+import { L as LanguageModule } from './language-DB8KG1z5.js';
+export { T as TokenizerConfig } from './language-DB8KG1z5.js';
+export { A as ArrayFilter, B as BatchResult, C as ComparisonFilter, E as ErrorCode, a as ErrorCodes, F as FacetConfig, b as FacetResult, c as FieldFilter, d as FilterExpression, G as GeoFilter, e as GeoPolygonFilter, f as GeoRadiusFilter, g as GroupConfig, h as GroupReducer, i as GroupResult, H as HighlightConfig, j as HighlightMatch, k as Hit, l as HybridConfig, I as IndexInfo, m as IndexStats, M as MemoryStats, N as NarsilError, P as PartitionStatsResult, n as PreflightResult, o as PresenceFilter, p as ProcessMemoryReport, Q as QueryParams, q as QueryResult, S as ScoreComponents, r as SearchMode, s as StringFilter, t as SuggestParams, u as SuggestResult, T as TermMatchPolicy, V as VectorMaintenanceResult, v as VectorQueryConfig } from './search-CgQQIciB.js';
+import { N as NarsilConfig } from './config-CAjTHzkH.js';
+export { A as AnalysisConfig, D as DurabilityConfig, I as IndexContext, a as InsertContext, b as NarsilPlugin, P as PartitionContext, R as RemoveContext, S as SearchContext, c as StaleAnalysis, U as UpdateContext, W as WorkerConfig, d as WorkerContext } from './config-CAjTHzkH.js';
+import { N as Narsil } from './engine-CCrlfLyq.js';
+export { a as NarsilEventMap } from './engine-CCrlfLyq.js';
+export { E as EmbeddingAdapter, I as InvalidationAdapter, a as InvalidationEvent, P as PartitionStatistics, b as PersistenceAdapter } from './adapters-CVqI5NBt.js';
 
+/**
+ * Registers a tokeniser under a name that an index config can then reference.
+ *
+ * An index holds the name rather than the function, because a function
+ * survives neither a worker boundary nor a restart. Register the tokeniser on
+ * every thread that opens the index, and in the worker bootstrap module when
+ * the engine promotes to workers.
+ *
+ * @param name - The name an index config sets as its `tokenizer`.
+ * @param tokenizer - The tokeniser to bind to that name. Registering the same
+ * name again replaces what was there.
+ *
+ * @public
+ */
+declare function registerTokenizer(name: string, tokenizer: CustomTokenizer): void;
+/**
+ * Registers a stop word list under a name that an index config can then
+ * reference.
+ *
+ * An index holds the name rather than the list, so the same list survives a
+ * worker boundary and a restart. Register it on every thread that opens the
+ * index.
+ *
+ * @param name - The name an index config sets as its `stopWords`.
+ * @param stopWords - Either a fixed set of words, or a function that receives
+ * the language defaults and returns the set to use. Registering the same name
+ * again replaces what was there.
+ *
+ * @public
+ */
+declare function registerStopWords(name: string, stopWords: StopWordOverride): void;
+/**
+ * Reports whether a tokeniser is registered under this name, which is how a
+ * worker bootstrap checks its own work before an index needs it.
+ *
+ * @param name - The name to look for.
+ * @returns True once {@link registerTokenizer} has bound that name.
+ *
+ * @public
+ */
+declare function hasTokenizer(name: string): boolean;
+/**
+ * Reports whether a stop word list is registered under this name.
+ *
+ * @param name - The name to look for.
+ * @returns True once {@link registerStopWords} has bound that name.
+ *
+ * @public
+ */
+declare function hasStopWords(name: string): boolean;
+/**
+ * Returns the tokeniser registered under a name.
+ *
+ * @param name - The name {@link registerTokenizer} bound.
+ * @returns The registered tokeniser.
+ * @throws A `NarsilError` with `CONFIG_INVALID` when nothing holds that name.
+ *
+ * @public
+ */
+declare function getTokenizer(name: string): CustomTokenizer;
+/**
+ * Returns the stop word list registered under a name.
+ *
+ * @param name - The name {@link registerStopWords} bound.
+ * @returns The registered set, or the function that produces one.
+ * @throws A `NarsilError` with `CONFIG_INVALID` when nothing holds that name.
+ *
+ * @public
+ */
+declare function getStopWords(name: string): StopWordOverride;
+
+/**
+ * Sets how many normalised tokens the engine keeps cached.
+ *
+ * Normalising a token costs the same work every time the same word arrives, so
+ * the engine remembers the result. It sizes the cache from the host's memory
+ * on start-up; raise it for a corpus with a large vocabulary, and lower it on
+ * a memory-tight host.
+ *
+ * @param maxSize - Entries to keep. Zero switches the cache off.
+ * @throws A `NarsilError` with `CONFIG_INVALID` for a negative or non-finite
+ * size.
+ *
+ * @public
+ */
 declare function configureNormalizationCache(maxSize: number): void;
+/**
+ * Empties the normalisation cache while keeping the size you configured.
+ *
+ * Call it to release the memory the cache holds after a large load, or after
+ * registering a language module that normalises differently from the one whose
+ * results are cached.
+ *
+ * @public
+ */
 declare function clearNormalizationCache(): void;
+/**
+ * Empties the normalisation cache and returns its size to the value the engine
+ * derives from the host's memory, undoing
+ * {@link configureNormalizationCache}.
+ *
+ * @public
+ */
 declare function resetNormalizationCache(): void;
+/**
+ * Returns the number of normalised tokens the cache currently holds, which is
+ * what you watch to decide whether the configured size fits your vocabulary.
+ *
+ * @returns Entries held, across every language.
+ *
+ * @public
+ */
 declare function getNormalizationCacheSize(): number;
 
+/**
+ * Makes a language available to indexes that name it.
+ *
+ * English is registered already. Every other language is its own module, so a
+ * bundle carries only what you import. Import the one you want and register
+ * it before you create an index that sets it as `language`.
+ *
+ * @param module - The language module, which registers under its own `name`.
+ * Registering a name again replaces what was there.
+ *
+ * @public
+ */
 declare function registerLanguage(module: LanguageModule): void;
+/**
+ * Returns the language module registered under a name.
+ *
+ * @param name - The name the module registered under.
+ * @returns The registered module.
+ * @throws A `NarsilError` with `LANGUAGE_NOT_SUPPORTED` when nothing holds
+ * that name, which usually means the module was never imported.
+ *
+ * @public
+ */
 declare function getLanguage(name: string): LanguageModule;
 
+/**
+ * Creates a search engine that runs inside your process, with no server to
+ * provision.
+ *
+ * The engine holds every index you create, so an application usually keeps one
+ * instance for its lifetime. When you configure persistence, this recovers the
+ * indexes already on disk before it resolves, so the engine is ready to query
+ * as soon as you have it.
+ *
+ * @param config - This carries the persistence, partitioning, and worker
+ * invalidation settings. Omit it for an engine that keeps everything in memory.
+ * @returns The engine, ready for {@link Narsil.createIndex} to run against.
+ *
+ * @public
+ */
+declare function createNarsil(config?: NarsilConfig): Promise<Narsil>;
+
+/**
+ * Reports whether the runtime loaded the SIMD module that speeds up vector
+ * distance work.
+ *
+ * Vector search works either way, so read this to explain a difference in
+ * search latency between two hosts rather than to decide whether to search.
+ *
+ * @returns True once the SIMD module has loaded.
+ *
+ * @public
+ */
 declare function isSimdAvailable(): boolean;
 
+/**
+ * Engine version written into the engine-version bytes of every `.nrsl` file,
+ * for diagnostics.
+ *
+ * It records which engine produced a file and carries no relation to the
+ * published package version, so read it when inspecting a file rather than
+ * when checking what you installed. Change it by hand and on purpose.
+ *
+ * @public
+ */
 declare const VERSION = "0.1.0";
 
-export { LanguageModule, VERSION, clearNormalizationCache, configureNormalizationCache, getLanguage, getNormalizationCacheSize, isSimdAvailable, registerLanguage, resetNormalizationCache };
+export { CustomTokenizer, LanguageModule, Narsil, NarsilConfig, StopWordOverride, VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, isSimdAvailable, registerLanguage, registerStopWords, registerTokenizer, resetNormalizationCache };

package/dist/index.mjs

@@ -1,7 +1,8 @@
-export { VERSION, createNarsil } from './chunk-MEW7HD75.mjs';
-export { clearNormalizationCache, configureNormalizationCache, getLanguage, getNormalizationCacheSize, registerLanguage, resetNormalizationCache } from './chunk-HMO6OMZ3.mjs';
+export { VERSION, createNarsil } from './chunk-WLR5BFL3.mjs';
+export { clearNormalizationCache, configureNormalizationCache, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, registerLanguage, registerStopWords, registerTokenizer, resetNormalizationCache } from './chunk-ASI6AOW2.mjs';
+import './chunk-TGP5M5R7.mjs';
 export { isSimdAvailable } from './chunk-S7RJJMHI.mjs';
 import './chunk-KMMJMDUA.mjs';
-import './chunk-TW6BXTOL.mjs';
-export { ErrorCodes, NarsilError } from './chunk-ALB35PRQ.mjs';
-import './chunk-HRABJLPB.mjs';
+import './chunk-U2BKYEC7.mjs';
+export { ErrorCodes, NarsilError } from './chunk-UNJZ7ZDB.mjs';
+import './chunk-BKAI2WXI.mjs';

package/dist/invalidation/broadcast-channel.d.ts

@@ -1,8 +1,27 @@
-import { I as InvalidationAdapter } from '../adapters-Bo2JQg8b.js';
+import { I as InvalidationAdapter } from '../adapters-CVqI5NBt.js';
 
+/**
+ * Which channel {@link createBroadcastChannelInvalidation} publishes on.
+ *
+ * @public
+ */
 interface BroadcastChannelInvalidationConfig {
+    /** Every participating tab or worker must name the same channel. The adapter uses `narsil-invalidation` by default. */
     channelName?: string;
 }
+/**
+ * Builds an invalidation adapter that carries partition changes over the
+ * browser's `BroadcastChannel`, which is how several tabs sharing one
+ * IndexedDB index stay consistent.
+ *
+ * The browser delivers each event at once instead of on a poll, so a write in
+ * one tab reaches the others as soon as it dispatches.
+ *
+ * @param config - The channel name. Omit it to accept the default.
+ * @returns An adapter you pass as `invalidation` when creating an engine.
+ *
+ * @public
+ */
 declare function createBroadcastChannelInvalidation(config?: BroadcastChannelInvalidationConfig): InvalidationAdapter;
 
 export { type BroadcastChannelInvalidationConfig, createBroadcastChannelInvalidation };

package/dist/invalidation/filesystem.d.ts

@@ -1,10 +1,33 @@
-import { I as InvalidationAdapter } from '../adapters-Bo2JQg8b.js';
+import { I as InvalidationAdapter } from '../adapters-CVqI5NBt.js';
 
+/**
+ * How {@link createFilesystemInvalidation} exchanges events between processes.
+ *
+ * @public
+ */
 interface FilesystemInvalidationConfig {
+    /** The adapter writes its marker files to this directory. Every instance sharing an index must point at the same one. */
     directory: string;
+    /** The adapter sweeps the directory this many milliseconds apart, and 1000 apart by default. A shorter interval shortens the lag and costs more reads. */
     pollInterval?: number;
+    /** This identifies the instance, so it skips the markers it wrote itself. The adapter generates a random id by default. */
     instanceId?: string;
 }
+/**
+ * Builds an invalidation adapter that carries partition changes through marker
+ * files in a shared directory, which is how several Node processes on one host
+ * keep one persisted index consistent.
+ *
+ * Each publisher drops a marker and every other instance picks it up on its
+ * next sweep, so a change reaches the others within one poll interval. The
+ * adapter clears an old marker as it reads it.
+ *
+ * @param config - The shared directory, the poll interval, and this instance's
+ * identifier.
+ * @returns An adapter you pass as `invalidation` when creating an engine.
+ *
+ * @public
+ */
 declare function createFilesystemInvalidation(config: FilesystemInvalidationConfig): InvalidationAdapter;
 
 export { type FilesystemInvalidationConfig, createFilesystemInvalidation };

package/dist/invalidation/noop.d.ts

@@ -1,5 +1,18 @@
-import { I as InvalidationAdapter } from '../adapters-Bo2JQg8b.js';
+import { I as InvalidationAdapter } from '../adapters-CVqI5NBt.js';
 
+/**
+ * Builds an invalidation adapter that discards every event, which suits a
+ * single process holding a store nothing else writes to.
+ *
+ * Use it to satisfy code paths that expect an adapter without paying for a
+ * channel, and swap in {@link createFilesystemInvalidation} or
+ * {@link createBroadcastChannelInvalidation} once a second instance shares the
+ * store.
+ *
+ * @returns An adapter you pass as `invalidation` when creating an engine.
+ *
+ * @public
+ */
 declare function createNoopInvalidation(): InvalidationAdapter;
 
 export { createNoopInvalidation };

package/dist/language-DB8KG1z5.d.ts

@@ -0,0 +1,53 @@
+/**
+ * Everything the engine needs to analyse one language: how to split text into
+ * tokens, which words to drop, and how to reduce a word to its stem.
+ *
+ * Each language is its own module, so a bundle carries only the languages you
+ * import. Register one with {@link registerLanguage} before an index names it.
+ *
+ * @public
+ */
+interface LanguageModule {
+    /** An index config sets this name as its `language`. */
+    name: string;
+    /**
+     * This fingerprints the module's analysis. The engine records it with each
+     * index and reports the index as stale once the two differ, which is how a
+     * corrected stemmer or stop word list reaches indexes already written.
+     */
+    revision: string;
+    /**
+     * This reduces one token to the stem the engine indexes and searches on. It
+     * is `null` for a language the project analyses no further than its
+     * normaliser.
+     */
+    stemmer: ((token: string) => string) | null;
+    /** The engine drops these words before indexing, because they appear too widely to separate one document from another. */
+    stopWords: Set<string>;
+    /** This folds a token's script before stemming, which is where case, diacritics, and lookalike letters are settled. */
+    normalizer?: (token: string) => string;
+    /** These rules split text into tokens, when this language needs something other than the engine's default. */
+    tokenizer?: TokenizerConfig;
+}
+/**
+ * How the engine splits text into tokens for one language.
+ *
+ * Leave every field unset to accept the engine's default rules, which suit a
+ * space-separated script written in words.
+ *
+ * @public
+ */
+interface TokenizerConfig {
+    /** The engine splits on this pattern, which a script without spaces between words replaces. */
+    splitPattern?: RegExp;
+    /** Setting this folds accented letters onto their base letters, so `café` and `cafe` match. */
+    normalizeDiacritics?: boolean;
+    /** The engine drops a token shorter than this, which keeps stray fragments out of the index. */
+    minTokenLength?: number;
+    /** Setting this strips a trailing possessive, so `reader's` indexes as `reader`. */
+    stripPossessive?: boolean;
+    /** The engine cuts each run of characters into overlapping n-grams this long, which is how a script without word breaks is indexed. */
+    ngramSize?: number;
+}
+
+export type { LanguageModule as L, TokenizerConfig as T };

package/dist/languages/albanian.d.ts

@@ -0,0 +1,15 @@
+import { L as LanguageModule } from '../language-DB8KG1z5.js';
+
+/**
+ * Albanian analysis: the stop word list and the rules that split Albanian
+ * text into tokens. Tokens are indexed as the normaliser leaves them,
+ * without stemming.
+ *
+ * Import it and pass it to {@link registerLanguage} before you create an index
+ * whose `language` is `albanian`.
+ *
+ * @public
+ */
+declare const albanian: LanguageModule;
+
+export { albanian };

package/dist/languages/albanian.mjs

@@ -0,0 +1,103 @@
+// src/languages/albanian.ts
+var stopWords = /* @__PURE__ */ new Set([
+  "ai",
+  "ajo",
+  "as",
+  "asnj\xEB",
+  "ata",
+  "atje",
+  "ato",
+  "aty",
+  "brenda",
+  "\xE7do",
+  "\xE7far\xEB",
+  "cila",
+  "cil\xEBt",
+  "cili",
+  "deri",
+  "dhe",
+  "disa",
+  "do",
+  "e",
+  "edhe",
+  "\xEBsht\xEB",
+  "gjat\xEB",
+  "gjithashtu",
+  "gjith\xEB",
+  "i",
+  "im",
+  "ime",
+  "ishte",
+  "jan\xEB",
+  "jasht\xEB",
+  "jon\xEB",
+  "jote",
+  "ju",
+  "juaj",
+  "ka",
+  "kam",
+  "kan\xEB",
+  "ke",
+  "kemi",
+  "keni",
+  "k\xEBta",
+  "k\xEBto",
+  "k\xEBtu",
+  "kjo",
+  "ku",
+  "kur",
+  "ky",
+  "lart",
+  "mbi",
+  "me",
+  "m\xEB",
+  "midis",
+  "mos",
+  "nd\xEBr",
+  "nd\xEBrmjet",
+  "ne",
+  "n\xEB",
+  "n\xEBn",
+  "nga",
+  "nj\xEB",
+  "nuk",
+  "ose",
+  "pa",
+  "pak",
+  "para",
+  "pas",
+  "pastaj",
+  "p\xEBr",
+  "por",
+  "posht\xEB",
+  "prej",
+  "q\xEB",
+  "sa",
+  "saj",
+  "shum\xEB",
+  "si",
+  "sipas",
+  "tani",
+  "te",
+  "t\xEB",
+  "tek",
+  "ti",
+  "tij",
+  "tjet\xEBr",
+  "tyre",
+  "u",
+  "un\xEB",
+  "vet\xEB",
+  "vet\xEBm",
+  "veten",
+  "yt"
+]);
+var albanian = {
+  name: "albanian",
+  revision: "1",
+  stemmer: null,
+  stopWords,
+  tokenizer: { splitPattern: /[^a-z0-9çë]+/gi }
+};
+
+export { albanian };

package/dist/languages/amharic.d.ts

@@ -0,0 +1,15 @@
+import { L as LanguageModule } from '../language-DB8KG1z5.js';
+
+/**
+ * Amharic analysis: the stop word list and the rules that split Amharic
+ * text into tokens. Tokens are indexed as the normaliser leaves them,
+ * without stemming.
+ *
+ * Import it and pass it to {@link registerLanguage} before you create an index
+ * whose `language` is `amharic`.
+ *
+ * @public
+ */
+declare const amharic: LanguageModule;
+
+export { amharic };