@delali/narsil 0.1.15 → 0.2.0

package/dist/index.d.ts

@@ -1,8 +1,17 @@
-export { A as ArrayFilter, B as BatchResult, C as ComparisonFilter, E as ErrorCode, a as ErrorCodes, F as FacetConfig, b as FacetResult, c as FilterExpression, d as FlushConfig, G as GeoPolygonFilter, e as GeoRadiusFilter, f as GroupConfig, g as GroupReducer, h as GroupResult, H as HighlightConfig, i as HighlightMatch, j as Hit, k as HybridConfig, I as IndexContext, l as IndexInfo, m as IndexStats, n as InsertContext, M as MemoryStats, N as NarsilConfig, o as NarsilError, p as NarsilPlugin, P as PartitionContext, q as PartitionStatsResult, r as PreflightResult, s as PresenceFilter, t as ProcessMemoryReport, Q as QueryParams, u as QueryResult, R as RemoveContext, S as ScoreComponents, v as SearchContext, w as SearchMode, x as StringFilter, y as SuggestParams, z as SuggestResult, T as TermMatchPolicy, U as UpdateContext, V as VectorMaintenanceResult, D as VectorQueryConfig, W as WorkerConfig, J as WorkerContext } from './config-CojEPOgS.js';
-import { L as LanguageModule } from './language-ONQ3iVDf.js';
-export { N as Narsil, a as NarsilEventMap, c as createNarsil } from './narsil-D-84Nvm4.js';
+import { S as StopWordOverride, C as CustomTokenizer } from './schema-BnvLvjRp.js';
+export { A as AnyDocument, B as BM25Params, E as EmbeddingFieldConfig, F as FieldType, I as IndexConfig, a as InsertOptions, P as PartitionConfig, b as SchemaDefinition, c as ScoringMode } from './schema-BnvLvjRp.js';
+import { L as LanguageModule } from './language-CmJBHf1j.js';
+export { T as TokenizerConfig } from './language-CmJBHf1j.js';
+export { A as ArrayFilter, B as BatchResult, C as ComparisonFilter, D as DurabilityConfig, E as ErrorCode, a as ErrorCodes, F as FacetConfig, b as FacetResult, c as FilterExpression, G as GeoPolygonFilter, d as GeoRadiusFilter, e as GroupConfig, f as GroupReducer, g as GroupResult, H as HighlightConfig, h as HighlightMatch, i as Hit, j as HybridConfig, I as IndexContext, k as IndexInfo, l as IndexStats, m as InsertContext, M as MemoryStats, N as NarsilConfig, n as NarsilError, o as NarsilPlugin, P as PartitionContext, p as PartitionStatsResult, q as PreflightResult, r as PresenceFilter, s as ProcessMemoryReport, Q as QueryParams, t as QueryResult, R as RemoveContext, S as ScoreComponents, u as SearchContext, v as SearchMode, w as StringFilter, x as SuggestParams, y as SuggestResult, T as TermMatchPolicy, U as UpdateContext, V as VectorMaintenanceResult, z as VectorQueryConfig, W as WorkerConfig, J as WorkerContext } from './config-P27AtSn4.js';
+export { N as Narsil, a as NarsilEventMap, c as createNarsil } from './narsil-CMZRPDlQ.js';
 export { E as EmbeddingAdapter, I as InvalidationAdapter, a as InvalidationEvent, P as PartitionStatistics, b as PersistenceAdapter } from './adapters-Bo2JQg8b.js';
-export { A as AnyDocument, B as BM25Params, C as CustomTokenizer, E as EmbeddingFieldConfig, F as FieldType, I as IndexConfig, a as InsertOptions, P as PartitionConfig, S as SchemaDefinition, b as ScoringMode } from './schema-DCgkO1j4.js';
+
+declare function registerTokenizer(name: string, tokenizer: CustomTokenizer): void;
+declare function registerStopWords(name: string, stopWords: StopWordOverride): void;
+declare function hasTokenizer(name: string): boolean;
+declare function hasStopWords(name: string): boolean;
+declare function getTokenizer(name: string): CustomTokenizer;
+declare function getStopWords(name: string): StopWordOverride;
 
 declare function configureNormalizationCache(maxSize: number): void;
 declare function clearNormalizationCache(): void;
@@ -16,4 +25,4 @@ declare function isSimdAvailable(): boolean;
 
 declare const VERSION = "0.1.0";
 
-export { LanguageModule, VERSION, clearNormalizationCache, configureNormalizationCache, getLanguage, getNormalizationCacheSize, isSimdAvailable, registerLanguage, resetNormalizationCache };
+export { CustomTokenizer, LanguageModule, StopWordOverride, VERSION, clearNormalizationCache, configureNormalizationCache, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, isSimdAvailable, registerLanguage, registerStopWords, registerTokenizer, resetNormalizationCache };

package/dist/index.mjs

@@ -1,7 +1,8 @@
-export { VERSION, createNarsil } from './chunk-MEW7HD75.mjs';
-export { clearNormalizationCache, configureNormalizationCache, getLanguage, getNormalizationCacheSize, registerLanguage, resetNormalizationCache } from './chunk-HMO6OMZ3.mjs';
+export { VERSION, createNarsil } from './chunk-XAKNSWIK.mjs';
+export { clearNormalizationCache, configureNormalizationCache, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, registerLanguage, registerStopWords, registerTokenizer, resetNormalizationCache } from './chunk-S6IAXOBA.mjs';
+import './chunk-TGP5M5R7.mjs';
 export { isSimdAvailable } from './chunk-S7RJJMHI.mjs';
 import './chunk-KMMJMDUA.mjs';
-import './chunk-TW6BXTOL.mjs';
-export { ErrorCodes, NarsilError } from './chunk-ALB35PRQ.mjs';
-import './chunk-HRABJLPB.mjs';
+import './chunk-U2BKYEC7.mjs';
+export { ErrorCodes, NarsilError } from './chunk-M2EDG55Y.mjs';
+import './chunk-BKAI2WXI.mjs';

package/dist/{language-ONQ3iVDf.d.ts → language-CmJBHf1j.d.ts}

@@ -1,7 +1,9 @@
 interface LanguageModule {
     name: string;
+    revision: string;
     stemmer: ((token: string) => string) | null;
     stopWords: Set<string>;
+    normalizer?: (token: string) => string;
     tokenizer?: TokenizerConfig;
 }
 interface TokenizerConfig {
@@ -9,6 +11,7 @@ interface TokenizerConfig {
     normalizeDiacritics?: boolean;
     minTokenLength?: number;
     stripPossessive?: boolean;
+    ngramSize?: number;
 }
 
-export type { LanguageModule as L };
+export type { LanguageModule as L, TokenizerConfig as T };

package/dist/languages/albanian.d.ts

@@ -0,0 +1,5 @@
+import { L as LanguageModule } from '../language-CmJBHf1j.js';
+
+declare const albanian: LanguageModule;
+
+export { albanian };

package/dist/languages/albanian.mjs

@@ -0,0 +1,103 @@
+// src/languages/albanian.ts
+var stopWords = /* @__PURE__ */ new Set([
+  "ai",
+  "ajo",
+  "as",
+  "asnj\xEB",
+  "ata",
+  "atje",
+  "ato",
+  "aty",
+  "brenda",
+  "\xE7do",
+  "\xE7far\xEB",
+  "cila",
+  "cil\xEBt",
+  "cili",
+  "deri",
+  "dhe",
+  "disa",
+  "do",
+  "e",
+  "edhe",
+  "\xEBsht\xEB",
+  "gjat\xEB",
+  "gjithashtu",
+  "gjith\xEB",
+  "i",
+  "im",
+  "ime",
+  "ishte",
+  "jan\xEB",
+  "jasht\xEB",
+  "jon\xEB",
+  "jote",
+  "ju",
+  "juaj",
+  "ka",
+  "kam",
+  "kan\xEB",
+  "ke",
+  "kemi",
+  "keni",
+  "k\xEBta",
+  "k\xEBto",
+  "k\xEBtu",
+  "kjo",
+  "ku",
+  "kur",
+  "ky",
+  "lart",
+  "mbi",
+  "me",
+  "m\xEB",
+  "midis",
+  "mos",
+  "nd\xEBr",
+  "nd\xEBrmjet",
+  "ne",
+  "n\xEB",
+  "n\xEBn",
+  "nga",
+  "nj\xEB",
+  "nuk",
+  "ose",
+  "pa",
+  "pak",
+  "para",
+  "pas",
+  "pastaj",
+  "p\xEBr",
+  "por",
+  "posht\xEB",
+  "prej",
+  "q\xEB",
+  "sa",
+  "saj",
+  "shum\xEB",
+  "si",
+  "sipas",
+  "tani",
+  "te",
+  "t\xEB",
+  "tek",
+  "ti",
+  "tij",
+  "tjet\xEBr",
+  "tyre",
+  "u",
+  "un\xEB",
+  "vet\xEB",
+  "vet\xEBm",
+  "veten",
+  "yt"
+]);
+var albanian = {
+  name: "albanian",
+  revision: "1",
+  stemmer: null,
+  stopWords,
+  tokenizer: { splitPattern: /[^a-z0-9çë]+/gi }
+};
+
+export { albanian };

package/dist/languages/amharic.d.ts

@@ -0,0 +1,5 @@
+import { L as LanguageModule } from '../language-CmJBHf1j.js';
+
+declare const amharic: LanguageModule;
+
+export { amharic };

package/dist/languages/amharic.mjs

@@ -0,0 +1,240 @@
+// src/languages/amharic.ts
+var stopWords = /* @__PURE__ */ new Set([
+  "\u1201\u1208\u1271\u1295\u121D",
+  "\u1201\u1209",
+  "\u1201\u120D",
+  "\u1201\u1294\u1273",
+  "\u1206\u1290",
+  "\u1206\u1290\u12CD",
+  "\u1206\u1291",
+  "\u1206\u1296\u121D",
+  "\u1208\u1206\u1290",
+  "\u1208\u1206\u1290\u12CD",
+  "\u1208\u1206\u1291",
+  "\u1208\u120C\u120E\u127D",
+  "\u1208\u121B\u1293\u1278\u12CD\u121D",
+  "\u1208\u121B\u1295\u129B\u12CD\u121D",
+  "\u1208\u1290\u12DA\u1205",
+  "\u1208\u12A5\u12EB\u1295\u12F3\u1295\u12F1",
+  "\u1208\u12DA\u1201",
+  "\u1208\u12DA\u1205",
+  "\u120A\u1206\u1295",
+  "\u120B\u12ED",
+  "\u120C\u120B",
+  "\u120C\u120E\u127D",
+  "\u120D\u12E9",
+  "\u1218\u1206\u1291",
+  "\u1218\u1206\u1291\u1295",
+  "\u1218\u1206\u1293\u1278\u12CD",
+  "\u1218\u1206\u1293\u1278\u12CD\u1295",
+  "\u1218\u1206\u1295",
+  "\u121B\u1208\u1271",
+  "\u121B\u1208\u1275",
+  "\u121B\u1293\u1278\u12CD\u121D",
+  "\u121B\u1293\u1278\u12CD\u1295\u121D",
+  "\u121B\u1295",
+  "\u121B\u1295\u121D",
+  "\u121B\u1295\u129B\u12CD\u121D",
+  "\u121B\u12F5\u1228\u130D",
+  "\u121D\u1295\u130A\u12DC\u121D",
+  "\u122B\u1233\u127D\u1201\u1295",
+  "\u122B\u1235\u1205\u1295",
+  "\u122B\u1235\u123D\u1295",
+  "\u1230\u121E\u1291\u1295",
+  "\u1230\u12A3\u1275",
+  "\u1232\u1206\u1291",
+  "\u1232\u1206\u1295",
+  "\u1232\u1206\u1295\u1293",
+  "\u1232\u120D",
+  "\u1232\u1263\u120D",
+  "\u1232\u1296\u122D",
+  "\u1233\u1208",
+  "\u1233\u12ED\u1206\u1295",
+  "\u1235\u1208\u1206\u1290",
+  "\u1235\u1208\u121B\u1293\u1278\u12CD\u121D",
+  "\u1235\u1208\u12DA\u1201",
+  "\u1235\u121D",
+  "\u1236\u1235\u1270\u129B",
+  "\u1241\u1325\u122D",
+  "\u1260\u1201\u1294\u1273\u12CD",
+  "\u1260\u1206\u1290",
+  "\u1260\u1206\u1291",
+  "\u1260\u120B\u12ED",
+  "\u1260\u120C\u120B",
+  "\u1260\u120C\u120E\u127D",
+  "\u1260\u1219\u1209",
+  "\u1260\u1219\u1209\u121D",
+  "\u1260\u121A\u120D",
+  "\u1260\u121A\u127D\u1209",
+  "\u1260\u121A\u1308\u1263",
+  "\u1260\u121B\u1240\u12F5",
+  "\u1260\u121B\u1293\u1278\u12CD\u121D",
+  "\u1260\u122D\u12AB\u1273",
+  "\u1260\u1230\u121E\u1291",
+  "\u1260\u1235\u1270\u1240\u122D",
+  "\u1260\u1240\u122D",
+  "\u1260\u1270\u1208\u12ED",
+  "\u1260\u1270\u1208\u12ED\u121D",
+  "\u1260\u1270\u1218\u1208\u12A8\u1270",
+  "\u1260\u1270\u1218\u1233\u1233\u12ED",
+  "\u1260\u1273\u127D",
+  "\u1260\u128B\u120B",
+  "\u1260\u1290\u1231",
+  "\u1260\u1290\u12DA\u1201",
+  "\u1260\u1290\u12DA\u1205",
+  "\u1260\u12A5\u1290\u12DA\u1201",
+  "\u1260\u12A5\u1295\u12F0\u12DA\u1205",
+  "\u1260\u12A9\u120D",
+  "\u1260\u12CB\u1293",
+  "\u1260\u12CD\u1235\u1325",
+  "\u1260\u12DA\u1205",
+  "\u1260\u12DA\u1205\u121D",
+  "\u1260\u12DA\u1205\u1293",
+  "\u1262\u1206\u1295",
+  "\u1262\u1206\u1295\u121D",
+  "\u1262\u12EB\u1295\u1235",
+  "\u1263\u1209",
+  "\u1263\u1323\u121D",
+  "\u1265\u1208\u12CB\u120D",
+  "\u1265\u120E",
+  "\u1265\u127B",
+  "\u1265\u12D9",
+  "\u1266\u1273",
+  "\u1270\u1263\u1208",
+  "\u1270\u1265\u120E",
+  "\u1270\u12A8\u1293\u12C8\u1290",
+  "\u1270\u1308\u1208\u1338",
+  "\u1270\u1308\u120D\u133F\u120D",
+  "\u1270\u1328\u121B\u122A",
+  "\u1275\u1293\u1295\u1275",
+  "\u127D\u130D\u122D",
+  "\u128B\u120B",
+  "\u1290\u1260\u1228\u127D",
+  "\u1290\u1260\u1229",
+  "\u1290\u1260\u122D",
+  "\u1290\u12CD",
+  "\u1290\u1308\u122D",
+  "\u1290\u1308\u122E\u127D",
+  "\u1293",
+  "\u1293\u1275",
+  "\u1293\u1278\u12CD",
+  "\u12A0\u1201\u1295",
+  "\u12A0\u1208",
+  "\u12A0\u1209",
+  "\u12A0\u1233\u1230\u1260",
+  "\u12A0\u1233\u1235\u1260\u12CB\u120D",
+  "\u12A0\u1235\u1228\u12F5\u1270\u12CB\u120D",
+  "\u12A0\u1235\u1273\u12CD\u1230\u12CB\u120D",
+  "\u12A0\u1235\u1273\u12CD\u1240\u12CB\u120D",
+  "\u12A0\u1235\u1308\u1290\u12D8\u1261",
+  "\u12A0\u1235\u1308\u1295\u12DD\u1260\u12CB\u120D",
+  "\u12A0\u1235\u1348\u120B\u130A",
+  "\u12A0\u1265\u122B\u122D\u1270\u12CB\u120D",
+  "\u12A0\u1295\u1270",
+  "\u12A0\u1295\u127A",
+  "\u12A0\u1295\u12F0\u1206\u1290",
+  "\u12A0\u1295\u12F0\u129B",
+  "\u12A0\u1295\u12F1",
+  "\u12A0\u1295\u12F1\u1295",
+  "\u12A0\u1295\u12F2\u1275",
+  "\u12A0\u1295\u12F3\u1295\u12F5",
+  "\u12A0\u1295\u133B\u122D",
+  "\u12A5\u1235\u12A8",
+  "\u12A5\u1235\u12AA\u12F0\u122D\u1235",
+  "\u12A5\u1235\u12AB\u1201\u1295",
+  "\u12A5\u1263\u12AD\u1205",
+  "\u12A5\u1263\u12AD\u123D",
+  "\u12A5\u1290\u12DA\u1205",
+  "\u12A5\u1290\u12DA\u1205\u121D",
+  "\u12A5\u1290\u12DA\u1205\u1291",
+  "\u12A5\u1290\u12DA\u1205\u1295",
+  "\u12A5\u1290\u12F0\u1206\u1290",
+  "\u12A5\u1293\u1295\u1270",
+  "\u12A5\u1295\u12B3",
+  "\u12A5\u1295\u12B3\u1295",
+  "\u12A5\u1295\u12F0",
+  "\u12A5\u1295\u12F0\u1206\u1290\u1293",
+  "\u12A5\u1295\u12F0\u1270\u1293\u1308\u1229\u1275",
+  "\u12A5\u1295\u12F0\u1290\u12DA\u1205",
+  "\u12A5\u1295\u12F0\u1308\u1208\u1339\u1275",
+  "\u12A5\u1295\u12F0\u1308\u1293",
+  "\u12A5\u1295\u12F2\u1201\u121D",
+  "\u12A5\u1295\u12F2\u1205",
+  "\u12A5\u1295\u12F2\u1206\u1291",
+  "\u12A5\u1295\u12F2\u1206\u1295",
+  "\u12A5\u1295\u12F3\u1235\u1228\u12F1\u1275",
+  "\u12A5\u1295\u1302",
+  "\u12A5\u12DA\u1201",
+  "\u12A5\u1305\u130D",
+  "\u12A8\u1201\u1208\u1275",
+  "\u12A8\u1206\u1290",
+  "\u12A8\u1206\u1290\u12CD",
+  "\u12A8\u120C\u120B",
+  "\u12A8\u120C\u120E\u127D",
+  "\u12A8\u1290\u12DA\u1201",
+  "\u12A8\u12A0\u1235\u122D",
+  "\u12A8\u12A0\u1295\u12F3\u1295\u12F5",
+  "\u12A8\u12A0\u1295\u12F5",
+  "\u12A8\u12A5\u1290\u12DA\u1201",
+  "\u12A8\u12A5\u1290\u12DA\u1205",
+  "\u12A8\u12DA\u1205",
+  "\u12A8\u12DA\u1205\u121D",
+  "\u12A8\u12DA\u12EB",
+  "\u12AB\u120D\u1206\u1290",
+  "\u12C8\u122D",
+  "\u12C8\u1245\u1275",
+  "\u12C8\u12ED",
+  "\u12C8\u12ED\u121D",
+  "\u12C8\u12F0",
+  "\u12CD\u1235\u1325",
+  "\u12E8\u1206\u1290",
+  "\u12E8\u1206\u1290\u127D\u1295",
+  "\u12E8\u1206\u1290\u12CD",
+  "\u12E8\u1206\u1291\u1275",
+  "\u12E8\u1208\u12CD\u121D",
+  "\u12E8\u120C\u120B",
+  "\u12E8\u120C\u120B\u1278\u12CD\u1295",
+  "\u12E8\u120C\u120B\u12CD\u1295",
+  "\u12E8\u121A\u1206\u1291\u1275",
+  "\u12E8\u121A\u127D\u1208\u12CD\u1295",
+  "\u12E8\u121A\u1308\u129D",
+  "\u12E8\u121B\u1293\u1278\u12CD\u121D",
+  "\u12E8\u121B\u12EB\u1295\u1235",
+  "\u12E8\u121B\u12ED\u1260\u120D\u1325",
+  "\u12E8\u1270\u1208\u12EB\u12E8",
+  "\u12E8\u1270\u1208\u12EB\u12E9",
+  "\u12E8\u1290\u1260\u1228\u12CD\u1295",
+  "\u12E8\u1290\u1308\u1229",
+  "\u12E8\u1293\u1295\u1270",
+  "\u12E8\u12A0\u1295\u1240\u1339",
+  "\u12E8\u12DA\u1201",
+  "\u12E8\u12DA\u1205",
+  "\u12EB\u1208",
+  "\u12EB\u1209",
+  "\u12EB\u120B\u1278\u12CD",
+  "\u12EB\u120B\u1278\u12CD\u1295",
+  "\u12EB\u120D\u1206\u1290",
+  "\u12EB\u1295\u1270",
+  "\u12EB\u1295\u127A",
+  "\u12ED\u1205",
+  "\u12ED\u1205\u121D",
+  "\u12ED\u1205\u127D\u12CD",
+  "\u12ED\u1205\u1295",
+  "\u12ED\u1205\u1295\u1291",
+  "\u12ED\u1205\u1295\u1295\u121D",
+  "\u12ED\u1206\u1291\u1260\u1273\u120D",
+  "\u12ED\u1206\u1293\u1209",
+  "\u12ED\u12B8\u12CD",
+  "\u130A\u12DC\u1293",
+  "\u130B\u122D",
+  "\u130D\u1295"
+]);
+var amharic = {
+  name: "amharic",
+  revision: "1",
+  stemmer: null,
+  stopWords,
+  tokenizer: { splitPattern: /[^\u1200-\u135f\u1369-\u137ca-z0-9]+/gi }
+};
+
+export { amharic };

package/dist/languages/arabic.d.ts

@@ -1,4 +1,4 @@
-import { L as LanguageModule } from '../language-ONQ3iVDf.js';
+import { L as LanguageModule } from '../language-CmJBHf1j.js';
 
 declare const arabic: LanguageModule;