indickit 0.6.0 → 0.8.0

package/dist/phonetic.js

@@ -1,11 +1,14 @@
 // phonetic/rules.json
 var rules_default = {
-  version: "2026-10-06.1",
+  version: "2026-10-07",
   folds: [
     "bn-case",
+    "brahmic-av-u",
     "drop-vowels",
     "drop-y",
+    "flap-r",
     "gu-case",
+    "latin-ngh",
     "latin-w-u",
     "ml-case",
     "nasal-all",
@@ -100,7 +103,13 @@ var rules_default = {
     },
     virama_offset: 77,
     candrabindu_offset: 1,
-    candrabindu_class: "ṁ"
+    candrabindu_class: "ṁ",
+    nukta_offset: 60,
+    flap_offsets: [
+      33,
+      34
+    ],
+    flap_class: "ṛ"
   },
   extra: {
     ৎ: "t",
@@ -117,7 +126,14 @@ var rules_default = {
     ൿ: "k",
     ൔ: "m",
     ൕ: "y",
-    ൖ: "l"
+    ൖ: "l",
+    ड़: "ṛ",
+    ढ़: "ṛ",
+    ড়: "ṛ",
+    ঢ়: "ṛ",
+    ଡ଼: "ṛ",
+    ଢ଼: "ṛ",
+    ੜ: "ṛ"
   },
   urdu: {
     from: 1536,
@@ -139,7 +155,7 @@ var rules_default = {
       ڈ: "ṭ",
       ذ: "c",
       ر: "r",
-      ڑ: "ṭ",
+      ڑ: "ṛ",
       ز: "c",
       ژ: "c",
       س: "s",
@@ -181,7 +197,7 @@ var rules_default = {
       ڌ: "t",
       ڍ: "ṭ",
       ڏ: "t",
-      ڙ: "ṭ",
+      ڙ: "ṛ",
       ڦ: "p",
       ڪ: "k",
       ڱ: "ṅ",
@@ -301,7 +317,7 @@ var rules_default = {
       ᱯ: "p",
       ᱰ: "ṭ",
       ᱱ: "n",
-      ᱲ: "ṭ",
+      ᱲ: "ṛ",
       ᱳ: "o",
       ᱴ: "ṭ",
       ᱵ: "p",
@@ -313,6 +329,10 @@ var rules_default = {
   },
   latin: {
     groups: [
+      [
+        "ngh",
+        "nĝ"
+      ],
       [
         "chh",
         "c"
@@ -420,9 +440,11 @@ var rules_default = {
     v: "p",
     w: "p",
     ñ: "n",
+    ĝ: "k",
     ṁ: "n",
     ṅ: "n",
     ṇ: "n",
+    ṛ: "t",
     ṭ: "t"
   },
   vowels: "aeiou",
@@ -436,6 +458,26 @@ var rules_default = {
     "drop-final-vowel": false
   },
   branches: [
+    {
+      from: 2304,
+      to: 3455,
+      class: "v",
+      also: "u",
+      after: "aṁ",
+      not_before: "aieo"
+    },
+    {
+      from: 1536,
+      to: 7295,
+      class: "ṛ",
+      also: "r"
+    },
+    {
+      from: 0,
+      to: 127,
+      class: "ĝ",
+      also: "h"
+    },
     {
       from: 0,
       to: 127,
@@ -455,6 +497,10 @@ var rules_default = {
       also: "u"
     }
   ],
+  joined: {
+    min_classes: 8,
+    when: "the strict keys find nobody"
+  },
   digit_zeros: [
     48,
     1632,
@@ -565,6 +611,7 @@ var rules_default = {
 };
 
 // js/phonetic.ts
+var DELETED = "\x00";
 function compile(rules) {
   const b = rules.brahmic;
   const byOffset = new Array(128);
@@ -588,26 +635,31 @@ function compile(rules) {
   }
   function readBrahmic(word) {
     let out = "";
+    let last = -1;
     for (let i = 0;i < word.length; i++) {
       const ch = word[i];
       const x = extra.get(ch);
       if (x !== undefined) {
         out += x;
+        last = -1;
         continue;
       }
       const o = word.charCodeAt(i);
       if (o < b.from || o > b.to)
         continue;
       const off = o & 127;
-      if (off === b.virama_offset)
-        continue;
-      if (off === b.candrabindu_offset) {
+      const c = byOffset[off];
+      if (off === b.nukta_offset && b.flap_offsets.includes(last) && out) {
+        out = out.slice(0, -1) + b.flap_class;
+      } else if (off === b.virama_offset) {} else if (off === b.candrabindu_offset) {
         out += b.candrabindu_class;
-        continue;
+      } else if (c !== undefined) {
+        out += c;
       }
-      const c = byOffset[off];
       if (c !== undefined)
-        out += c;
+        last = off;
+      else if (off !== b.nukta_offset && off !== b.virama_offset)
+        last = -1;
     }
     return out;
   }
@@ -728,6 +780,11 @@ function compile(rules) {
         runs += out[i];
     return runs;
   }
+  function fits(classes, i, br) {
+    const prev = i > 0 ? classes[i - 1] : "^";
+    const next = i + 1 < classes.length ? classes[i + 1] : "$";
+    return (br.after === undefined || br.after.includes(prev)) && (br.before === undefined || br.before.includes(next)) && !(br.not_before ?? "").includes(next);
+  }
   function variants(word) {
     const classes = read(word);
     let vs = [classes];
@@ -745,12 +802,13 @@ function compile(rules) {
       if (!inScript)
         continue;
       for (let i = 0;i < classes.length; i++) {
-        if (classes[i] === br.class && vs.length * 2 <= rules.max_keys) {
-          vs = vs.concat(vs.map((v) => v.slice(0, i) + br.also + v.slice(i + 1)));
+        if (classes[i] === br.class && vs.length * 2 <= rules.max_keys && fits(classes, i, br)) {
+          const also = br.also || DELETED;
+          vs = vs.concat(vs.map((v) => v.slice(0, i) + also + v.slice(i + 1)));
         }
       }
     }
-    return vs;
+    return vs.map((v) => v.replaceAll(DELETED, ""));
   }
   function number(word) {
     let out = "";
@@ -788,6 +846,7 @@ function compile(rules) {
   };
 }
 var engine = compile(rules_default);
+var _internal = { compile: (r) => compile(r), normalize: (w) => normalize(w), engine };
 var RULES_VERSION = rules_default.version;
 var MAX_NAME_KEYS = 256;
 function normalize(word) {
@@ -817,6 +876,13 @@ function nameKeys(name) {
   }
   return out[0] === "" ? [] : out;
 }
+var JOINED_MIN_CLASSES = rules_default.joined.min_classes;
+function joinedKeys(name) {
+  const joined = words(name).map(normalize).join("");
+  if (!joined)
+    return [];
+  return engine(joined).filter((k) => [...k].length >= JOINED_MIN_CLASSES);
+}
 function match(a, b) {
   const wa = words(a), wb = words(b);
   if (!wa.length || wa.length !== wb.length)
@@ -831,6 +897,9 @@ export {
   nameKeys,
   match,
   keys,
+  joinedKeys,
+  _internal,
   RULES_VERSION,
-  MAX_NAME_KEYS
+  MAX_NAME_KEYS,
+  JOINED_MIN_CLASSES
 };

package/dist/romanize.js

@@ -383,7 +383,7 @@ var rules_default = {
   version: 1
 };
 // package.json
-var version = "0.6.0";
+var version = "0.8.0";
 
 // js/lang.ts
 var ISO_639_2 = "asm as ben bn guj gu hin hi kan kn kas ks mal ml mar mr nep ne ori or pan pa san sa snd sd tam ta tel te urd ur ";

package/dist/types/js/deromanize.d.ts

@@ -0,0 +1,57 @@
+/**
+ * Indian-language text from Latin typing, the reverse of romanize:
+ * namaste → नमस्ते, ahmad → احمد, vanakkam → வணக்கம்.
+ *
+ * A ranked list: one typed word can stand for several native words (kamal:
+ * कमल, कमाल), and the first is the most likely. Two modes: "words" for
+ * running text (the default) and "names" for fields known to hold person
+ * names.
+ *
+ * It writes the word it is given in the language's script, so English words
+ * in mixed text ("kal meeting hai") are written in it too: identify each
+ * word's language first.
+ *
+ * The tables are learned from data and loaded per language, on demand:
+ *
+ *     const d = await load("hi");   // fetches the Hindi and Brahmic files (beside this module, else jsDelivr)
+ *     d.word("namaste");            // ["नमस्ते", ...]
+ *
+ * The output is deterministic: the same input gives the same list in Go and
+ * in TypeScript.
+ *
+ * @module
+ */
+/** The tables: "words" for running text, "names" for person names. */
+export type Mode = "words" | "names";
+/** Changes whenever any output changes. */
+export declare const RULES_VERSION: string;
+/** The language codes that have tables, for one mode. */
+export declare function languages(mode?: Mode): string[];
+/** Fetches one data file's bytes. The default reads it beside this module. */
+export type Fetcher = (url: URL) => Promise<ArrayBuffer>;
+/** Where the tables are when they are not beside this module: jsDelivr, at this package's version. */
+export declare const CDN: string;
+/** One language's loaded tables. */
+export interface Deromanizer {
+    readonly lang: string;
+    readonly mode: Mode;
+    /** Up to n native spellings of one Latin-typed word (default 4), most likely first. */
+    word(latin: string, n?: number): string[];
+    /** Every run of Latin letters written as its first native spelling; everything else kept. */
+    text(text: string): string;
+}
+/** The input as the training pairs were read: accents removed, lower case, a-z only. */
+export declare function clean(s: string): string;
+/**
+ * Loads a language's tables for one mode: its own files, its word list and its
+ * script group's pooled files (shared by the group, fetched once). It reads them
+ * beside this module, else from jsDelivr at this package's version (the npm
+ * package carries no tables); `base` or `fetch` replaces both. Rejects when the
+ * language has no tables for the mode.
+ */
+export declare function load(lang: string, mode?: Mode, opts?: {
+    fetch?: Fetcher;
+    base?: URL | string;
+}): Promise<Deromanizer>;
+/** Builds a Deromanizer from its files, by file name (for tests and offline use). */
+export declare function fromBytes(lang: string, mode: Mode, files: Record<string, Uint8Array>): Deromanizer;

package/dist/types/js/phonetic-search.d.ts

@@ -0,0 +1,104 @@
+/**
+ * Phonetic search: the key finds candidates, and a scorer learned from data
+ * ranks them 0..100. A Latin query finds a name in any script; a name in one
+ * Indian script finds it in another.
+ *
+ *     const s = await loadSearch("hi");                 // fetches the Hindi table (a few KB)
+ *     const ix = s.index(["नरेंद्र मोदी", "नरेश मोदी"]);
+ *     ix.search("Narendra Modi", THRESHOLDS.names);     // [{ name: 0, score: 100 }]
+ *
+ * Two profiles: "names" (a list of names: a roster, a directory) and "text"
+ * (a name among the words of a running text). A score is relative to the
+ * other candidates of its search, so it belongs to the search, not to a pair.
+ * Every step is integer arithmetic: Go, TypeScript and the reference give the
+ * same scores.
+ *
+ * @module
+ */
+/** "names": a list of names; "text": a name among a text's words. */
+export type Profile = "names" | "text";
+type ProfileRules = {
+    key: number;
+    cost: number;
+    vowel: number;
+    div: number;
+    stop_permille: number | null;
+    stop_min_names: number;
+    round: boolean;
+};
+/** Changes whenever any score changes. */
+export declare const SEARCH_VERSION: string;
+/** The scores measured for each use: names in a list 74, villages 72, a name
+ * in text 80 (at most 10% of hits wrong) or 70 (20%). */
+export declare const THRESHOLDS: Readonly<Record<string, number>>;
+type Sparse = [number, [number, number][]];
+type RawTable = {
+    latin?: string[];
+    native?: string[];
+    x?: string[];
+    y?: string[];
+    c: Sparse[];
+    units?: [number, number, number, [number, number][]][];
+};
+declare class CostTable {
+    readonly idx: Map<string, number>;
+    readonly v: number;
+    readonly c: Float64Array;
+    readonly units: Map<number, Float64Array<ArrayBufferLike>>;
+    constructor(raw: RawTable);
+    ids(rs: string[]): number[];
+    align(a: number[], b: number[], withUnits: boolean): number;
+}
+/** Fetches one data file's bytes. The default reads it beside this module, else from jsDelivr. */
+export type Fetcher = (url: URL) => Promise<ArrayBuffer>;
+/** Where the tables are when they are not beside this module: jsDelivr, at this package's version. */
+export declare const CDN: string;
+declare class Scorer {
+    readonly lang: string;
+    private latin;
+    private pairs;
+    constructor(lang: string, latin: CostTable | undefined, pairs: Map<string, CostTable>);
+    cost(a: string, b: string): number | undefined;
+    calibrations(a: string, cands: string[]): Map<string, number>;
+    wordSim(a: string, b: string, cal: Map<string, number> | undefined, p: ProfileRules): number;
+}
+/** A name found by a search: its position among the indexed names, and its score. */
+export interface Hit {
+    name: number;
+    score: number;
+}
+/** Names of one language, keyed once for searching. */
+export declare class Index {
+    private readonly scorer;
+    private readonly words;
+    private readonly n;
+    private readonly df;
+    private readonly maps;
+    private readonly p;
+    /** @internal Use Search.index. */
+    constructor(scorer: Scorer, names: string[], profile: Profile);
+    private common;
+    private weight;
+    private candidates;
+    private nameScore;
+    /** @internal */
+    scoreAll(query: string, which: number[]): Hit[];
+    /** The names scoring at least threshold, best first (ties in index order). */
+    search(query: string, threshold?: number): Hit[];
+}
+/** A loaded language: index names, or score candidates. */
+export interface Search {
+    readonly lang: string;
+    /** Key names (of this language, or Latin) once for searching. */
+    index(names: string[], profile?: Profile): Index;
+    /** One score per candidate name, calibrated among these candidates. */
+    score(query: string, candidates: string[], profile?: Profile): number[];
+}
+/** Loads one language's table (Latin queries against its script) and, for
+ * native-to-native search, the tables between the scripts named in `scripts`
+ * (e.g. ["deva", "arab", "beng"]). A missing table falls back to the fine keys. */
+export declare function loadSearch(lang: string, options?: {
+    scripts?: string[];
+    fetcher?: Fetcher;
+}): Promise<Search>;
+export {};

package/dist/types/js/phonetic.d.ts

@@ -10,6 +10,13 @@
  *
  * @module
  */
+/** @internal For ./phonetic-search: the engine compiler, the normalizer and the
+ * search key's engine (keys of a normalized word). Not part of the API. */
+export declare const _internal: {
+    compile: (r: unknown) => (word: string) => string[];
+    normalize: (w: string) => string;
+    engine: (word: string) => string[];
+};
 /** The rules every key from this module was made with. */
 export declare const RULES_VERSION: string;
 /** NameKeys stops here: a long Tamil name can have many key combinations. */
@@ -23,6 +30,13 @@ export declare function words(name: string): string[];
  * separated by a space. Two names match when these share an element, so
  * these are what to index. At most MAX_NAME_KEYS. */
 export declare function nameKeys(name: string): string[];
+/** Joined keys shorter than this (in classes) are not returned: they find too many names. */
+export declare const JOINED_MIN_CLASSES: number;
+/** The keys of a name written as one word, its words joined: "Ram Nath" and
+ * இராம்நாத் share no nameKeys, but can share a joined key. Only keys of at
+ * least JOINED_MIN_CLASSES classes. A search looks them up only when
+ * nameKeys find nobody; match does not use them. */
+export declare function joinedKeys(name: string): string[];
 /** Whether two names have the same number of words and every pair of words,
  * in order, shares a key. */
 export declare function match(a: string, b: string): boolean;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "indickit",
-  "version": "0.6.0",
+  "version": "0.8.0",
   "description": "Small, exact utilities for text in Indian languages: a phonetic key that matches a name across 21 Indian scripts and English, a normalizer that gives text that looks the same one encoding, a segmenter for the letters a reader sees, and a stemmer for search.",
   "license": "MIT",
   "packageManager": "bun@1.3.14",
@@ -29,6 +29,14 @@
     "./romanize": {
       "types": "./dist/types/js/romanize.d.ts",
       "default": "./dist/romanize.js"
+    },
+    "./phonetic-search": {
+      "types": "./dist/types/js/phonetic-search.d.ts",
+      "default": "./dist/phonetic-search.js"
+    },
+    "./deromanize": {
+      "types": "./dist/types/js/deromanize.d.ts",
+      "default": "./dist/deromanize.js"
     }
   },
   "files": [
@@ -36,7 +44,7 @@
   ],
   "sideEffects": false,
   "scripts": {
-    "build": "bun build js/phonetic.ts --outfile dist/phonetic.js --target browser --format esm && bun build js/normalize.ts --outfile dist/normalize.js --target browser --format esm && bun build js/segment.ts --outfile dist/segment.js --target browser --format esm && bun build js/stem.ts --outfile dist/stem.js --target browser --format esm && bun build js/romanize.ts --outfile dist/romanize.js --target browser --format esm && tsc -p tsconfig.json",
+    "build": "bun build js/phonetic.ts --outfile dist/phonetic.js --target browser --format esm && bun build js/normalize.ts --outfile dist/normalize.js --target browser --format esm && bun build js/segment.ts --outfile dist/segment.js --target browser --format esm && bun build js/stem.ts --outfile dist/stem.js --target browser --format esm && bun build js/romanize.ts --outfile dist/romanize.js --target browser --format esm && bun build js/phonetic-search.ts --outfile dist/phonetic-search.js --target browser --format esm && bun build js/deromanize.ts --outfile dist/deromanize.js --target browser --format esm && tsc -p tsconfig.json",
     "test": "bun test",
     "check-dist": "node js/check-dist.mjs"
   },