indickit 0.5.0 → 0.8.0

package/dist/types/js/deromanize.d.ts

@@ -0,0 +1,57 @@
+/**
+ * Indian-language text from Latin typing, the reverse of romanize:
+ * namaste → नमस्ते, ahmad → احمد, vanakkam → வணக்கம்.
+ *
+ * A ranked list: one typed word can stand for several native words (kamal:
+ * कमल, कमाल), and the first is the most likely. Two modes: "words" for
+ * running text (the default) and "names" for fields known to hold person
+ * names.
+ *
+ * It writes the word it is given in the language's script, so English words
+ * in mixed text ("kal meeting hai") are written in it too: identify each
+ * word's language first.
+ *
+ * The tables are learned from data and loaded per language, on demand:
+ *
+ *     const d = await load("hi");   // fetches the Hindi and Brahmic files (beside this module, else jsDelivr)
+ *     d.word("namaste");            // ["नमस्ते", ...]
+ *
+ * The output is deterministic: the same input gives the same list in Go and
+ * in TypeScript.
+ *
+ * @module
+ */
+/** The tables: "words" for running text, "names" for person names. */
+export type Mode = "words" | "names";
+/** Changes whenever any output changes. */
+export declare const RULES_VERSION: string;
+/** The language codes that have tables, for one mode. */
+export declare function languages(mode?: Mode): string[];
+/** Fetches one data file's bytes. The default reads it beside this module. */
+export type Fetcher = (url: URL) => Promise<ArrayBuffer>;
+/** Where the tables are when they are not beside this module: jsDelivr, at this package's version. */
+export declare const CDN: string;
+/** One language's loaded tables. */
+export interface Deromanizer {
+    readonly lang: string;
+    readonly mode: Mode;
+    /** Up to n native spellings of one Latin-typed word (default 4), most likely first. */
+    word(latin: string, n?: number): string[];
+    /** Every run of Latin letters written as its first native spelling; everything else kept. */
+    text(text: string): string;
+}
+/** The input as the training pairs were read: accents removed, lower case, a-z only. */
+export declare function clean(s: string): string;
+/**
+ * Loads a language's tables for one mode: its own files, its word list and its
+ * script group's pooled files (shared by the group, fetched once). It reads them
+ * beside this module, else from jsDelivr at this package's version (the npm
+ * package carries no tables); `base` or `fetch` replaces both. Rejects when the
+ * language has no tables for the mode.
+ */
+export declare function load(lang: string, mode?: Mode, opts?: {
+    fetch?: Fetcher;
+    base?: URL | string;
+}): Promise<Deromanizer>;
+/** Builds a Deromanizer from its files, by file name (for tests and offline use). */
+export declare function fromBytes(lang: string, mode: Mode, files: Record<string, Uint8Array>): Deromanizer;

package/dist/types/js/phonetic-search.d.ts

@@ -0,0 +1,104 @@
+/**
+ * Phonetic search: the key finds candidates, and a scorer learned from data
+ * ranks them 0..100. A Latin query finds a name in any script; a name in one
+ * Indian script finds it in another.
+ *
+ *     const s = await loadSearch("hi");                 // fetches the Hindi table (a few KB)
+ *     const ix = s.index(["नरेंद्र मोदी", "नरेश मोदी"]);
+ *     ix.search("Narendra Modi", THRESHOLDS.names);     // [{ name: 0, score: 100 }]
+ *
+ * Two profiles: "names" (a list of names: a roster, a directory) and "text"
+ * (a name among the words of a running text). A score is relative to the
+ * other candidates of its search, so it belongs to the search, not to a pair.
+ * Every step is integer arithmetic: Go, TypeScript and the reference give the
+ * same scores.
+ *
+ * @module
+ */
+/** "names": a list of names; "text": a name among a text's words. */
+export type Profile = "names" | "text";
+type ProfileRules = {
+    key: number;
+    cost: number;
+    vowel: number;
+    div: number;
+    stop_permille: number | null;
+    stop_min_names: number;
+    round: boolean;
+};
+/** Changes whenever any score changes. */
+export declare const SEARCH_VERSION: string;
+/** The scores measured for each use: names in a list 74, villages 72, a name
+ * in text 80 (at most 10% of hits wrong) or 70 (20%). */
+export declare const THRESHOLDS: Readonly<Record<string, number>>;
+type Sparse = [number, [number, number][]];
+type RawTable = {
+    latin?: string[];
+    native?: string[];
+    x?: string[];
+    y?: string[];
+    c: Sparse[];
+    units?: [number, number, number, [number, number][]][];
+};
+declare class CostTable {
+    readonly idx: Map<string, number>;
+    readonly v: number;
+    readonly c: Float64Array;
+    readonly units: Map<number, Float64Array<ArrayBufferLike>>;
+    constructor(raw: RawTable);
+    ids(rs: string[]): number[];
+    align(a: number[], b: number[], withUnits: boolean): number;
+}
+/** Fetches one data file's bytes. The default reads it beside this module, else from jsDelivr. */
+export type Fetcher = (url: URL) => Promise<ArrayBuffer>;
+/** Where the tables are when they are not beside this module: jsDelivr, at this package's version. */
+export declare const CDN: string;
+declare class Scorer {
+    readonly lang: string;
+    private latin;
+    private pairs;
+    constructor(lang: string, latin: CostTable | undefined, pairs: Map<string, CostTable>);
+    cost(a: string, b: string): number | undefined;
+    calibrations(a: string, cands: string[]): Map<string, number>;
+    wordSim(a: string, b: string, cal: Map<string, number> | undefined, p: ProfileRules): number;
+}
+/** A name found by a search: its position among the indexed names, and its score. */
+export interface Hit {
+    name: number;
+    score: number;
+}
+/** Names of one language, keyed once for searching. */
+export declare class Index {
+    private readonly scorer;
+    private readonly words;
+    private readonly n;
+    private readonly df;
+    private readonly maps;
+    private readonly p;
+    /** @internal Use Search.index. */
+    constructor(scorer: Scorer, names: string[], profile: Profile);
+    private common;
+    private weight;
+    private candidates;
+    private nameScore;
+    /** @internal */
+    scoreAll(query: string, which: number[]): Hit[];
+    /** The names scoring at least threshold, best first (ties in index order). */
+    search(query: string, threshold?: number): Hit[];
+}
+/** A loaded language: index names, or score candidates. */
+export interface Search {
+    readonly lang: string;
+    /** Key names (of this language, or Latin) once for searching. */
+    index(names: string[], profile?: Profile): Index;
+    /** One score per candidate name, calibrated among these candidates. */
+    score(query: string, candidates: string[], profile?: Profile): number[];
+}
+/** Loads one language's table (Latin queries against its script) and, for
+ * native-to-native search, the tables between the scripts named in `scripts`
+ * (e.g. ["deva", "arab", "beng"]). A missing table falls back to the fine keys. */
+export declare function loadSearch(lang: string, options?: {
+    scripts?: string[];
+    fetcher?: Fetcher;
+}): Promise<Search>;
+export {};

package/dist/types/js/phonetic.d.ts

@@ -10,6 +10,13 @@
  *
  * @module
  */
+/** @internal For ./phonetic-search: the engine compiler, the normalizer and the
+ * search key's engine (keys of a normalized word). Not part of the API. */
+export declare const _internal: {
+    compile: (r: unknown) => (word: string) => string[];
+    normalize: (w: string) => string;
+    engine: (word: string) => string[];
+};
 /** The rules every key from this module was made with. */
 export declare const RULES_VERSION: string;
 /** NameKeys stops here: a long Tamil name can have many key combinations. */
@@ -23,6 +30,13 @@ export declare function words(name: string): string[];
  * separated by a space. Two names match when these share an element, so
  * these are what to index. At most MAX_NAME_KEYS. */
 export declare function nameKeys(name: string): string[];
+/** Joined keys shorter than this (in classes) are not returned: they find too many names. */
+export declare const JOINED_MIN_CLASSES: number;
+/** The keys of a name written as one word, its words joined: "Ram Nath" and
+ * இராம்நாத் share no nameKeys, but can share a joined key. Only keys of at
+ * least JOINED_MIN_CLASSES classes. A search looks them up only when
+ * nameKeys find nobody; match does not use them. */
+export declare function joinedKeys(name: string): string[];
 /** Whether two names have the same number of words and every pair of words,
  * in order, shares a key. */
 export declare function match(a: string, b: string): boolean;

package/dist/types/js/romanize.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Indian-language text in Latin letters, the way people spell it:
+ * लक्ष्मी → lakshmi, മലൈക → malaika, پرویز → parvez.
+ *
+ * A ranked list: one word has several accepted spellings (Choudhury,
+ * Chowdhury), and the first is the most likely. Two modes: "words" for
+ * running text (the default) and "names" for fields known to hold person
+ * names, where a lookup of known names comes first.
+ *
+ * The tables are learned from data and loaded per language, on demand:
+ *
+ *     const r = await load("hi");        // fetches the Hindi and Brahmic files (beside this module, else jsDelivr)
+ *     r.word("लक्ष्मी");                   // ["lakshmi", ...]
+ *
+ * The output is deterministic: the same input gives the same list in Go and
+ * in TypeScript.
+ *
+ * @module
+ */
+/** The tables: "words" for running text, "names" for person names. */
+export type Mode = "words" | "names";
+/** Changes whenever any output changes: store it beside romanized text. */
+export declare const RULES_VERSION: string;
+/** The language codes that have tables, for one mode. */
+export declare function languages(mode?: Mode): string[];
+/** Fetches one data file's bytes. The default reads it beside this module. */
+export type Fetcher = (url: URL) => Promise<ArrayBuffer>;
+/** Where the tables are when they are not beside this module: jsDelivr, at this package's version. */
+export declare const CDN: string;
+/** One language's loaded tables. */
+export interface Romanizer {
+    readonly lang: string;
+    readonly mode: Mode;
+    /** Up to n spellings of one word (default 4), most likely first. */
+    word(word: string, n?: number): string[];
+    /** Every run of the language's script romanized (its top spelling); everything else kept. */
+    text(text: string): string;
+}
+/**
+ * Loads a language's tables for one mode: its own file and its script
+ * group's pooled file (shared by the group, fetched once). It reads them
+ * beside this module, else from jsDelivr at this package's version (the npm
+ * package carries no tables); `base` or `fetch` replaces both. Rejects when
+ * the language has no tables for the mode.
+ */
+export declare function load(lang: string, mode?: Mode, opts?: {
+    fetch?: Fetcher;
+    base?: URL | string;
+}): Promise<Romanizer>;
+/** Builds a Romanizer from a language's two files (for tests and offline use). */
+export declare function fromBytes(lang: string, mode: Mode, own: Uint8Array, pool: Uint8Array): Romanizer;
+/** Moves every Brahmic code point to the Devanagari block by its offset (rules.json "unify"). */
+export declare function unify(w: string): string;
+/** @internal The model's own list for a unified word (no lookup), for the conformance test. */
+export declare function _decode(r: Romanizer, word: string, n: number): string[];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "indickit",
-  "version": "0.5.0",
+  "version": "0.8.0",
   "description": "Small, exact utilities for text in Indian languages: a phonetic key that matches a name across 21 Indian scripts and English, a normalizer that gives text that looks the same one encoding, a segmenter for the letters a reader sees, and a stemmer for search.",
   "license": "MIT",
   "packageManager": "bun@1.3.14",
@@ -25,6 +25,18 @@
     "./stem": {
       "types": "./dist/types/js/stem.d.ts",
       "default": "./dist/stem.js"
+    },
+    "./romanize": {
+      "types": "./dist/types/js/romanize.d.ts",
+      "default": "./dist/romanize.js"
+    },
+    "./phonetic-search": {
+      "types": "./dist/types/js/phonetic-search.d.ts",
+      "default": "./dist/phonetic-search.js"
+    },
+    "./deromanize": {
+      "types": "./dist/types/js/deromanize.d.ts",
+      "default": "./dist/deromanize.js"
     }
   },
   "files": [
@@ -32,7 +44,7 @@
   ],
   "sideEffects": false,
   "scripts": {
-    "build": "bun build js/phonetic.ts --outfile dist/phonetic.js --target browser --format esm && bun build js/normalize.ts --outfile dist/normalize.js --target browser --format esm && bun build js/segment.ts --outfile dist/segment.js --target browser --format esm && bun build js/stem.ts --outfile dist/stem.js --target browser --format esm && tsc -p tsconfig.json",
+    "build": "bun build js/phonetic.ts --outfile dist/phonetic.js --target browser --format esm && bun build js/normalize.ts --outfile dist/normalize.js --target browser --format esm && bun build js/segment.ts --outfile dist/segment.js --target browser --format esm && bun build js/stem.ts --outfile dist/stem.js --target browser --format esm && bun build js/romanize.ts --outfile dist/romanize.js --target browser --format esm && bun build js/phonetic-search.ts --outfile dist/phonetic-search.js --target browser --format esm && bun build js/deromanize.ts --outfile dist/deromanize.js --target browser --format esm && tsc -p tsconfig.json",
     "test": "bun test",
     "check-dist": "node js/check-dist.mjs"
   },