indickit 0.4.3 → 0.6.0

package/dist/stem.js

@@ -7,6 +7,14 @@ var rules_default = {
   languages: { as: { vowel_step: true, endings: ["ও", "ক", "কক", "কত", "কে", "কৰ", "খন", "খনৰ", "গত", "ছিল", "ছে", "জন", "টো", "টোক", "টোত", "টোৰ", "টোৱে", "ত", "তত", "তে", "তো", "তৰ", "বলৈ", "বোৰ", "বোৰৰ", "মূলক", "য়ে", "লৈ", "লৈকে", "সকলক", "সকলে", "সকলৰ", "সমূহ", "সমূহক", "সমূহে", "সমূহৰ", "াই", "াক", "াব", "ায়", "ালে", "াৰ", "াৰত", "াৰে", "িব", "িবলৈ", "িয়ে", "িলে", "িৰ", "ীয়", "ীয়ে", "ীৰ", "েৰে", "োৱা", "োৱে", "্ত", "ৰ", "ৰক", "ৰত", "ৰলৈ", "ৰি", "ৰে", "ৰো", "ৰৰ"] }, bn: { vowel_step: true, endings: ["ও", "করণ", "গত", "গুলি", "গুলিকে", "গুলিতে", "গুলির", "ছে", "টার", "তে", "দের", "মূলক", "রা", "রার", "াই", "াকে", "াটি", "ানো", "ায়", "ার", "ারকে", "ারে", "ারের", "িকে", "িয়ে", "ির", "ীর", "ুর", "েন", "ের"] }, gu: { vowel_step: false, endings: ["ં", "એ", "ઓ", "ણા", "થી", "ના", "ની", "નું", "ને", "નો", "માં", "વા", "વાની", "વાનું", "વાનો", "વામાં", "વી", "વું", "શે", "ા", "ાં", "ાએ", "ાઓ", "ાઓની", "ાઓને", "ાઓમાં", "ાના", "ાની", "ાનું", "ાને", "ાનો", "ામાં", "ાય", "િત", "ી", "ીએ", "ીઓ", "ીઓને", "ીના", "ીની", "ીને", "ીમાં", "ીય", "ુ", "ું", "ે", "ો", "ોએ", "ોના", "ોની", "ોનું", "ોને", "ોનો", "ોમાં", "્યા", "્યું", "્સ"] }, hi: { vowel_step: true, endings: ["ं", "एँ", "एं", "ओं", "कर", "ता", "ताओं", "ती", "ते", "ना", "नी", "ने", "पूर्ण", "यां", "ाना", "ाने", "ाया", "ियाँ", "ियों", "्"] }, kn: { vowel_step: true, endings: ["ಕ್ಕಾಗಿ", "ಕ್ಕೆ", "ಗಳ", "ಗಳನ್ನು", "ಗಳಲ್ಲಿ", "ಗಳಿಂದ", "ಗಳಿಗೆ", "ಗಳು", "ಗಾಗಿ", "ಗೆ", "ತ್ತವೆ", "ತ್ತಾನೆ", "ತ್ತಾರೆ", "ದ", "ದನು", "ದರು", "ದಲ್ಲಿ", "ದವು", "ದಿಂದ", "ದ್ದಾರೆ", "ನನ್ನು", "ನು", "ನ್ನು", "ಬೇಕು", "ಯನ್ನು", "ಯಲ್ಲಿ", "ಯಾಗಿ", "ಯಿಂದ", "ಯಿತು", "ಯು", "ರೆ", "ಲಿನ", "ಲು", "ಲ್ಲಿ", "ವನ್ನು", "ವರು", "ವಾಗಿ", "ವಾಗಿದೆ", "ವಾದ", "ವು", "ಾಗಿ", "ಾಗಿದೆ", "ಿಂದ", "ಿಗೂ", "ಿತ", "ಿತು", "ಿದೆ", "ಿನ", "ಿನಿಂದ", "ಿಯ", "ುತ್ತದೆ", "ುವ", "ುವುದು", "ೆಯ", "ೊಂದಿಗೆ", "್"] }, ml: { vowel_step: true, endings: ["ം", "കയും", "കളിലും", "കളിൽ", "കളും", "കളുടെ", "കളെ", "കൾ", "കൾക്ക്", "ങ്ങളിൽ", "ങ്ങളും", "ങ്ങളെ", "ങ്ങൾ", "ച്ച്", "ത്തിന്", "ത്തിലെ", "ത്തിൽ", "നും", "നുള്ള", "നെ", "ന്", "ന്ന", "ന്നത്", "ന്റെ", "മാണ്", "മായ", "മായി", "മാർ", "മുള്ള", "യാണ്", "യായി", "യിലും", "യിലെ", "യിൽ", "യും", "യുടെ", "യെ", "യ്ക്ക്", "രുടെ", "രെ", "റെ", "ല്ല", "ള്ള", "വും", "വ്", "ായി", "ിക്ക്", "ിലുള്ള", "ിലെ", "ിൽ", "ുക", "ുന്നു", "്", "്ടു", "്സ്", "ൺ", "ൻ", "ർക്ക്"] }, mr: { vowel_step: false, endings: ["ं", "चा", "ची", "चे", "च्या", "ने", "पणे", "मधील", "मध्ये", "ही", "ा", "ांचा", "ांची", "ांचे", "ांच्या", "ांत", "ांना", "ांनी", "ांवर", "ांशी", "ांसाठी", "ाचा", "ाची", "ाचे", "ाच्या", "ात", "ातील", "ातून", "ानंतर", "ाना", "ाने", "ाला", "ावर", "ाशी", "ासाठी", "ित", "ी", "ीचा", "ीची", "ीचे", "ीच्या", "ीत", "ीतील", "ीने", "ीमध्ये", "ीय", "ील", "ीला", "ीसाठी", "े", "ेचा", "ेची", "ेचे", "ेच्या", "ेत", "ेने", "ेला", "ेवर", "ो", "्य", "्या", "्यांचा", "्यांची", "्यांना", "्याचा", "्याची", "्याचे", "्यात", "्याने", "्याला", "्ये"] }, ne: { vowel_step: true, endings: ["ं", "का", "की", "को", "देखि", "द्वारा", "बाट", "भन्दा", "मा", "माथि", "सँग", "सम्म", "हरूका", "हरूको", "हरूमा", "हरूलाई", "हरूले", "ामा", "ालाई", "ाले", "ाहरू", "िएका", "ित", "िन", "िने", "िन्", "िहरू", "ीमा", "ीलाई", "ीले", "ीहरू", "ेका", "ेन", "ेर", "्छ", "्छन्", "्दा", "्दै", "्न", "्नु", "्ने", "्यो"] }, pa: { vowel_step: true, endings: ["ਆਂ", "ਇਆ", "ਈ", "ਈਆਂ", "ਉਂਦਾ", "ਉਂਦੇ", "ਉਣ", "ਉਣਾ", "ਏ", "ਕ", "ਕਤਾ", "ਕਰਨ", "ਕਾਂ", "ਣ", "ਣਾ", "ਣਾਂ", "ਤਾ", "ਨ", "ਨਾ", "ਨਾਂ", "਼", "਼ਾਂ", "਼ੀ", "ਾਂ", "ਾਏ", "ਾਨ", "ਾਵਾਂ", "ਿਆ", "ਿਤ", "ੀਂ", "ੀਆ", "ੇਂ", "ੋਂ"] }, sa: { vowel_step: false, endings: ["ं", "ः", "त्", "द्", "न्", "म्", "य", "या", "याः", "याम्", "यै", "योः", "षु", "सु", "स्य", "ा", "ाः", "ाणि", "ात्", "ानाम्", "ानि", "ान्", "ाम्", "ाय", "ायाः", "ायाम्", "ासु", "िनः", "ी", "े", "ेण", "ेन", "ेभ्यः", "ेषु", "ैः", "ौ"] }, ta: { vowel_step: true, endings: ["களால்", "களின்", "களில்", "களுடன்", "களை", "களைப்", "கள்", "கின்றன", "கிறார்", "க்", "க்கள்", "க்காக", "க்கான", "ச்", "த்", "ன", "னர்", "னார்", "னால்", "னின்", "ன்", "ப்", "ப்பட்ட", "மான", "ய", "யாக", "யான", "யால்", "யின்", "யில்", "யுடன்", "யே", "யை", "ய்", "ற்கு", "லாம்", "லின்", "வது", "வர்", "வர்கள்", "வின்", "வில்", "வில்லை", "ாக", "ாத", "ார்", "ார்கள்", "ால்", "ாவின்", "ிக்கு", "ின்", "ிய", "ியது", "ியில்", "ிறது", "ில்", "ுகிறது", "ுக்கு", "ுடன்", "ுதல்", "ும்", "ுள்ள", "ுள்ளது", "ைக்கு", "ையில்", "ையும்", "ோடு", "்டது"] }, te: { vowel_step: true, endings: ["ం", "ంగా", "ంతో", "ందని", "ంది", "ంలో", "ంలోని", "గా", "టు", "డు", "తో", "నాడు", "ని", "నికి", "ను", "న్న", "న్నాడు", "న్నాయి", "న్నారు", "పై", "మని", "ము", "మైన", "లకు", "లతో", "లను", "లపై", "లలో", "లు", "లో", "లోకి", "లోని", "ల్లో", "ాను", "ాన్ని", "ాయి", "ారు", "ించారు", "ించిన", "ికి", "ుడు", "ుతో", "ుని", "ుల", "్", "్న", "్‌లో"] }, ur: { vowel_step: false, endings: ["ئی", "ئیں", "ئے", "اء", "ائی", "ات", "اں", "تا", "تہ", "تی", "تے", "را", "ری", "ریوں", "رے", "ز", "نا", "نی", "نیوں", "نے", "و", "وں", "ِ", "ٰ", "ٹنگ", "ں", "ہ", "ہوں", "یا", "یت", "ین", "ینی", "یں", "ے"] } }
 };
 
+// js/lang.ts
+var ISO_639_2 = "asm as ben bn guj gu hin hi kan kn kas ks mal ml mar mr nep ne ori or pan pa san sa snd sd tam ta tel te urd ur ";
+function langCode(tag) {
+  const t = (tag ?? "").toLowerCase().split(/[-_]/)[0];
+  const i = t.length === 3 ? ISO_639_2.indexOf(t + " ") : -1;
+  return i >= 0 && i % 7 === 0 ? ISO_639_2.slice(i + 4, i + 6) : t;
+}
+
 // js/stem.ts
 var MIN_STEM = rules_default.min_stem;
 var MAX_END = rules_default.max_end;
@@ -18,7 +26,7 @@ var TABLES = new Map(Object.entries(rules_default.languages).map(([lang, l]) =>
 var RULES_VERSION = rules_default.version;
 var LANGUAGES = [...TABLES.keys()].sort();
 function stem(word, lang) {
-  const t = TABLES.get(lang);
+  const t = TABLES.get(langCode(lang));
   if (!t)
     return word;
   const r = Array.from(word);

package/dist/types/js/lang.d.ts

@@ -0,0 +1 @@
+export declare function langCode(tag: string | undefined): string;

package/dist/types/js/normalize.d.ts

@@ -9,7 +9,8 @@
  * information on purpose: apply it to a query and to an index, never to
  * stored text.
  *
- * `lang` is a language code ("as", "hi", ...). Assamese text needs it: after
+ * `lang` is a language tag ("as", "hi", "as-IN", "asm"); only its language
+ * counts, not case, region or script. Assamese text needs it: after
  * a virama, Assamese ৰ and Bengali র look alike, and each language keeps its
  * own.
  *

package/dist/types/js/romanize.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Indian-language text in Latin letters, the way people spell it:
+ * लक्ष्मी → lakshmi, മലൈക → malaika, پرویز → parvez.
+ *
+ * A ranked list: one word has several accepted spellings (Choudhury,
+ * Chowdhury), and the first is the most likely. Two modes: "words" for
+ * running text (the default) and "names" for fields known to hold person
+ * names, where a lookup of known names comes first.
+ *
+ * The tables are learned from data and loaded per language, on demand:
+ *
+ *     const r = await load("hi");        // fetches the Hindi and Brahmic files (beside this module, else jsDelivr)
+ *     r.word("लक्ष्मी");                   // ["lakshmi", ...]
+ *
+ * The output is deterministic: the same input gives the same list in Go and
+ * in TypeScript.
+ *
+ * @module
+ */
+/** The tables: "words" for running text, "names" for person names. */
+export type Mode = "words" | "names";
+/** Changes whenever any output changes: store it beside romanized text. */
+export declare const RULES_VERSION: string;
+/** The language codes that have tables, for one mode. */
+export declare function languages(mode?: Mode): string[];
+/** Fetches one data file's bytes. The default reads it beside this module. */
+export type Fetcher = (url: URL) => Promise<ArrayBuffer>;
+/** Where the tables are when they are not beside this module: jsDelivr, at this package's version. */
+export declare const CDN: string;
+/** One language's loaded tables. */
+export interface Romanizer {
+    readonly lang: string;
+    readonly mode: Mode;
+    /** Up to n spellings of one word (default 4), most likely first. */
+    word(word: string, n?: number): string[];
+    /** Every run of the language's script romanized (its top spelling); everything else kept. */
+    text(text: string): string;
+}
+/**
+ * Loads a language's tables for one mode: its own file and its script
+ * group's pooled file (shared by the group, fetched once). It reads them
+ * beside this module, else from jsDelivr at this package's version (the npm
+ * package carries no tables); `base` or `fetch` replaces both. Rejects when
+ * the language has no tables for the mode.
+ */
+export declare function load(lang: string, mode?: Mode, opts?: {
+    fetch?: Fetcher;
+    base?: URL | string;
+}): Promise<Romanizer>;
+/** Builds a Romanizer from a language's two files (for tests and offline use). */
+export declare function fromBytes(lang: string, mode: Mode, own: Uint8Array, pool: Uint8Array): Romanizer;
+/** Moves every Brahmic code point to the Devanagari block by its offset (rules.json "unify"). */
+export declare function unify(w: string): string;
+/** @internal The model's own list for a unified word (no lookup), for the conformance test. */
+export declare function _decode(r: Romanizer, word: string, n: number): string[];

package/dist/types/js/stem.d.ts

@@ -19,6 +19,7 @@ export declare const RULES_VERSION: string;
 export declare const LANGUAGES: readonly string[];
 /**
  * The search key of `word` in language `lang` (as bn gu hi kn ml mr ne pa
- * sa ta te ur). For any other `lang`, `word` unchanged.
+ * sa ta te ur). Only the tag's language counts: "ta-IN", "TA" and "tam" are
+ * "ta". For any other `lang`, `word` unchanged.
  */
 export declare function stem(word: string, lang: string): string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "indickit",
-  "version": "0.4.3",
+  "version": "0.6.0",
   "description": "Small, exact utilities for text in Indian languages: a phonetic key that matches a name across 21 Indian scripts and English, a normalizer that gives text that looks the same one encoding, a segmenter for the letters a reader sees, and a stemmer for search.",
   "license": "MIT",
   "packageManager": "bun@1.3.14",
@@ -25,6 +25,10 @@
     "./stem": {
       "types": "./dist/types/js/stem.d.ts",
       "default": "./dist/stem.js"
+    },
+    "./romanize": {
+      "types": "./dist/types/js/romanize.d.ts",
+      "default": "./dist/romanize.js"
     }
   },
   "files": [
@@ -32,7 +36,7 @@
   ],
   "sideEffects": false,
   "scripts": {
-    "build": "bun build js/phonetic.ts --outfile dist/phonetic.js --target browser --format esm && bun build js/normalize.ts --outfile dist/normalize.js --target browser --format esm && bun build js/segment.ts --outfile dist/segment.js --target browser --format esm && bun build js/stem.ts --outfile dist/stem.js --target browser --format esm && tsc -p tsconfig.json",
+    "build": "bun build js/phonetic.ts --outfile dist/phonetic.js --target browser --format esm && bun build js/normalize.ts --outfile dist/normalize.js --target browser --format esm && bun build js/segment.ts --outfile dist/segment.js --target browser --format esm && bun build js/stem.ts --outfile dist/stem.js --target browser --format esm && bun build js/romanize.ts --outfile dist/romanize.js --target browser --format esm && tsc -p tsconfig.json",
     "test": "bun test",
     "check-dist": "node js/check-dist.mjs"
   },