indickit 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +458 -0
- package/README.md +25 -21
- package/dist/deromanize.js +1 -1
- package/dist/normalize.js +1 -1
- package/dist/phonetic-search.js +1 -1
- package/dist/phonetic.js +1 -1
- package/dist/romanize.js +2 -1
- package/dist/search-key.js +1 -1
- package/dist/stem.js +1 -1
- package/dist/types/js/phonetic-engine.d.ts +3 -1
- package/dist/types/js/phonetic-search.d.ts +11 -1
- package/dist/types/js/romanize.d.ts +26 -7
- package/package.json +3 -2
package/dist/stem.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
var p={version:"2026-10-
|
|
1
|
+
var p={version:"2026-10-12",min_stem:2,max_end:6,vowel_signs:[2362,2363,2366,2367,2368,2369,2370,2371,2372,2373,2374,2375,2376,2377,2378,2379,2380,2382,2383,2389,2390,2391,2402,2403,2494,2495,2496,2497,2498,2499,2500,2503,2504,2507,2508,2530,2531,2622,2623,2624,2625,2626,2631,2632,2635,2636,2750,2751,2752,2753,2754,2755,2756,2757,2759,2760,2761,2763,2764,2786,2787,2878,2879,2880,2881,2882,2883,2884,2887,2888,2891,2892,2914,2915,3006,3007,3008,3009,3010,3014,3015,3016,3018,3019,3020,3134,3135,3136,3137,3138,3139,3140,3142,3143,3144,3146,3147,3148,3170,3171,3262,3263,3264,3265,3266,3267,3268,3270,3271,3272,3274,3275,3276,3298,3299,3390,3391,3392,3393,3394,3395,3396,3398,3399,3400,3402,3403,3404,3426,3427],languages:{as:{vowel_step:!0,passes:1,endings:["ও","ক","কক","কত","কে","কৰ","খন","খনৰ","গত","ছিল","ছে","জন","টো","টোক","টোত","টোৰ","টোৱে","ত","তত","তে","তো","তৰ","বলৈ","বোৰ","বোৰৰ","মূলক","য়ে","লৈ","লৈকে","সকলক","সকলে","সকলৰ","সমূহ","সমূহক","সমূহে","সমূহৰ","াই","াক","াব","ায়","ালে","াৰ","াৰত","াৰে","িব","িবলৈ","িয়ে","িলে","িৰ","ীয়","ীয়ে","ীৰ","েৰে","োৱা","োৱে","্ত","ৰ","ৰক","ৰত","ৰলৈ","ৰি","ৰে","ৰো","ৰৰ"]},bn:{vowel_step:!0,passes:1,endings:["ও","করণ","গত","গুলি","গুলিকে","গুলিতে","গুলির","ছে","টার","তে","দের","মূলক","রা","রার","াই","াকে","াটি","ানো","ায়","ার","ারকে","ারে","ারের","িকে","িয়ে","ির","ীর","ুর","েন","ের"]},brx:{vowel_step:!1,passes:1,endings:["आ","आव","खौ","नि","फोर","फोरखौ","फोरनि","फोरा","फोराव","मोन","याव","सिम","हा","ाव"]},doi:{vowel_step:!1,passes:1,endings:["ं","एं","ा","ां","ाएं","ियां","े","ें","ै"]},gom:{vowel_step:!1,passes:1,endings:["पणान","ल्लो","ा","ां","ांक","ांची","ांचो","ांत","ांनी","ाक","ाची","ाचे","ाचें","ाचेर","ाचो","ाच्या","ात","ी","ींत","े","ें","ेंत","ेक","ो","्या","्याक","्यो"]},gu:{vowel_step:!1,passes:2,endings:["ં","એ","ઓ","ણા","થી","ના","ની","નું","ને","નો","માં","વા","વાની","વાનું","વાનો","વામાં","વી","વું","શે","ા","ાં","ાએ","ાઓ","ાઓની","ાઓને","ાઓમાં","ાના","ાની","ાનું","ાને","ાનો","ામાં","ાય","િત","ી","ીએ","ીઓ","ીઓને","ીના","ીની","ીને","ીમાં","ીય","ુ","ું","ે","ો","ોએ","ોના","ોની","ોનું","ોને","ોનો","ોમાં","્યા","્યું","્સ"]},hi:{vowel_step:!0,passes:1,endings:["ं","एँ","एं","ओं","कर","ता","ताओं","ती","ते","ना","नी","ने","पूर्ण","यां","ाना","ाने","ाया","ियाँ","ियों","्"]},kn:{vowel_step:!0,passes:2,endings:["ಂತೆ","ಂದು","ಕ್ಕಾಗಿ","ಕ್ಕೆ","ಗಳ","ಗಳನ್ನು","ಗಳಲ್ಲಿ","ಗಳಿಂದ","ಗಳಿಗೆ","ಗಳು","ಗಾಗಿ","ಗೆ","ತ್ತ","ತ್ತವೆ","ತ್ತಾನೆ","ತ್ತಾರೆ","ದ","ದನು","ದನ್ನು","ದರು","ದಲ್ಲಿ","ದವು","ದಿಂದ","ದ್ದಾರೆ","ನನ್ನು","ನು","ನ್ನು","ಬೇಕು","ಯನ್ನು","ಯಲ್ಲಿ","ಯಾಗಿ","ಯಿಂದ","ಯಿತು","ಯು","ರೆ","ಲಿನ","ಲು","ಲ್ಲಿ","ವನ್ನು","ವರು","ವಾಗಿ","ವಾಗಿದೆ","ವಾದ","ವು","ಾಗ","ಾಗಿ","ಾಗಿದೆ","ಿಂದ","ಿಗೂ","ಿತ","ಿತು","ಿದೆ","ಿನ","ಿನಿಂದ","ಿಯ","ಿವೆ","ುತ್ತದೆ","ುವ","ುವುದು","ೆಯ","ೊಂದಿಗೆ","್"]},ks:{vowel_step:!1,passes:1,endings:["س","ن","َس","َن","ُک","ِ","ٕ","ٕچ","ک","ہٕ","ی","ین"]},mai:{vowel_step:!1,passes:2,endings:["क","कक","केँ","गत","तः","पर","मे","सँ","सब","सभ","सभक","सभकेँ","सभमे","हु","ित","ीय","ो"]},ml:{vowel_step:!0,passes:1,endings:["ം","കയും","കളിലും","കളിൽ","കളും","കളുടെ","കളെ","കൾ","കൾക്ക്","ങ്ങളിൽ","ങ്ങളും","ങ്ങളെ","ങ്ങൾ","ച്ച്","ത്തിന്","ത്തിലെ","ത്തിൽ","നും","നുള്ള","നെ","ന്","ന്ന","ന്നത്","ന്റെ","മാണ്","മായ","മായി","മാർ","മുള്ള","യാണ്","യായി","യിലും","യിലെ","യിൽ","യും","യുടെ","യെ","യ്ക്ക്","രുടെ","രെ","റെ","ല്ല","ള്ള","വും","വ്","ായി","ിക്ക്","ിലുള്ള","ിലെ","ിൽ","ുക","ുന്നു","്","്ടു","്സ്","ൺ","ൻ","ർക്ക്"]},mni:{vowel_step:!1,passes:1,endings:["কী","গী","দগী","দা","দি","না","বু","রিবা","শিং","শিংগী","শিংদা","শিংনা","্না","ꯀꯤ","ꯁꯤꯡ","ꯁꯤꯡꯅ","ꯁꯤꯡꯒꯤ","ꯁꯤꯡꯗ","ꯅ","ꯇ","ꯒꯤ","ꯗ","ꯗꯒꯤ","꯫"]},mr:{vowel_step:!1,passes:2,endings:["ं","ंचा","ंना","ंनी","ंमध्ये","कडे","चा","ची","चे","च्या","द्वारे","ने","पणे","बद्दल","मधील","मध्ये","मुळे","ही","ा","ांचा","ांची","ांचे","ांच्या","ांत","ांना","ांनी","ांवर","ांशी","ांसाठी","ाचा","ाची","ाचे","ाच्या","ात","ातील","ातून","ान","ानंतर","ाना","ाने","ाला","ावर","ाशी","ासाठी","िक","ित","ी","ीचा","ीची","ीचे","ीच्या","ीत","ीतील","ीने","ीमध्ये","ीय","ील","ीला","ीसाठी","े","ेचा","ेची","ेचे","ेच्या","ेत","ेने","ेला","ेवर","ो","्य","्या","्यांचा","्यांची","्यांना","्याचा","्याची","्याचे","्यात","्याने","्याला","्ये"]},ne:{vowel_step:!0,passes:1,endings:["ं","का","की","को","देखि","द्वारा","बाट","भन्दा","मा","माथि","सँग","सम्म","हरूका","हरूको","हरूमा","हरूलाई","हरूले","ामा","ालाई","ाले","ाहरू","िएका","ित","िन","िने","िन्","िहरू","ीमा","ीलाई","ीले","ीहरू","ेका","ेन","ेर","्छ","्छन्","्दा","्दै","्न","्नु","्ने","्यो"]},or:{vowel_step:!1,passes:1,endings:["କୁ","ଗୁଡ଼ିକ","ଙ୍କ","ଙ୍କୁ","ର","ରକୁ","ରର","ରରେ","ରୁ","ରେ","୍"]},pa:{vowel_step:!0,passes:1,endings:["ਆਂ","ਇਆ","ਈ","ਈਆਂ","ਉਂਦਾ","ਉਂਦੇ","ਉਣ","ਉਣਾ","ਏ","ਕ","ਕਤਾ","ਕਰਨ","ਕਾਂ","ਣ","ਣਾ","ਣਾਂ","ਤਾ","ਨ","ਨਾ","ਨਾਂ","਼","਼ਾਂ","਼ੀ","ਾਂ","ਾਏ","ਾਨ","ਾਵਾਂ","ਿਆ","ਿਤ","ੀਂ","ੀਆ","ੇਂ","ੋਂ"]},sa:{vowel_step:!1,passes:1,endings:["ं","ः","णि","तुं","त्","त्वा","द्","नि","न्","भिः","म्","य","यते","यन्ते","या","यां","याः","याम्","यै","योः","षु","सु","स्य","ा","ां","ाः","ाणां","ाणि","ात्","ानां","ानाम्","ानि","ान्","ाम्","ाय","ायां","ायाः","ायाम्","ासु","िनः","ी","ीं","ीनां","ुः","े","ेण","ेन","ेभ्यः","ेषु","ैः","ो","ोः","ौ"]},sat:{vowel_step:!1,passes:2,endings:["ᱚᱛᱚ","ᱚᱜ","ᱚᱜᱼᱟ","ᱛᱚ","ᱛᱮ","ᱟ","ᱟᱨᱮ","ᱟᱭ","ᱟᱱ","ᱠᱚ","ᱦᱚᱸ","ᱨᱮ","ᱭ","ᱭᱟ","ᱭᱟᱱ","ᱮ","ᱮᱭ","ᱼᱟ","ᱽ"]},sd:{vowel_step:!1,passes:1,endings:["ءَ","ءِ","ا","ائڻ","ج","جي","دا","دي","دين","دڙ","ن","ندا","ندو","ندي","نن","هن","هو","و","ون","يا","يل","يو","يون","کي","ڻ","ं","ंदो","अ","नि","ा","ां","ि","ियुनि","ियूं","ु","ुनि","ूं","े","ो"]},ta:{vowel_step:!0,passes:1,endings:["களால்","களின்","களில்","களுடன்","களை","களைப்","கள்","கின்றன","கிறார்","க்","க்கள்","க்காக","க்கான","ச்","த்","ன","னர்","னார்","னால்","னின்","ன்","ப்","ப்பட்ட","மான","ய","யாக","யான","யால்","யின்","யில்","யுடன்","யே","யை","ய்","ற்கு","லாம்","லின்","வது","வர்","வர்கள்","வின்","வில்","வில்லை","ாக","ாத","ார்","ார்கள்","ால்","ாவின்","ிக்கு","ின்","ிய","ியது","ியில்","ிறது","ில்","ுகிறது","ுக்கு","ுடன்","ுதல்","ும்","ுள்ள","ுள்ளது","ைக்கு","ையில்","ையும்","ோடு","்டது"]},te:{vowel_step:!0,passes:1,endings:["ం","ంగా","ంతో","ందని","ంది","ంలో","ంలోని","గా","టు","డు","తో","నాడు","ని","నికి","ను","న్న","న్నాడు","న్నాయి","న్నారు","పై","మని","ము","మైన","లకు","లతో","లను","లపై","లలో","లు","లో","లోకి","లోని","ల్లో","ాను","ాన్ని","ాయి","ారు","ించారు","ించిన","ికి","ుడు","ుతో","ుని","ుల","్","్న","్లో"]},ur:{vowel_step:!1,passes:1,endings:["ئی","ئیں","ئے","اء","ائی","ات","اں","تا","تہ","تی","تے","را","ری","ریوں","رے","ز","نا","نی","نیوں","نے","و","وں","ِ","ٰ","ٹنگ","ں","ہ","ہوں","یا","یت","ین","ینی","یں","ے"]}}};function v(e){let s=(e??"").toLowerCase().split(/[-_]/)[0];if(s==="kok")return"gom";let a=s.length===3?"asm as ben bn guj gu hin hi kan kn kas ks mal ml mar mr nep ne ori or pan pa san sa snd sd tam ta tel te urd ur ory or npi ne ".indexOf(s+" "):-1;return a>=0&&a%7===0?"asm as ben bn guj gu hin hi kan kn kas ks mal ml mar mr nep ne ori or pan pa san sa snd sd tam ta tel te urd ur ory or npi ne ".slice(a+4,a+6):s}var o=p.min_stem,h=p.max_end,f=new Set(p.vowel_signs.map((e)=>String.fromCodePoint(e))),_=new Map(Object.entries(p.languages).map(([e,s])=>[e,{endings:new Set(s.endings),vowel:s.vowel_step,passes:s.passes}])),b=p.version,x=[..._.keys()].sort();function O(e,s,a){let n=o;if(a?.minStem!==void 0){if(n=a.minStem,!Number.isInteger(n)||n<o||h+n>=16)throw RangeError(`stem: minStem must be ${o}..${15-h}`)}let t=_.get(v(s));if(!t)return e;for(let l=0;l<t.passes;l++)e=c(e,t,n);return e}function c(e,s,a){let n=[e.length],t=e.length;while(n.length<=h+a&&t>0){let i=e.charCodeAt(--t);if(i>=56320&&i<=57343&&t>0)t--;n.push(t)}let l=t>0?1/0:n.length-1,g=e.length;for(let i=Math.min(h,l-a);i>0;i--)if(s.endings.has(e.slice(n[i]))){g=n[i],l-=i;break}if(s.vowel&&l>a&&f.has(e[g-1]))g--;return g===e.length?e:e.slice(0,g)}export{O as stem,b as RULES_VERSION,x as LANGUAGES};
|
|
@@ -6,9 +6,11 @@
|
|
|
6
6
|
* @module
|
|
7
7
|
*/
|
|
8
8
|
import rulesJson from "../phonetic/rules.json";
|
|
9
|
-
type Rules = Omit<typeof rulesJson, "initials" | "titles" | "initial_branches" | "brahmic" | "extra"> & {
|
|
9
|
+
type Rules = Omit<typeof rulesJson, "initials" | "abbreviations" | "second_readings" | "titles" | "initial_branches" | "brahmic" | "extra"> & {
|
|
10
10
|
extra: Record<string, string>;
|
|
11
11
|
initials?: Record<string, string[]>;
|
|
12
|
+
abbreviations?: Record<string, string[]>;
|
|
13
|
+
second_readings?: string[][];
|
|
12
14
|
initial_branches?: {
|
|
13
15
|
from: number;
|
|
14
16
|
to: number;
|
|
@@ -31,6 +31,16 @@ export declare const SEARCH_VERSION: string;
|
|
|
31
31
|
/** The scores measured for each use: names in a list 76, villages 73, a name
|
|
32
32
|
* in text 81 (at most 10% of hits wrong) or 71 (20%). */
|
|
33
33
|
export declare const THRESHOLDS: Readonly<Record<string, number>>;
|
|
34
|
+
declare class Memo {
|
|
35
|
+
private readonly std;
|
|
36
|
+
private readonly fine;
|
|
37
|
+
private readonly fine2;
|
|
38
|
+
keys(w: string): string[];
|
|
39
|
+
fineKey(w: string): string[];
|
|
40
|
+
private fine2Keys;
|
|
41
|
+
vowelRatio(a: string, b: string): number;
|
|
42
|
+
joinedOf(ws: string[], min: number): string[];
|
|
43
|
+
}
|
|
34
44
|
type Sparse = [number, [number, number][]];
|
|
35
45
|
type RawTable = {
|
|
36
46
|
latin?: string[];
|
|
@@ -60,7 +70,7 @@ declare class Scorer {
|
|
|
60
70
|
constructor(lang: string, latin: CostTable | undefined, pairs: Map<string, CostTable>);
|
|
61
71
|
cost(a: string, b: string): number | undefined;
|
|
62
72
|
calibrations(a: string, cands: string[]): Map<string, number>;
|
|
63
|
-
wordSim(a: string, b: string, cal: Map<string, number> | undefined, p: ProfileRules): number;
|
|
73
|
+
wordSim(a: string, b: string, cal: Map<string, number> | undefined, p: ProfileRules, m: Memo): number;
|
|
64
74
|
}
|
|
65
75
|
/** A name found by a search: its position among the indexed names, and its score. */
|
|
66
76
|
export interface Hit {
|
|
@@ -7,6 +7,9 @@
|
|
|
7
7
|
* running text (the default); "names" for fields known to hold person
|
|
8
8
|
* names, where a lookup of known names comes first; and "places" for place
|
|
9
9
|
* names (its own tables in nine languages, the "words" tables in the others).
|
|
10
|
+
* In Hindi, `load("hi", "names", { route: true })` spells only the names the
|
|
11
|
+
* names tables were trained on with them, and every other word with the
|
|
12
|
+
* "words" tables.
|
|
10
13
|
*
|
|
11
14
|
* The tables are learned from data and loaded per language, on demand:
|
|
12
15
|
*
|
|
@@ -41,17 +44,33 @@ export interface Romanizer {
|
|
|
41
44
|
text(text: string): string;
|
|
42
45
|
}
|
|
43
46
|
/**
|
|
44
|
-
* Loads a language's tables for one mode: its own file
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
*
|
|
48
|
-
*
|
|
47
|
+
* Loads a language's tables for one mode: its own file, its script group's
|
|
48
|
+
* pooled file (shared by the group, fetched once) and, for the names mode,
|
|
49
|
+
* the letter n-gram every names page shares. With `route: true`, Hindi's
|
|
50
|
+
* names mode also loads its list of known names and its words tables
|
|
51
|
+
* (rules 2026-10-13): a name the names tables were trained on keeps its
|
|
52
|
+
* names list, and any other word gets the words list. On names in news held
|
|
53
|
+
* out (PIB, August 2023) Hindi's first spelling goes from 55.0% to 62.2%;
|
|
54
|
+
* the page grows from 0.56 MB to 2.5 MB (26 KB on a page that has Hindi's
|
|
55
|
+
* words tables). In a language or mode with no route, `route` changes nothing. It reads them beside this
|
|
56
|
+
* module, else from jsDelivr at this package's version (the npm package
|
|
57
|
+
* carries no tables); `base` or `fetch` replaces both. Rejects when the
|
|
58
|
+
* language has no tables for the mode.
|
|
49
59
|
*/
|
|
50
60
|
export declare function load(lang: string, mode?: Mode, opts?: {
|
|
51
61
|
fetch?: Fetcher;
|
|
52
62
|
base?: URL | string;
|
|
63
|
+
route?: boolean;
|
|
53
64
|
}): Promise<Romanizer>;
|
|
54
|
-
/**
|
|
55
|
-
|
|
65
|
+
/**
|
|
66
|
+
* Builds a Romanizer from a language's files (for tests and offline use): its own, its
|
|
67
|
+
* group's pooled one and, for a mode with a letter n-gram (names), that file too. `route`
|
|
68
|
+
* turns the route on where the mode has one (Hindi names): the known list's file and the
|
|
69
|
+
* Romanizer of the other mode (Hindi words), as `load(..., { route: true })` passes them.
|
|
70
|
+
*/
|
|
71
|
+
export declare function fromBytes(lang: string, mode: Mode, own: Uint8Array, pool: Uint8Array, lm?: Uint8Array, route?: {
|
|
72
|
+
known: Uint8Array;
|
|
73
|
+
other: Romanizer;
|
|
74
|
+
}): Romanizer;
|
|
56
75
|
/** @internal The model's own list for a unified word (no lookup), for the conformance test. */
|
|
57
76
|
export declare function _decode(r: Romanizer, word: string, n: number): string[];
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "indickit",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.13.0",
|
|
4
4
|
"description": "Small, exact utilities for text in Indian languages: a phonetic key that matches a name across 21 Indian scripts and English, a normalizer that gives text that looks the same one encoding, a segmenter for the letters a reader sees, and a stemmer for search.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"packageManager": "bun@1.3.14",
|
|
@@ -52,7 +52,8 @@
|
|
|
52
52
|
}
|
|
53
53
|
},
|
|
54
54
|
"files": [
|
|
55
|
-
"dist"
|
|
55
|
+
"dist",
|
|
56
|
+
"CHANGELOG.md"
|
|
56
57
|
],
|
|
57
58
|
"sideEffects": false,
|
|
58
59
|
"scripts": {
|