indickit 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +389 -0
- package/README.md +128 -66
- package/dist/deromanize.js +1 -1
- package/dist/normalize.js +1 -1
- package/dist/phonetic-search.js +1 -1
- package/dist/phonetic.js +1 -1
- package/dist/romanize.js +1 -1
- package/dist/search-key.js +1 -0
- package/dist/segment.js +1 -1
- package/dist/stem-compounds.js +1 -0
- package/dist/stem.js +1 -1
- package/dist/types/js/deromanize.d.ts +12 -0
- package/dist/types/js/phonetic-engine.d.ts +17 -1
- package/dist/types/js/phonetic-search.d.ts +13 -3
- package/dist/types/js/phonetic.d.ts +6 -4
- package/dist/types/js/romanize.d.ts +20 -12
- package/dist/types/js/search-key.d.ts +14 -0
- package/dist/types/js/stem-compounds.d.ts +16 -0
- package/dist/types/js/stem.d.ts +12 -3
- package/package.json +12 -3
package/dist/stem.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
var
|
|
1
|
+
var p={version:"2026-10-12",min_stem:2,max_end:6,vowel_signs:[2362,2363,2366,2367,2368,2369,2370,2371,2372,2373,2374,2375,2376,2377,2378,2379,2380,2382,2383,2389,2390,2391,2402,2403,2494,2495,2496,2497,2498,2499,2500,2503,2504,2507,2508,2530,2531,2622,2623,2624,2625,2626,2631,2632,2635,2636,2750,2751,2752,2753,2754,2755,2756,2757,2759,2760,2761,2763,2764,2786,2787,2878,2879,2880,2881,2882,2883,2884,2887,2888,2891,2892,2914,2915,3006,3007,3008,3009,3010,3014,3015,3016,3018,3019,3020,3134,3135,3136,3137,3138,3139,3140,3142,3143,3144,3146,3147,3148,3170,3171,3262,3263,3264,3265,3266,3267,3268,3270,3271,3272,3274,3275,3276,3298,3299,3390,3391,3392,3393,3394,3395,3396,3398,3399,3400,3402,3403,3404,3426,3427],languages:{as:{vowel_step:!0,passes:1,endings:["ও","ক","কক","কত","কে","কৰ","খন","খনৰ","গত","ছিল","ছে","জন","টো","টোক","টোত","টোৰ","টোৱে","ত","তত","তে","তো","তৰ","বলৈ","বোৰ","বোৰৰ","মূলক","য়ে","লৈ","লৈকে","সকলক","সকলে","সকলৰ","সমূহ","সমূহক","সমূহে","সমূহৰ","াই","াক","াব","ায়","ালে","াৰ","াৰত","াৰে","িব","িবলৈ","িয়ে","িলে","িৰ","ীয়","ীয়ে","ীৰ","েৰে","োৱা","োৱে","্ত","ৰ","ৰক","ৰত","ৰলৈ","ৰি","ৰে","ৰো","ৰৰ"]},bn:{vowel_step:!0,passes:1,endings:["ও","করণ","গত","গুলি","গুলিকে","গুলিতে","গুলির","ছে","টার","তে","দের","মূলক","রা","রার","াই","াকে","াটি","ানো","ায়","ার","ারকে","ারে","ারের","িকে","িয়ে","ির","ীর","ুর","েন","ের"]},brx:{vowel_step:!1,passes:1,endings:["आ","आव","खौ","नि","फोर","फोरखौ","फोरनि","फोरा","फोराव","मोन","याव","सिम","हा","ाव"]},doi:{vowel_step:!1,passes:1,endings:["ं","एं","ा","ां","ाएं","ियां","े","ें","ै"]},gom:{vowel_step:!1,passes:1,endings:["पणान","ल्लो","ा","ां","ांक","ांची","ांचो","ांत","ांनी","ाक","ाची","ाचे","ाचें","ाचेर","ाचो","ाच्या","ात","ी","ींत","े","ें","ेंत","ेक","ो","्या","्याक","्यो"]},gu:{vowel_step:!1,passes:2,endings:["ં","એ","ઓ","ણા","થી","ના","ની","નું","ને","નો","માં","વા","વાની","વાનું","વાનો","વામાં","વી","વું","શે","ા","ાં","ાએ","ાઓ","ાઓની","ાઓને","ાઓમાં","ાના","ાની","ાનું","ાને","ાનો","ામાં","ાય","િત","ી","ીએ","ીઓ","ીઓને","ીના","ીની","ીને","ીમાં","ીય","ુ","ું","ે","ો","ોએ","ોના","ોની","ોનું","ોને","ોનો","ોમાં","્યા","્યું","્સ"]},hi:{vowel_step:!0,passes:1,endings:["ं","एँ","एं","ओं","कर","ता","ताओं","ती","ते","ना","नी","ने","पूर्ण","यां","ाना","ाने","ाया","ियाँ","ियों","्"]},kn:{vowel_step:!0,passes:2,endings:["ಂತೆ","ಂದು","ಕ್ಕಾಗಿ","ಕ್ಕೆ","ಗಳ","ಗಳನ್ನು","ಗಳಲ್ಲಿ","ಗಳಿಂದ","ಗಳಿಗೆ","ಗಳು","ಗಾಗಿ","ಗೆ","ತ್ತ","ತ್ತವೆ","ತ್ತಾನೆ","ತ್ತಾರೆ","ದ","ದನು","ದನ್ನು","ದರು","ದಲ್ಲಿ","ದವು","ದಿಂದ","ದ್ದಾರೆ","ನನ್ನು","ನು","ನ್ನು","ಬೇಕು","ಯನ್ನು","ಯಲ್ಲಿ","ಯಾಗಿ","ಯಿಂದ","ಯಿತು","ಯು","ರೆ","ಲಿನ","ಲು","ಲ್ಲಿ","ವನ್ನು","ವರು","ವಾಗಿ","ವಾಗಿದೆ","ವಾದ","ವು","ಾಗ","ಾಗಿ","ಾಗಿದೆ","ಿಂದ","ಿಗೂ","ಿತ","ಿತು","ಿದೆ","ಿನ","ಿನಿಂದ","ಿಯ","ಿವೆ","ುತ್ತದೆ","ುವ","ುವುದು","ೆಯ","ೊಂದಿಗೆ","್"]},ks:{vowel_step:!1,passes:1,endings:["س","ن","َس","َن","ُک","ِ","ٕ","ٕچ","ک","ہٕ","ی","ین"]},mai:{vowel_step:!1,passes:2,endings:["क","कक","केँ","गत","तः","पर","मे","सँ","सब","सभ","सभक","सभकेँ","सभमे","हु","ित","ीय","ो"]},ml:{vowel_step:!0,passes:1,endings:["ം","കയും","കളിലും","കളിൽ","കളും","കളുടെ","കളെ","കൾ","കൾക്ക്","ങ്ങളിൽ","ങ്ങളും","ങ്ങളെ","ങ്ങൾ","ച്ച്","ത്തിന്","ത്തിലെ","ത്തിൽ","നും","നുള്ള","നെ","ന്","ന്ന","ന്നത്","ന്റെ","മാണ്","മായ","മായി","മാർ","മുള്ള","യാണ്","യായി","യിലും","യിലെ","യിൽ","യും","യുടെ","യെ","യ്ക്ക്","രുടെ","രെ","റെ","ല്ല","ള്ള","വും","വ്","ായി","ിക്ക്","ിലുള്ള","ിലെ","ിൽ","ുക","ുന്നു","്","്ടു","്സ്","ൺ","ൻ","ർക്ക്"]},mni:{vowel_step:!1,passes:1,endings:["কী","গী","দগী","দা","দি","না","বু","রিবা","শিং","শিংগী","শিংদা","শিংনা","্না","ꯀꯤ","ꯁꯤꯡ","ꯁꯤꯡꯅ","ꯁꯤꯡꯒꯤ","ꯁꯤꯡꯗ","ꯅ","ꯇ","ꯒꯤ","ꯗ","ꯗꯒꯤ","꯫"]},mr:{vowel_step:!1,passes:2,endings:["ं","ंचा","ंना","ंनी","ंमध्ये","कडे","चा","ची","चे","च्या","द्वारे","ने","पणे","बद्दल","मधील","मध्ये","मुळे","ही","ा","ांचा","ांची","ांचे","ांच्या","ांत","ांना","ांनी","ांवर","ांशी","ांसाठी","ाचा","ाची","ाचे","ाच्या","ात","ातील","ातून","ान","ानंतर","ाना","ाने","ाला","ावर","ाशी","ासाठी","िक","ित","ी","ीचा","ीची","ीचे","ीच्या","ीत","ीतील","ीने","ीमध्ये","ीय","ील","ीला","ीसाठी","े","ेचा","ेची","ेचे","ेच्या","ेत","ेने","ेला","ेवर","ो","्य","्या","्यांचा","्यांची","्यांना","्याचा","्याची","्याचे","्यात","्याने","्याला","्ये"]},ne:{vowel_step:!0,passes:1,endings:["ं","का","की","को","देखि","द्वारा","बाट","भन्दा","मा","माथि","सँग","सम्म","हरूका","हरूको","हरूमा","हरूलाई","हरूले","ामा","ालाई","ाले","ाहरू","िएका","ित","िन","िने","िन्","िहरू","ीमा","ीलाई","ीले","ीहरू","ेका","ेन","ेर","्छ","्छन्","्दा","्दै","्न","्नु","्ने","्यो"]},or:{vowel_step:!1,passes:1,endings:["କୁ","ଗୁଡ଼ିକ","ଙ୍କ","ଙ୍କୁ","ର","ରକୁ","ରର","ରରେ","ରୁ","ରେ","୍"]},pa:{vowel_step:!0,passes:1,endings:["ਆਂ","ਇਆ","ਈ","ਈਆਂ","ਉਂਦਾ","ਉਂਦੇ","ਉਣ","ਉਣਾ","ਏ","ਕ","ਕਤਾ","ਕਰਨ","ਕਾਂ","ਣ","ਣਾ","ਣਾਂ","ਤਾ","ਨ","ਨਾ","ਨਾਂ","਼","਼ਾਂ","਼ੀ","ਾਂ","ਾਏ","ਾਨ","ਾਵਾਂ","ਿਆ","ਿਤ","ੀਂ","ੀਆ","ੇਂ","ੋਂ"]},sa:{vowel_step:!1,passes:1,endings:["ं","ः","णि","तुं","त्","त्वा","द्","नि","न्","भिः","म्","य","यते","यन्ते","या","यां","याः","याम्","यै","योः","षु","सु","स्य","ा","ां","ाः","ाणां","ाणि","ात्","ानां","ानाम्","ानि","ान्","ाम्","ाय","ायां","ायाः","ायाम्","ासु","िनः","ी","ीं","ीनां","ुः","े","ेण","ेन","ेभ्यः","ेषु","ैः","ो","ोः","ौ"]},sat:{vowel_step:!1,passes:2,endings:["ᱚᱛᱚ","ᱚᱜ","ᱚᱜᱼᱟ","ᱛᱚ","ᱛᱮ","ᱟ","ᱟᱨᱮ","ᱟᱭ","ᱟᱱ","ᱠᱚ","ᱦᱚᱸ","ᱨᱮ","ᱭ","ᱭᱟ","ᱭᱟᱱ","ᱮ","ᱮᱭ","ᱼᱟ","ᱽ"]},sd:{vowel_step:!1,passes:1,endings:["ءَ","ءِ","ا","ائڻ","ج","جي","دا","دي","دين","دڙ","ن","ندا","ندو","ندي","نن","هن","هو","و","ون","يا","يل","يو","يون","کي","ڻ","ं","ंदो","अ","नि","ा","ां","ि","ियुनि","ियूं","ु","ुनि","ूं","े","ो"]},ta:{vowel_step:!0,passes:1,endings:["களால்","களின்","களில்","களுடன்","களை","களைப்","கள்","கின்றன","கிறார்","க்","க்கள்","க்காக","க்கான","ச்","த்","ன","னர்","னார்","னால்","னின்","ன்","ப்","ப்பட்ட","மான","ய","யாக","யான","யால்","யின்","யில்","யுடன்","யே","யை","ய்","ற்கு","லாம்","லின்","வது","வர்","வர்கள்","வின்","வில்","வில்லை","ாக","ாத","ார்","ார்கள்","ால்","ாவின்","ிக்கு","ின்","ிய","ியது","ியில்","ிறது","ில்","ுகிறது","ுக்கு","ுடன்","ுதல்","ும்","ுள்ள","ுள்ளது","ைக்கு","ையில்","ையும்","ோடு","்டது"]},te:{vowel_step:!0,passes:1,endings:["ం","ంగా","ంతో","ందని","ంది","ంలో","ంలోని","గా","టు","డు","తో","నాడు","ని","నికి","ను","న్న","న్నాడు","న్నాయి","న్నారు","పై","మని","ము","మైన","లకు","లతో","లను","లపై","లలో","లు","లో","లోకి","లోని","ల్లో","ాను","ాన్ని","ాయి","ారు","ించారు","ించిన","ికి","ుడు","ుతో","ుని","ుల","్","్న","్లో"]},ur:{vowel_step:!1,passes:1,endings:["ئی","ئیں","ئے","اء","ائی","ات","اں","تا","تہ","تی","تے","را","ری","ریوں","رے","ز","نا","نی","نیوں","نے","و","وں","ِ","ٰ","ٹنگ","ں","ہ","ہوں","یا","یت","ین","ینی","یں","ے"]}}};function v(e){let s=(e??"").toLowerCase().split(/[-_]/)[0];if(s==="kok")return"gom";let a=s.length===3?"asm as ben bn guj gu hin hi kan kn kas ks mal ml mar mr nep ne ori or pan pa san sa snd sd tam ta tel te urd ur ory or npi ne ".indexOf(s+" "):-1;return a>=0&&a%7===0?"asm as ben bn guj gu hin hi kan kn kas ks mal ml mar mr nep ne ori or pan pa san sa snd sd tam ta tel te urd ur ory or npi ne ".slice(a+4,a+6):s}var o=p.min_stem,h=p.max_end,f=new Set(p.vowel_signs.map((e)=>String.fromCodePoint(e))),_=new Map(Object.entries(p.languages).map(([e,s])=>[e,{endings:new Set(s.endings),vowel:s.vowel_step,passes:s.passes}])),b=p.version,x=[..._.keys()].sort();function O(e,s,a){let n=o;if(a?.minStem!==void 0){if(n=a.minStem,!Number.isInteger(n)||n<o||h+n>=16)throw RangeError(`stem: minStem must be ${o}..${15-h}`)}let t=_.get(v(s));if(!t)return e;for(let l=0;l<t.passes;l++)e=c(e,t,n);return e}function c(e,s,a){let n=[e.length],t=e.length;while(n.length<=h+a&&t>0){let i=e.charCodeAt(--t);if(i>=56320&&i<=57343&&t>0)t--;n.push(t)}let l=t>0?1/0:n.length-1,g=e.length;for(let i=Math.min(h,l-a);i>0;i--)if(s.endings.has(e.slice(n[i]))){g=n[i],l-=i;break}if(s.vowel&&l>a&&f.has(e[g-1]))g--;return g===e.length?e:e.slice(0,g)}export{O as stem,b as RULES_VERSION,x as LANGUAGES};
|
|
@@ -39,6 +39,17 @@ export interface Deromanizer {
|
|
|
39
39
|
word(latin: string, n?: number): string[];
|
|
40
40
|
/** Every run of Latin letters written as its first native spelling; everything else kept. */
|
|
41
41
|
text(text: string): string;
|
|
42
|
+
/**
|
|
43
|
+
* Is the Latin-typed word, read as `word` reads it, in the language's English list
|
|
44
|
+
* (an English word a typist of the language writes in English)? Only Hindi has a list:
|
|
45
|
+
* load it with `{ english: true }`. False for a language with no list.
|
|
46
|
+
*/
|
|
47
|
+
isEnglish(word: string): boolean;
|
|
48
|
+
/**
|
|
49
|
+
* `text` for mixed text such as Hinglish: a word in the English list stays as typed
|
|
50
|
+
* ("kal meeting hai" → "कल meeting है"). Equals `text` for a language with no list.
|
|
51
|
+
*/
|
|
52
|
+
textMixed(text: string): string;
|
|
42
53
|
}
|
|
43
54
|
/** The input as the training pairs were read: accents removed, lower case, a-z only. */
|
|
44
55
|
export declare function clean(s: string): string;
|
|
@@ -52,6 +63,7 @@ export declare function clean(s: string): string;
|
|
|
52
63
|
export declare function load(lang: string, mode?: Mode, opts?: {
|
|
53
64
|
fetch?: Fetcher;
|
|
54
65
|
base?: URL | string;
|
|
66
|
+
english?: boolean;
|
|
55
67
|
}): Promise<Deromanizer>;
|
|
56
68
|
/** Builds a Deromanizer from its files, by file name (for tests and offline use). */
|
|
57
69
|
export declare function fromBytes(lang: string, mode: Mode, files: Record<string, Uint8Array>): Deromanizer;
|
|
@@ -6,9 +6,25 @@
|
|
|
6
6
|
* @module
|
|
7
7
|
*/
|
|
8
8
|
import rulesJson from "../phonetic/rules.json";
|
|
9
|
-
type Rules = typeof rulesJson
|
|
9
|
+
type Rules = Omit<typeof rulesJson, "initials" | "titles" | "initial_branches" | "brahmic" | "extra"> & {
|
|
10
|
+
extra: Record<string, string>;
|
|
11
|
+
initials?: Record<string, string[]>;
|
|
12
|
+
initial_branches?: {
|
|
13
|
+
from: number;
|
|
14
|
+
to: number;
|
|
15
|
+
also: string;
|
|
16
|
+
}[];
|
|
17
|
+
brahmic: Omit<typeof rulesJson.brahmic, "rewrite" | "rewrite_bases"> & {
|
|
18
|
+
rewrite?: string[][];
|
|
19
|
+
rewrite_bases?: string[][];
|
|
20
|
+
};
|
|
21
|
+
};
|
|
10
22
|
export declare function compile(rules: Rules): (word: string) => string[];
|
|
11
23
|
export declare const engine: (word: string) => string[];
|
|
24
|
+
/** The words of a name written in comma-separated parts (normalized words),
|
|
25
|
+
* each part without its leading titles (Shri, Smt., श्रीमती, திரு); if every
|
|
26
|
+
* word is a title, the last word stays (reference.py#name_parts). */
|
|
27
|
+
export declare function dropTitles(parts: string[][]): string[];
|
|
12
28
|
/** nameKeys and the search index stop here: a long Tamil name can have many key combinations. */
|
|
13
29
|
export declare const MAX_NAME_KEYS = 256;
|
|
14
30
|
export declare function normalize(word: string): string;
|
|
@@ -28,9 +28,19 @@ type ProfileRules = {
|
|
|
28
28
|
};
|
|
29
29
|
/** Changes whenever any score changes. */
|
|
30
30
|
export declare const SEARCH_VERSION: string;
|
|
31
|
-
/** The scores measured for each use: names in a list
|
|
32
|
-
* in text
|
|
31
|
+
/** The scores measured for each use: names in a list 76, villages 73, a name
|
|
32
|
+
* in text 81 (at most 10% of hits wrong) or 71 (20%). */
|
|
33
33
|
export declare const THRESHOLDS: Readonly<Record<string, number>>;
|
|
34
|
+
declare class Memo {
|
|
35
|
+
private readonly std;
|
|
36
|
+
private readonly fine;
|
|
37
|
+
private readonly fine2;
|
|
38
|
+
keys(w: string): string[];
|
|
39
|
+
fineKey(w: string): string[];
|
|
40
|
+
private fine2Keys;
|
|
41
|
+
vowelRatio(a: string, b: string): number;
|
|
42
|
+
joinedOf(ws: string[], min: number): string[];
|
|
43
|
+
}
|
|
34
44
|
type Sparse = [number, [number, number][]];
|
|
35
45
|
type RawTable = {
|
|
36
46
|
latin?: string[];
|
|
@@ -60,7 +70,7 @@ declare class Scorer {
|
|
|
60
70
|
constructor(lang: string, latin: CostTable | undefined, pairs: Map<string, CostTable>);
|
|
61
71
|
cost(a: string, b: string): number | undefined;
|
|
62
72
|
calibrations(a: string, cands: string[]): Map<string, number>;
|
|
63
|
-
wordSim(a: string, b: string, cal: Map<string, number> | undefined, p: ProfileRules): number;
|
|
73
|
+
wordSim(a: string, b: string, cal: Map<string, number> | undefined, p: ProfileRules, m: Memo): number;
|
|
64
74
|
}
|
|
65
75
|
/** A name found by a search: its position among the indexed names, and its score. */
|
|
66
76
|
export interface Hit {
|
|
@@ -16,8 +16,10 @@ export { MAX_NAME_KEYS };
|
|
|
16
16
|
export declare const RULES_VERSION: string;
|
|
17
17
|
/** The sorted keys of one word; empty when it holds no letter the rules read. */
|
|
18
18
|
export declare function keys(word: string): string[];
|
|
19
|
-
/** A name split
|
|
20
|
-
*
|
|
19
|
+
/** A name split on spaces and punctuation, with anything in brackets removed.
|
|
20
|
+
* nameKeys, joinedKeys and match split this way, then drop the leading
|
|
21
|
+
* titles (Shri, Smt., श्रीमती, திரு) of each comma-separated part ("Bajwa,
|
|
22
|
+
* Sardar Partap Singh"); a name of titles only keeps its last word. */
|
|
21
23
|
export declare function words(name: string): string[];
|
|
22
24
|
/** The keys of a whole name, one per combination of its words' keys, words
|
|
23
25
|
* separated by a space. Two names match when these share an element, so
|
|
@@ -30,6 +32,6 @@ export declare const JOINED_MIN_CLASSES: number;
|
|
|
30
32
|
* least JOINED_MIN_CLASSES classes. A search looks them up only when
|
|
31
33
|
* nameKeys find nobody; match does not use them. */
|
|
32
34
|
export declare function joinedKeys(name: string): string[];
|
|
33
|
-
/** Whether two names have the same number of
|
|
34
|
-
* in order, shares a key. */
|
|
35
|
+
/** Whether two names, their leading titles dropped, have the same number of
|
|
36
|
+
* words and every pair of words, in order, shares a key. */
|
|
35
37
|
export declare function match(a: string, b: string): boolean;
|
|
@@ -3,9 +3,10 @@
|
|
|
3
3
|
* लक्ष्मी → lakshmi, മലൈക → malaika, پرویز → parvez.
|
|
4
4
|
*
|
|
5
5
|
* A ranked list: one word has several accepted spellings (Choudhury,
|
|
6
|
-
* Chowdhury), and the first is the most likely.
|
|
7
|
-
* running text (the default)
|
|
8
|
-
* names, where a lookup of known names comes first
|
|
6
|
+
* Chowdhury), and the first is the most likely. Three modes: "words" for
|
|
7
|
+
* running text (the default); "names" for fields known to hold person
|
|
8
|
+
* names, where a lookup of known names comes first; and "places" for place
|
|
9
|
+
* names (its own tables in nine languages, the "words" tables in the others).
|
|
9
10
|
*
|
|
10
11
|
* The tables are learned from data and loaded per language, on demand:
|
|
11
12
|
*
|
|
@@ -17,8 +18,11 @@
|
|
|
17
18
|
*
|
|
18
19
|
* @module
|
|
19
20
|
*/
|
|
20
|
-
/**
|
|
21
|
-
|
|
21
|
+
/**
|
|
22
|
+
* The tables: "words" for running text, "names" for person names, "places" for
|
|
23
|
+
* place names (own tables for bn hi ml mr or sat ta te ur; the "words" tables elsewhere).
|
|
24
|
+
*/
|
|
25
|
+
export type Mode = "words" | "names" | "places";
|
|
22
26
|
/** Changes whenever any output changes: store it beside romanized text. */
|
|
23
27
|
export declare const RULES_VERSION: string;
|
|
24
28
|
/** The language codes that have tables, for one mode. */
|
|
@@ -37,17 +41,21 @@ export interface Romanizer {
|
|
|
37
41
|
text(text: string): string;
|
|
38
42
|
}
|
|
39
43
|
/**
|
|
40
|
-
* Loads a language's tables for one mode: its own file
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
44
|
+
* Loads a language's tables for one mode: its own file, its script group's
|
|
45
|
+
* pooled file (shared by the group, fetched once) and, for the names mode,
|
|
46
|
+
* the letter n-gram every names page shares. It reads them beside this
|
|
47
|
+
* module, else from jsDelivr at this package's version (the npm package
|
|
48
|
+
* carries no tables); `base` or `fetch` replaces both. Rejects when the
|
|
49
|
+
* language has no tables for the mode.
|
|
45
50
|
*/
|
|
46
51
|
export declare function load(lang: string, mode?: Mode, opts?: {
|
|
47
52
|
fetch?: Fetcher;
|
|
48
53
|
base?: URL | string;
|
|
49
54
|
}): Promise<Romanizer>;
|
|
50
|
-
/**
|
|
51
|
-
|
|
55
|
+
/**
|
|
56
|
+
* Builds a Romanizer from a language's files (for tests and offline use): its own, its
|
|
57
|
+
* group's pooled one and, for a mode with a letter n-gram (names), that file too.
|
|
58
|
+
*/
|
|
59
|
+
export declare function fromBytes(lang: string, mode: Mode, own: Uint8Array, pool: Uint8Array, lm?: Uint8Array): Romanizer;
|
|
52
60
|
/** @internal The model's own list for a unified word (no lookup), for the conformance test. */
|
|
53
61
|
export declare function _decode(r: Romanizer, word: string, n: number): string[];
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The one key to index and to query by:
|
|
3
|
+
* searchKey("किताबों", "hi") === searchKey("किताब", "hi"), and
|
|
4
|
+
* searchKey("हिन्दी", "hi") === searchKey("हिंदी", "hi").
|
|
5
|
+
*
|
|
6
|
+
* It normalizes, stems, then folds. The order was measured on search: fold
|
|
7
|
+
* after stem finds 2.4 points more of the right passages in Hindi and
|
|
8
|
+
* changes nothing in Bengali, Telugu or Assamese; fold before stem costs
|
|
9
|
+
* Assamese 3.7 points. A key is for matching only: never store it as text.
|
|
10
|
+
*
|
|
11
|
+
* @module
|
|
12
|
+
*/
|
|
13
|
+
/** The search key of `word` in language `lang` (a tag such as "hi", "hi-IN" or "hin"). */
|
|
14
|
+
export declare function searchKey(word: string, lang: string): string;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Malayalam compounds for search: the first member that starts a word.
|
|
3
|
+
* Use it in place of the stem for Malayalam:
|
|
4
|
+
*
|
|
5
|
+
* const key = compoundKey(w) ?? stem(w, "ml");
|
|
6
|
+
*
|
|
7
|
+
* It finds compounds and some oblique forms (കേരളസർക്കാർ and കേരളത്തിലെ
|
|
8
|
+
* share കേരള). On held-out news it found 5.5 points more of the right
|
|
9
|
+
* sentences than `stem` alone, and was 2.9 points less precise. A separate
|
|
10
|
+
* file, so that `stem` stays small for every other language. Normalize text
|
|
11
|
+
* first.
|
|
12
|
+
*
|
|
13
|
+
* @module
|
|
14
|
+
*/
|
|
15
|
+
/** The longest first member that starts `word` and leaves at least two code points, or null. */
|
|
16
|
+
export declare function compoundKey(word: string): string | null;
|
package/dist/types/js/stem.d.ts
CHANGED
|
@@ -19,8 +19,17 @@ export declare const RULES_VERSION: string;
|
|
|
19
19
|
/** The language codes that have a table. */
|
|
20
20
|
export declare const LANGUAGES: readonly string[];
|
|
21
21
|
/**
|
|
22
|
-
* The search key of `word` in language `lang` (
|
|
23
|
-
*
|
|
22
|
+
* The search key of `word` in language `lang` (any of the 22
|
|
23
|
+
* languages of the Eighth Schedule). Only the tag's language counts: "ta-IN", "TA" and "tam" are
|
|
24
24
|
* "ta". For any other `lang`, `word` unchanged.
|
|
25
25
|
*/
|
|
26
|
-
export declare function stem(word: string, lang: string): string;
|
|
26
|
+
export declare function stem(word: string, lang: string, options?: StemOptions): string;
|
|
27
|
+
/** Options for {@link stem}. */
|
|
28
|
+
export type StemOptions = {
|
|
29
|
+
/**
|
|
30
|
+
* The fewest code points a stem keeps (default 2, at most 9). A field of
|
|
31
|
+
* names wants 4: it halves the names that lose their end (करीना → कर) and
|
|
32
|
+
* costs 0.5–2 points of search recall.
|
|
33
|
+
*/
|
|
34
|
+
minStem?: number;
|
|
35
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "indickit",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.12.0",
|
|
4
4
|
"description": "Small, exact utilities for text in Indian languages: a phonetic key that matches a name across 21 Indian scripts and English, a normalizer that gives text that looks the same one encoding, a segmenter for the letters a reader sees, and a stemmer for search.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"packageManager": "bun@1.3.14",
|
|
@@ -26,6 +26,14 @@
|
|
|
26
26
|
"types": "./dist/types/js/stem.d.ts",
|
|
27
27
|
"default": "./dist/stem.js"
|
|
28
28
|
},
|
|
29
|
+
"./search-key": {
|
|
30
|
+
"types": "./dist/types/js/search-key.d.ts",
|
|
31
|
+
"default": "./dist/search-key.js"
|
|
32
|
+
},
|
|
33
|
+
"./stem-compounds": {
|
|
34
|
+
"types": "./dist/types/js/stem-compounds.d.ts",
|
|
35
|
+
"default": "./dist/stem-compounds.js"
|
|
36
|
+
},
|
|
29
37
|
"./romanize": {
|
|
30
38
|
"types": "./dist/types/js/romanize.d.ts",
|
|
31
39
|
"default": "./dist/romanize.js"
|
|
@@ -44,11 +52,12 @@
|
|
|
44
52
|
}
|
|
45
53
|
},
|
|
46
54
|
"files": [
|
|
47
|
-
"dist"
|
|
55
|
+
"dist",
|
|
56
|
+
"CHANGELOG.md"
|
|
48
57
|
],
|
|
49
58
|
"sideEffects": false,
|
|
50
59
|
"scripts": {
|
|
51
|
-
"build": "bun build js/phonetic.ts --outfile dist/phonetic.js --target browser --format esm --minify && bun build js/normalize.ts --outfile dist/normalize.js --target browser --format esm --minify && bun build js/segment.ts --outfile dist/segment.js --target browser --format esm --minify && bun build js/stem.ts --outfile dist/stem.js --target browser --format esm --minify && bun build js/romanize.ts --outfile dist/romanize.js --target browser --format esm --minify && bun build js/phonetic-search.ts --outfile dist/phonetic-search.js --target browser --format esm --minify && bun build js/deromanize.ts --outfile dist/deromanize.js --target browser --format esm --minify && bun build js/wellformed.ts --outfile dist/wellformed.js --target browser --format esm --minify && tsc -p tsconfig.json",
|
|
60
|
+
"build": "bun build js/phonetic.ts --outfile dist/phonetic.js --target browser --format esm --minify && bun build js/normalize.ts --outfile dist/normalize.js --target browser --format esm --minify && bun build js/segment.ts --outfile dist/segment.js --target browser --format esm --minify && bun build js/stem.ts --outfile dist/stem.js --target browser --format esm --minify && bun build js/stem-compounds.ts --outfile dist/stem-compounds.js --target browser --format esm --minify && bun build js/search-key.ts --outfile dist/search-key.js --target browser --format esm --minify && bun build js/romanize.ts --outfile dist/romanize.js --target browser --format esm --minify && bun build js/phonetic-search.ts --outfile dist/phonetic-search.js --target browser --format esm --minify && bun build js/deromanize.ts --outfile dist/deromanize.js --target browser --format esm --minify && bun build js/wellformed.ts --outfile dist/wellformed.js --target browser --format esm --minify && tsc -p tsconfig.json",
|
|
52
61
|
"test": "bun test",
|
|
53
62
|
"check-dist": "node js/check-dist.mjs"
|
|
54
63
|
},
|