@ozgurguler/trnorm 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +77 -3
- package/index.d.ts +30 -0
- package/package.json +50 -6
- package/src/case_tables.js +89 -0
- package/src/casing.js +34 -0
- package/src/index.js +19 -0
- package/src/itn.js +255 -0
- package/src/lexicon.js +489 -0
- package/src/normalizer.js +1513 -0
- package/src/numbers.js +325 -0
- package/src/py.js +438 -0
- package/src/unassigned.js +100 -0
|
@@ -0,0 +1,1513 @@
|
|
|
1
|
+
// trnorm/normalizer.py in JavaScript: written Turkish -> spoken Turkish (normalize), and the canonical form for scoring
|
|
2
|
+
// ASR and TTS (normalize_for_eval).
|
|
3
|
+
//
|
|
4
|
+
// The normalizer is a sequence of regex passes, from the most specific patterns (URLs, phone numbers, dates, times,
|
|
5
|
+
// money) to the most generic (bare numbers, acronyms, symbols). Every numeric pass understands an optional apostrophe
|
|
6
|
+
// suffix and re-agrees it with the spoken form:
|
|
7
|
+
//
|
|
8
|
+
// "15.00'de" -> "on beşte" "SGK'ye" -> "se ge kaya"
|
|
9
|
+
// "4'üncü" -> "dördüncü" "%15'lik" -> "yüzde on beşlik"
|
|
10
|
+
//
|
|
11
|
+
// A separately written question particle (mI, miydi, misiniz...) or clitic dA right after an expanded token is
|
|
12
|
+
// re-agreed the same way: "5 TL mi" -> "beş lira mı".
|
|
13
|
+
//
|
|
14
|
+
// The port keeps Python's names, order and patterns. Patterns are written with py.js's fragments: W/WC for \w, D for
|
|
15
|
+
// \d, S/SC for \s, B for \b, END for $, lb/nlb for a lookbehind on \w or \d. m[n] is match.group(n) (undefined for
|
|
16
|
+
// None), m.index and end(m) its span in UTF-16 units, so every slice Python sizes in characters goes through cp_back /
|
|
17
|
+
// cp_fwd / plen. Numbers are BigInts. Where a Python pattern takes O(n²) on a long run (the URL and e-mail patterns
|
|
18
|
+
// on "1.1.1...", the alnum lookaheads on "a-a-a...", a postcode search over everything before it), the port gets the
|
|
19
|
+
// same matches without the quadratic part (finditer objects below, each argued at its definition).
|
|
20
|
+
|
|
21
|
+
import * as L from './lexicon.js';
|
|
22
|
+
import { is_all_caps, tr_capitalize, tr_lower, tr_upper } from './casing.js';
|
|
23
|
+
import {
|
|
24
|
+
VOWELS,
|
|
25
|
+
attach_suffix,
|
|
26
|
+
digits_to_words,
|
|
27
|
+
harmonize_clitic,
|
|
28
|
+
harmonize_suffix,
|
|
29
|
+
number_to_words,
|
|
30
|
+
ordinal_to_words,
|
|
31
|
+
read_decimal_fraction,
|
|
32
|
+
read_digit_group,
|
|
33
|
+
read_long_digits,
|
|
34
|
+
} from './numbers.js';
|
|
35
|
+
import {
|
|
36
|
+
ANY, B, CI_A_Z, D, END, S, SC, U_PAIR_BUG, U_PAIR_RE, W, WC, char_at, char_before, ci, cp_back, cp_fwd, end, first_cp,
|
|
37
|
+
has, has_digit, in_table, isalnum, isalpha, isdecimal, isdigit, islower, isupper, last_cp, lb, lower, nlb, plen, py_int,
|
|
38
|
+
re, rstrip, split, split1, strip, type_error, upper,
|
|
39
|
+
} from './py.js';
|
|
40
|
+
import { UNASSIGNED } from './unassigned.js';
|
|
41
|
+
|
|
42
|
+
const LOW = 'a-zçğıöşüâîû';
|
|
43
|
+
const UP = 'A-ZÇĞİÖŞÜÂÎÛ';
|
|
44
|
+
const LET = LOW + UP;
|
|
45
|
+
|
|
46
|
+
// optional apostrophe suffix: "'te", "'DA" (uppercase suffixes in all-caps text)
|
|
47
|
+
const SUF = `(?:'([${LET}]+))?`;
|
|
48
|
+
// integer with optional dot thousands, optional comma decimal
|
|
49
|
+
const INT_TH = String.raw`${D}{1,3}(?:\.${D}{3})+`;
|
|
50
|
+
const NUM = `(?:${INT_TH}|${D}+)(?:,${D}+)?`;
|
|
51
|
+
// number in unit/currency context: also accepts dot decimals ("1.5 kg")
|
|
52
|
+
const NUMU = `(?:${INT_TH}(?:,${D}+)?|${D}+[.,]${D}+|${D}+)`;
|
|
53
|
+
const NB = nlb(`${WC}.,`); // number must not continue a word or another number
|
|
54
|
+
// NB where the number starts with a digit (or a sign): the same, with that first character tested first (faster)
|
|
55
|
+
const NBD = `(?=${D})${NB}`;
|
|
56
|
+
const NBS = `(?=[\\-${D}])${NB}`;
|
|
57
|
+
|
|
58
|
+
const _ORDINAL_SUFFIX_RE = re.compile(`^(?:[ıiuü]?nc[ıiuü])(${ANY}*)${END}`);
|
|
59
|
+
|
|
60
|
+
// a small int (a regex group of at most a few digits) as a JS number
|
|
61
|
+
const int = (s) => Number(py_int(s));
|
|
62
|
+
|
|
63
|
+
// --------------------------------------------------------------------------- helpers
|
|
64
|
+
|
|
65
|
+
function _suffix(stem, suffix) {
|
|
66
|
+
if (!suffix) return stem;
|
|
67
|
+
return attach_suffix(stem, tr_lower(suffix));
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// Attach a suffix to a spoken integer; ordinal suffixes become ordinals.
|
|
71
|
+
function _attach_number_suffix(n, words, suffix) {
|
|
72
|
+
if (!suffix) return words;
|
|
73
|
+
const s = tr_lower(suffix);
|
|
74
|
+
const m = _ORDINAL_SUFFIX_RE.match(s);
|
|
75
|
+
if (m) { // 4'üncü, 2'nci, 1'inci, 3'ncü (misspelt) -> ordinal (+ rest: "si", "de")
|
|
76
|
+
return m[1] ? _suffix(ordinal_to_words(n), m[1]) : ordinal_to_words(n);
|
|
77
|
+
}
|
|
78
|
+
return attach_suffix(words, s);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// Read "1.500", "3,5", "1.250,75", "1.5" (dot decimal), "-5".
|
|
82
|
+
export function read_number(s, dot_word = 'nokta') {
|
|
83
|
+
s = strip(s);
|
|
84
|
+
const neg = '-−'.includes(s.slice(0, 1)); // s[:1] in "-−" ("" is in every string)
|
|
85
|
+
if (neg) s = s.slice(1);
|
|
86
|
+
let out;
|
|
87
|
+
if (s.includes(',')) {
|
|
88
|
+
const [ip, fp] = split1(s, ',');
|
|
89
|
+
const ipv = py_int(ip.replaceAll('.', '') || '0');
|
|
90
|
+
if (fp === '5' && ipv > 0n) out = `${number_to_words(ipv)} buçuk`; // 1,5 -> bir buçuk
|
|
91
|
+
else out = `${number_to_words(ipv)} virgül ${read_decimal_fraction(fp)}`;
|
|
92
|
+
} else if (s.includes('.')) {
|
|
93
|
+
if (re.fullmatch(INT_TH, s)) {
|
|
94
|
+
out = number_to_words(py_int(s.replaceAll('.', '')));
|
|
95
|
+
} else {
|
|
96
|
+
const [ip, fp] = split1(s, '.');
|
|
97
|
+
if (dot_word === 'virgül' && fp === '5' && py_int(ip || '0') > 0n) out = `${number_to_words(py_int(ip))} buçuk`;
|
|
98
|
+
else out = `${number_to_words(py_int(ip || '0'))} ${dot_word} ${read_decimal_fraction(fp.replaceAll('.', ''))}`;
|
|
99
|
+
}
|
|
100
|
+
} else {
|
|
101
|
+
out = plen(s) <= 21 ? number_to_words(py_int(s)) : digits_to_words(s);
|
|
102
|
+
}
|
|
103
|
+
return neg ? `eksi ${out}` : out;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function _int_value(s) {
|
|
107
|
+
s = s.replaceAll('.', '');
|
|
108
|
+
return isdecimal(s) ? py_int(s) : null;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
const _BETWEEN_RE = re.compile(String.raw`[ \t]+(?:arası|arasında|arasındaki|aralığı|aralığında)${B}`);
|
|
112
|
+
|
|
113
|
+
// Range word: "ila" ("4 ila 5 dakika"), but "ile" before "arası(nda)" ("13 ile 17 arasında").
|
|
114
|
+
function _ila(text, end_) { // end_: Python's end (end() is the match helper here)
|
|
115
|
+
return _BETWEEN_RE.match(text, end_) ? 'ile' : 'ila';
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function _read_range(a, b, dot_word = 'nokta', word = 'ila') {
|
|
119
|
+
return `${read_number(a, dot_word)} ${word} ${read_number(b, dot_word)}`;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function _letters(token) {
|
|
123
|
+
const letters = [...token].filter(isalpha);
|
|
124
|
+
// a Latin I inside a spelled acronym is the letter i, not ı: IBM, HDMI, IETT typed without Turkish keys
|
|
125
|
+
return letters.map((ch) => (ch === 'I' && letters.length > 1 ? 'i' : (L.LETTER_NAMES.get(ch) ?? tr_lower(ch))))
|
|
126
|
+
.join(' ');
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function _pronounceable(word) {
|
|
130
|
+
const w = tr_lower(word);
|
|
131
|
+
const flags = [...w].filter(isalpha).map((c) => VOWELS.has(c));
|
|
132
|
+
const n = flags.length;
|
|
133
|
+
if (n <= 2 || !flags.some(Boolean)) return false;
|
|
134
|
+
let run = 0;
|
|
135
|
+
let maxrun = 0;
|
|
136
|
+
for (const v of flags) {
|
|
137
|
+
run = v ? 0 : run + 1;
|
|
138
|
+
maxrun = Math.max(maxrun, run);
|
|
139
|
+
}
|
|
140
|
+
if (n >= 5) return maxrun <= 3 && flags.filter(Boolean).length / n >= 0.3;
|
|
141
|
+
const starts_cc = !flags[0] && !flags[1];
|
|
142
|
+
const ends_cc = !flags[n - 1] && !flags[n - 2];
|
|
143
|
+
return maxrun <= 2 && !starts_cc && !ends_cc;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// --------------------------------------------------------------------------- clitics (mI, dA)
|
|
147
|
+
|
|
148
|
+
// Private-use char put after an expansion that is followed by a clitic; _pass_clitics consumes it.
|
|
149
|
+
const _MARK = '\u{e000}';
|
|
150
|
+
const _I4 = '[ıiuü]';
|
|
151
|
+
const _A2 = '[ae]';
|
|
152
|
+
const _PERS = `(?:m|n(?:${_I4}z)?|k|l${_A2}r)`; // -m, -n, -nIz, -k, -lAr after -DI / -sA
|
|
153
|
+
// question particle mI and its copula forms (miyim, misiniz, midir, miydi, miymiş, miyse...), clitic dA
|
|
154
|
+
const _CLITIC_RE = re.compile(
|
|
155
|
+
`d${_A2}|m${_I4}(?:y${_I4}[mz]|s${_I4}n(?:${_I4}z)?|d${_I4}r(?:l${_A2}r)?|yd${_I4}${_PERS}?` +
|
|
156
|
+
`|ym${_I4}ş(?:${_I4}[mz]|s${_I4}n(?:${_I4}z)?|l${_A2}r)?|ys${_A2}${_PERS}?)?`,
|
|
157
|
+
);
|
|
158
|
+
const _WORD_AFTER_RE = re.compile(String.raw`([ \t]+)([${LET}]+)(?![${WC}'])`);
|
|
159
|
+
const _QUESTION_NEXT_RE = re.compile(String.raw`${S}*["'’”»)\]]*${S}*\?`);
|
|
160
|
+
// source token of a plain cardinal / range / list: "2", "1.000", "2-3"
|
|
161
|
+
const _BARE_COUNT_RE = re.compile(String.raw`[${D}.,${SC}\-]+`);
|
|
162
|
+
|
|
163
|
+
// Match " <clitic>" at pos: all-lowercase or all-caps mI / dA forms only.
|
|
164
|
+
// count: the word before is a bare cardinal, after which "müdür" is the noun ("2 müdür?").
|
|
165
|
+
function _clitic_at(text, pos, count = false) {
|
|
166
|
+
const m = _WORD_AFTER_RE.match(text, pos);
|
|
167
|
+
if (!m) return null;
|
|
168
|
+
const word = m[2];
|
|
169
|
+
const low = tr_lower(word);
|
|
170
|
+
if ((word !== low && word !== tr_upper(low)) || !_CLITIC_RE.fullmatch(low)) return null;
|
|
171
|
+
// "müdür(ler)" is also a noun ("2 müdür, 3 memur", "İK müdür yardımcısı"): mI+dIr only right before "?"
|
|
172
|
+
if (low.startsWith('müdür') && (count || !_QUESTION_NEXT_RE.match(text, end(m)))) return null;
|
|
173
|
+
return m;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
function _last_word(s) {
|
|
177
|
+
const toks = split(s);
|
|
178
|
+
return toks.length ? tr_lower(toks[toks.length - 1]) : '';
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
// rx.sub(repl, text), marking expansions followed by a clitic so _pass_clitics re-agrees it.
|
|
182
|
+
// Only a changed last word counts as an expansion: "NATO" -> "nato" keeps the writer's "NATO mu".
|
|
183
|
+
function _sub(rx, repl, text) {
|
|
184
|
+
const out = [];
|
|
185
|
+
let pos = 0;
|
|
186
|
+
for (const m of rx.finditer(text)) {
|
|
187
|
+
const neu = repl(m);
|
|
188
|
+
out.push(text.slice(pos, m.index), neu);
|
|
189
|
+
const count = Boolean(_BARE_COUNT_RE.fullmatch(m[0]));
|
|
190
|
+
if (_last_word(neu) && _last_word(neu) !== _last_word(m[0]) && _clitic_at(text, end(m), count)) out.push(_MARK);
|
|
191
|
+
pos = end(m);
|
|
192
|
+
}
|
|
193
|
+
out.push(text.slice(pos));
|
|
194
|
+
return out.join('');
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
// Re-agree mI / dA after a marked expansion with its last spoken vowel: "beş lira mi" -> "beş lira mı".
|
|
198
|
+
// The only chain, dA + mI, is re-agreed left to right: "beş lira de mi" -> "beş lira da mı".
|
|
199
|
+
function _pass_clitics(text) {
|
|
200
|
+
const [head, ...parts] = text.split(_MARK);
|
|
201
|
+
let out = head;
|
|
202
|
+
for (const part of parts) {
|
|
203
|
+
let pos = 0;
|
|
204
|
+
let prev = '';
|
|
205
|
+
let m;
|
|
206
|
+
while ((m = _clitic_at(part, pos)) && (!prev || ((prev === 'da' || prev === 'de') && tr_lower(m[2])[0] === 'm'))) {
|
|
207
|
+
const word = m[2];
|
|
208
|
+
prev = harmonize_clitic(tr_lower(out.slice(cp_back(out, out.length, 48)) + m[1]), tr_lower(word));
|
|
209
|
+
out += m[1] + (word !== tr_lower(word) ? tr_upper(prev) : prev);
|
|
210
|
+
pos = end(m);
|
|
211
|
+
}
|
|
212
|
+
out += part.slice(pos);
|
|
213
|
+
}
|
|
214
|
+
return out;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function _context_before(text, pos, width = 32) {
|
|
218
|
+
return tr_lower(text.slice(cp_back(text, pos, width), pos));
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
// the text after the last digit (re.split(r"\d", context)[-1]) and its lowercase words
|
|
222
|
+
const _LOW_WORD_RE = re.compile(`[${LOW}]+`);
|
|
223
|
+
const _DIGIT_RE = re.compile(D);
|
|
224
|
+
const _tail_words = (context) => _LOW_WORD_RE.findall(_DIGIT_RE.split(context).pop());
|
|
225
|
+
|
|
226
|
+
// True if one of the last `last` words before a number starts with one of `words`.
|
|
227
|
+
// Digits break the window: in "kodunu gir, 21 Kasım 2025", the year is not a code.
|
|
228
|
+
function _has_word(context, words, last = 3) {
|
|
229
|
+
const toks = _tail_words(context);
|
|
230
|
+
if (toks.length && _MONTHS_LOWER.has(toks[toks.length - 1])) return false;
|
|
231
|
+
return toks.slice(-last).some((t) => words.some((w) => t.startsWith(w)));
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
// nominal suffixes after a cue word: possessive / plural / -lI / -sIz, then a case ending
|
|
235
|
+
// ("numara-nız", "kod-u-nu", "numara-sı-nda", "hesab-ı-na", "kod-lu", "kişi-ye", "yıl-ı-nda", "lira-lık")
|
|
236
|
+
const _NOMINAL_SUFFIX_RE = re.compile(
|
|
237
|
+
'(?:[ıiuü]|s[ıiuü]|[ıiuü]?m(?:[ıiuü]z)?|[ıiuü]?n(?:[ıiuü]z)?|l[ae]r(?:[ıi](?:m[ıi]z|n[ıi]z|[mn])?)?' +
|
|
238
|
+
'|l[ıiuü]k?|s[ıiuü]z)?' +
|
|
239
|
+
'(?:n?[ıiuü]n|y?[ae]|n[ae]|n?[dt][ae]n?|y?l[ae]|y?[ıiuü]|n[ıiuü])?',
|
|
240
|
+
);
|
|
241
|
+
|
|
242
|
+
// tok is one of `cues`, as a whole word or with nominal suffixes ("numaranız", "kişiye"); never a word that merely
|
|
243
|
+
// starts with a cue ("normalde" is not "no", "kartal" not "kart", "takipçi" not "takip").
|
|
244
|
+
function _cue_word(tok, cues) {
|
|
245
|
+
return cues.some(
|
|
246
|
+
(c) => tok === c || (!L.ID_EXACT.has(c) && tok.startsWith(c) && Boolean(_NOMINAL_SUFFIX_RE.fullmatch(tok, c.length))),
|
|
247
|
+
);
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// Like _has_word, with whole-word cue matching (_cue_word).
|
|
251
|
+
function _has_cue(context, cues, last) {
|
|
252
|
+
const toks = _tail_words(context);
|
|
253
|
+
if (toks.length && _MONTHS_LOWER.has(toks[toks.length - 1])) return false;
|
|
254
|
+
return toks.slice(-last).some((t) => _cue_word(t, cues));
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
const _NEXT_LOW_WORD_RE = re.compile(`[ \\t]*([${LOW}]+)`);
|
|
258
|
+
// A count, not an identifier: "Müşteri sayısı 25000", "Telefonla 15000 kişiye", "Kart limiti 25000 lira".
|
|
259
|
+
function _quantity(context, after) {
|
|
260
|
+
const toks = _tail_words(context);
|
|
261
|
+
if (toks.length && L.QUANTITY_BEFORE.has(toks[toks.length - 1])) return true;
|
|
262
|
+
const nxt = _NEXT_LOW_WORD_RE.match(after);
|
|
263
|
+
return Boolean(nxt) && _cue_word(nxt[1], L.QUANTITY_AFTER);
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
const _MONTHS_LOWER = new Set(L.MONTHS.filter(Boolean).map(tr_lower));
|
|
267
|
+
// spelled-out unit words after a dot decimal make it a quantity: "10.40 metre" -> on virgül kırk metre
|
|
268
|
+
const _UNIT_WORDS_RE = [...new Set([...L.UNITS.values(), 'kilo', 'lira', 'dolar', 'avro', 'kuruş', 'derece', 'saat',
|
|
269
|
+
'dakika', 'saniye', 'gün', 'yıl', 'ay', 'hafta'])].sort((a, b) => plen(b) - plen(a)).join('|');
|
|
270
|
+
|
|
271
|
+
// --------------------------------------------------------------------------- passes
|
|
272
|
+
|
|
273
|
+
// emoji, pictographs, flags, skin tones, keycap marks: not read aloud (a raw emoji reaches the TTS model otherwise).
|
|
274
|
+
// [\U0001F000-\U0001FAFF\U0001FC00-\U0001FFFD\u2600-\u27BF\u2B00-\u2BFF\u2300-\u23FF\uFE0E\uFE0F\u20E3]+
|
|
275
|
+
// Written without the u flag, the astral ranges as surrogate pairs (the text has no lone surrogate here): this and
|
|
276
|
+
// _INVISIBLE_RE delete characters, which can bring a pair Safari 26 misreads together, so they run before the check.
|
|
277
|
+
const _EMOJI_RE = new RegExp(
|
|
278
|
+
String.raw`(?:[\u2600-\u27bf\u2b00-\u2bff\u2300-\u23ff\ufe0e\ufe0f\u20e3]|\ud83c[\udc00-\udfff]` +
|
|
279
|
+
String.raw`|\ud83d[\udc00-\udfff]|\ud83e[\udc00-\udeff]|\ud83f[\udc00-\udffd])+`, 'g',
|
|
280
|
+
);
|
|
281
|
+
|
|
282
|
+
// the patterns of _preclean (compiled once; Python's re caches its inline patterns the same way)
|
|
283
|
+
const _INVISIBLE_RE = /[\u200b\u200c\u200d\ufeff]/g;
|
|
284
|
+
const _APOSTROPHE_RE = re.compile(String.raw`(?=[’‘\x60´ʼ′])${lb(WC)}[’‘\x60´ʼ′](?=${W})`, has('’‘`´ʼ′'));
|
|
285
|
+
const _QUOTE_RE = re.compile('[“”„«»]');
|
|
286
|
+
const _SINGLE_QUOTE_RE = re.compile('[’‘]');
|
|
287
|
+
const _DASH_NUM_RE = re.compile(`(?=[${SC}–—])${lb(D)}${S}?[–—]${S}?(?=${D})`, has('–—'));
|
|
288
|
+
const _DASH_RE = re.compile(`${S}[–—]${S}`, has('–—'));
|
|
289
|
+
const _BLANKS_RE = re.compile(String.raw`[ \t]+`);
|
|
290
|
+
|
|
291
|
+
// An Error for a text this engine cannot read right (py.js U_PAIR_BUG): not a Python exception, so the worker asks
|
|
292
|
+
// the Python front-end, which reads it.
|
|
293
|
+
function _engine_error(text) {
|
|
294
|
+
const at = text.search(U_PAIR_RE);
|
|
295
|
+
const pair = [...text.slice(at, at + 2)].map((c) => `U+${c.codePointAt(0).toString(16).toUpperCase()}`).join(' ');
|
|
296
|
+
const e = new Error(`this browser's RegExp misreads ${pair} (Safari 26.0 to 26.5)`);
|
|
297
|
+
e.name = 'EngineError';
|
|
298
|
+
return e;
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
function _preclean(text) {
|
|
302
|
+
text = text.normalize('NFC').replaceAll(_MARK, '');
|
|
303
|
+
text = text.replace(_EMOJI_RE, ' ');
|
|
304
|
+
text = text.replace(_INVISIBLE_RE, '');
|
|
305
|
+
// from here on a pass brings such a pair together only by deleting what stands between: _pass_symbols drops "#"
|
|
306
|
+
// and "@" (checked again there); the rest replace a match whose last character was already next to the following one
|
|
307
|
+
if (U_PAIR_BUG && U_PAIR_RE.test(text)) throw _engine_error(text);
|
|
308
|
+
text = text.replaceAll('\u{a0}', ' ').replaceAll('\u{202f}', ' ');
|
|
309
|
+
// apostrophes between word characters -> '
|
|
310
|
+
text = _APOSTROPHE_RE.sub("'", text);
|
|
311
|
+
text = _QUOTE_RE.sub('"', text);
|
|
312
|
+
text = _SINGLE_QUOTE_RE.sub("'", text);
|
|
313
|
+
text = text.replaceAll('…', '...');
|
|
314
|
+
// en/em dash between numbers -> hyphen (ranges); elsewhere a pause
|
|
315
|
+
text = _DASH_NUM_RE.sub('-', text);
|
|
316
|
+
text = _DASH_RE.sub(', ', text);
|
|
317
|
+
text = text.replaceAll('−', '-');
|
|
318
|
+
text = _BLANKS_RE.sub(' ', text);
|
|
319
|
+
return strip(text);
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
// Written forms of a FOREIGN_NAMES key: as written, ALL CAPS, and unless exact-case also lower/Capitalized.
|
|
323
|
+
// An exact-case key gets no Turkish ALL CAPS form either: "GEMİNİ" is the Turkish word "gemini" in capitals.
|
|
324
|
+
function _name_forms(key) {
|
|
325
|
+
if (L.FOREIGN_NAMES_EXACT_CASE.has(key)) return new Set([key, upper(key)]);
|
|
326
|
+
const low = lower(key);
|
|
327
|
+
const head = first_cp(low);
|
|
328
|
+
const tail = low.slice(head.length);
|
|
329
|
+
// ALL CAPS (Latin and Turkish: "INSTAGRAM", "İNSTAGRAM"), youtube, Youtube, İnstagram
|
|
330
|
+
return new Set([key, upper(key), tr_upper(key), low, upper(head) + tail, tr_upper(head) + tail]);
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
// Among keys of one length the order differs from Python's (a set's order there), which no match depends on: two
|
|
334
|
+
// different literals of one length never both match at one place.
|
|
335
|
+
function _build_names() {
|
|
336
|
+
const table = new Map();
|
|
337
|
+
for (const [key, spoken] of L.FOREIGN_NAMES) {
|
|
338
|
+
for (const form of _name_forms(key)) if (!table.has(form)) table.set(form, spoken);
|
|
339
|
+
}
|
|
340
|
+
for (const [key, spoken] of L.FOREIGN_NAMES) table.set(key, spoken); // an exact key beats another key's generated form
|
|
341
|
+
const alt = [...table.keys()].sort((a, b) => plen(b) - plen(a)).map(re.escape).join('|');
|
|
342
|
+
const first = [...new Set([...table.keys()].map(first_cp))].join('').replace(/[\\\]^-]/g, '\\$&');
|
|
343
|
+
// not inside a word, URL, e-mail address or handle ("google.com", "@google", "/google"); an optional
|
|
344
|
+
// apostrophe suffix ("Google'da"), then a word end ((?=[first]): a first letter of some name, tested first)
|
|
345
|
+
const rx = re.compile(
|
|
346
|
+
String.raw`(?=[${first}])${nlb(String.raw`${WC}@./:\\`)}(?<!(?=${W}).')(${alt})(?:'([${LET}]+))?` +
|
|
347
|
+
String.raw`(?![${WC}@+]|'[${LET}]|[./:]${W})`,
|
|
348
|
+
);
|
|
349
|
+
return [table, rx];
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
const [_NAMES, _NAMES_RE] = _build_names();
|
|
353
|
+
|
|
354
|
+
// Brands and loanwords to their Turkish pronunciation: "iPhone'umdan" -> "ayfonumdan", "Google'da" -> "Gugılda".
|
|
355
|
+
// A Capitalized name stays capitalized (Turkish rules: "Instagram" -> "İnstagram", never a dotless ı);
|
|
356
|
+
// ALL-CAPS and lowercase ones come out lowercase, like spelled acronyms.
|
|
357
|
+
function _pass_names(text) {
|
|
358
|
+
const repl = (m) => {
|
|
359
|
+
const written = m[1];
|
|
360
|
+
const spoken = _suffix(_NAMES.get(written), m[2]);
|
|
361
|
+
return isupper(first_cp(written)) && !is_all_caps(written) ? tr_capitalize(spoken) : spoken;
|
|
362
|
+
};
|
|
363
|
+
return _sub(_NAMES_RE, repl, text);
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
// re.I: ci() spells each letter the way Python's IGNORECASE matches it
|
|
367
|
+
const _HTTP = `${ci('http')}${ci('s')}?://`;
|
|
368
|
+
const _URL_TLDS = ['com', 'net', 'org', 'gov', 'edu', 'io', 'ai', 'tr', 'co', 'info', 'biz', 'me', 'app', 'dev'];
|
|
369
|
+
const _URL_TLD_ALT = _URL_TLDS.map(ci).join('|');
|
|
370
|
+
const _URL_SRC = String.raw`${B}(?:${_HTTP})?(?:${ci('www')}\.)?[${WC}\-]+(?:\.[${WC}\-]+)*\.(?:${_URL_TLD_ALT})` +
|
|
371
|
+
String.raw`(?:\.[${CI_A_Z}]{2})?(?:/[^${SC}]*)?`;
|
|
372
|
+
const _EMAIL_SRC = String.raw`${B}[${WC}.+\-]+@[${WC}\-]+(?:\.[${WC}\-]+)+${B}`;
|
|
373
|
+
const _LABEL_CHAR = new RegExp(`^[${WC}\\-]$`, 'u'); // [\w-]: a character of a domain label
|
|
374
|
+
const _LOCAL_CHAR = new RegExp(`^[${WC}.+\\-]$`, 'u'); // [\w.+-]: of an e-mail address before the "@"
|
|
375
|
+
|
|
376
|
+
// The labels joined by single dots ([\w-]+(?:\.[\w-]+)*, as long as it goes) around the dot at q, which has a label
|
|
377
|
+
// character before it and after it: [start, end).
|
|
378
|
+
function _chain(text, q) {
|
|
379
|
+
let a = q;
|
|
380
|
+
for (;;) {
|
|
381
|
+
while (a > 0 && _LABEL_CHAR.test(char_before(text, a))) a = cp_back(text, a, 1);
|
|
382
|
+
if (a >= 2 && text[a - 1] === '.' && _LABEL_CHAR.test(char_before(text, a - 1))) a--;
|
|
383
|
+
else break;
|
|
384
|
+
}
|
|
385
|
+
let e = q;
|
|
386
|
+
while (text[e] === '.' && _LABEL_CHAR.test(char_at(text, e + 1))) {
|
|
387
|
+
e++;
|
|
388
|
+
while (e < text.length && _LABEL_CHAR.test(char_at(text, e))) e = cp_fwd(text, e, 1);
|
|
389
|
+
}
|
|
390
|
+
return [a, e];
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
// re.compile(URL, re.I).finditer, the same matches without its O(n²) part. The labels before ".com" are
|
|
394
|
+
// [\w-]+(?:\.[\w-]+)*, so Python's search, at every word boundary of a long chain like "1.1.1..." or "a-a-a...", takes
|
|
395
|
+
// in the rest of the chain and gives it back looking for a ".tld". Here the pattern runs only where it can match.
|
|
396
|
+
// A match ends its labels at a "site", a dot with a label character before it and a TLD after it, and starts in the
|
|
397
|
+
// chain of that site (or up to 8 characters before it: "https://"). From a start p the labels take the chain to its
|
|
398
|
+
// end and give back to the last site after p, so p matches if any site of the chain comes after it (and the word
|
|
399
|
+
// boundary holds), and a start that fails because no site is left fails for every later start of the chain too. So
|
|
400
|
+
// for the first site after pos the starts from max(pos, chain start - 8) to the chain's last site are tried in order
|
|
401
|
+
// (the first word boundary before a label matches); if none matches, the chain is done.
|
|
402
|
+
const _URL_RE = {
|
|
403
|
+
sticky: new RegExp(_URL_SRC, 'uy'),
|
|
404
|
+
sites: new RegExp(`${lb(`${WC}\\-`)}\\.(?=${_URL_TLD_ALT})`, 'gu'),
|
|
405
|
+
*finditer(text) {
|
|
406
|
+
const sites = [];
|
|
407
|
+
this.sites.lastIndex = 0;
|
|
408
|
+
for (let m; (m = this.sites.exec(text));) sites.push(m.index);
|
|
409
|
+
let pos = 0;
|
|
410
|
+
let k = 0;
|
|
411
|
+
while (k < sites.length) {
|
|
412
|
+
if (sites[k] <= pos) { // a match from pos on ends its labels after pos
|
|
413
|
+
k++;
|
|
414
|
+
continue;
|
|
415
|
+
}
|
|
416
|
+
const [start, stop] = _chain(text, sites[k]);
|
|
417
|
+
let last = k;
|
|
418
|
+
while (last + 1 < sites.length && sites[last + 1] < stop) last++;
|
|
419
|
+
let m = null;
|
|
420
|
+
for (let p = Math.max(pos, cp_back(text, start, 8)); p < sites[last] && !m; p = cp_fwd(text, p, 1)) {
|
|
421
|
+
this.sticky.lastIndex = p;
|
|
422
|
+
m = this.sticky.exec(text);
|
|
423
|
+
}
|
|
424
|
+
if (m) {
|
|
425
|
+
yield m;
|
|
426
|
+
pos = end(m);
|
|
427
|
+
} else {
|
|
428
|
+
k = last + 1;
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
},
|
|
432
|
+
};
|
|
433
|
+
|
|
434
|
+
// _EMAIL_RE.finditer, the same matches without its O(n²) part. The part before "@" is [\w.+-]+, so Python's search
|
|
435
|
+
// takes in a whole run of those characters from each of its word boundaries ("ş+ş+ş+...", "1.1.1...") and fails at
|
|
436
|
+
// the end when no "@" follows. A match starts in the run right before an "@": from any start in it, that part runs to
|
|
437
|
+
// the "@" and the rest of the match does not depend on the start, so only the run's first word boundary (at or after
|
|
438
|
+
// pos) needs trying; if it fails, every start before that "@" does.
|
|
439
|
+
const _EMAIL_RE = {
|
|
440
|
+
sticky: new RegExp(_EMAIL_SRC, 'uy'),
|
|
441
|
+
boundary: new RegExp(B, 'gu'),
|
|
442
|
+
*finditer(text) {
|
|
443
|
+
let pos = 0;
|
|
444
|
+
for (let at = text.indexOf('@'); at >= 0; at = text.indexOf('@', at + 1)) {
|
|
445
|
+
if (at <= pos) continue;
|
|
446
|
+
let a = at;
|
|
447
|
+
while (a > pos && _LOCAL_CHAR.test(char_before(text, a))) a = cp_back(text, a, 1);
|
|
448
|
+
if (a === at) continue;
|
|
449
|
+
this.boundary.lastIndex = a;
|
|
450
|
+
const b = this.boundary.exec(text);
|
|
451
|
+
if (!b || b.index >= at) continue;
|
|
452
|
+
this.sticky.lastIndex = b.index;
|
|
453
|
+
const m = this.sticky.exec(text);
|
|
454
|
+
if (m) {
|
|
455
|
+
yield m;
|
|
456
|
+
pos = end(m);
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
},
|
|
460
|
+
};
|
|
461
|
+
|
|
462
|
+
const _TLD = new Map(Object.entries({ com: 'kom', tr: 'te re', net: 'net', org: 'org', gov: 'gov', edu: 'edu', io: 'ay o',
|
|
463
|
+
ai: 'ey ay' }));
|
|
464
|
+
|
|
465
|
+
function _read_web_token(tok) {
|
|
466
|
+
const parts = [];
|
|
467
|
+
for (const piece of re.split(String.raw`([./@_\-])`, tok)) {
|
|
468
|
+
if (!piece) continue;
|
|
469
|
+
if (piece === '.') parts.push('nokta');
|
|
470
|
+
else if (piece === '/') parts.push('eğik çizgi');
|
|
471
|
+
else if (piece === '@') parts.push('et');
|
|
472
|
+
else if (piece === '_') parts.push('alt çizgi');
|
|
473
|
+
else if (piece === '-') parts.push('tire');
|
|
474
|
+
else if (lower(piece) === 'www') parts.push('ve ve ve');
|
|
475
|
+
else if (_TLD.has(lower(piece))) parts.push(_TLD.get(lower(piece)));
|
|
476
|
+
else if (isdecimal(piece)) { // not "²" ("¹".isdigit() is True)
|
|
477
|
+
parts.push(plen(piece) <= 4 ? read_digit_group(piece) : digits_to_words(piece));
|
|
478
|
+
} else parts.push(tr_lower(piece));
|
|
479
|
+
}
|
|
480
|
+
return parts.join(' ');
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
function _pass_web(text) {
|
|
484
|
+
text = _sub(_EMAIL_RE, (m) => _read_web_token(m[0]), text);
|
|
485
|
+
|
|
486
|
+
const url = (m) => {
|
|
487
|
+
const tok = rstrip(re.sub(`^${_HTTP}`, '', m[0]), '/');
|
|
488
|
+
return _read_web_token(tok);
|
|
489
|
+
};
|
|
490
|
+
|
|
491
|
+
return _sub(_URL_RE, url, text);
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
function _build_abbr_re() {
|
|
495
|
+
const keys = [...L.ABBREVIATIONS.keys()].sort((a, b) => plen(b) - plen(a));
|
|
496
|
+
const alt = keys.map(re.escape).join('|');
|
|
497
|
+
const first = [...new Set(keys.map(first_cp))].join('');
|
|
498
|
+
// every key has a "." or a ":" ("Dr.", "No:")
|
|
499
|
+
return re.compile(`(?=[${first}])${nlb(`${WC}.`)}(${alt})(?=[${SC},;:!?)"']|${END})`, has('.:'));
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
const _ABBR_RE = _build_abbr_re();
|
|
503
|
+
const _PAGE_RE = re.compile(String.raw`(?=[sc])${nlb(`${WC}.`)}([sc])\.${S}?(?=${D})`, has_digit);
|
|
504
|
+
const _NUMBER_LABEL_RE = re.compile(`(?=N)${nlb(`${WC}.`)}(No|Nu|NO)${S}?[:.]${S}?(?=${D})`, has_digit);
|
|
505
|
+
const _LABEL_RE = re.compile(`(?=[KDBTG])${nlb(`${WC}.`)}(Kat|Daire|Blok|D|Tel|GSM)${S}?:${S}?(?=${D})`, has(':'));
|
|
506
|
+
const _LABEL_WORDS = new Map([['D', 'daire'], ['Tel', 'telefon'], ['GSM', 'ge se me']]);
|
|
507
|
+
const _SPACE_CAPITAL_RE = re.compile(`${S}+[${UP}]`);
|
|
508
|
+
|
|
509
|
+
function _pass_abbreviations(text) {
|
|
510
|
+
const repl = (m) => {
|
|
511
|
+
const key = m[1];
|
|
512
|
+
let out = L.ABBREVIATIONS.get(key);
|
|
513
|
+
if (L.SENTENCE_FINAL_ABBR.has(key)) {
|
|
514
|
+
const rest = text.slice(end(m));
|
|
515
|
+
if (!strip(rest) || _SPACE_CAPITAL_RE.match(rest)) out += '.';
|
|
516
|
+
}
|
|
517
|
+
return out;
|
|
518
|
+
};
|
|
519
|
+
|
|
520
|
+
text = _sub(_ABBR_RE, repl, text);
|
|
521
|
+
text = _NUMBER_LABEL_RE.sub('numara ', text);
|
|
522
|
+
text = _LABEL_RE.sub((m) => `${_LABEL_WORDS.get(m[1]) ?? m[1]} `, text);
|
|
523
|
+
return _PAGE_RE.sub((m) => (m[1] === 's' ? 'sayfa ' : 'cilt '), text);
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
const _PRODUCT_RE = re.compile(`(?=[ \\tx×*])${lb(D)}[ \\t]?[x×*][ \\t]?(?=[${D}₺$€£])`, has('x×*'));
|
|
527
|
+
|
|
528
|
+
// x, ×, * between numbers is "çarpı" ("12 x 833,25 TL", "4*5"); runs before the numbers become words.
|
|
529
|
+
function _pass_products(text) {
|
|
530
|
+
const repl = (m) => {
|
|
531
|
+
const s = m.index;
|
|
532
|
+
// hexadecimal 0x1F (text[s - 1] is "0", one UTF-16 unit, so s < 2 counts characters too)
|
|
533
|
+
if (strip(m[0]) === 'x' && text[s - 1] === '0' && (s < 2 || !isalnum(char_before(text, s - 1)))) return m[0];
|
|
534
|
+
return ' çarpı ';
|
|
535
|
+
};
|
|
536
|
+
|
|
537
|
+
return _PRODUCT_RE.sub(repl, text);
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
const _PHONE_RE = re.compile(
|
|
541
|
+
String.raw`(?=[+(${D}])${nlb(`${WC}+`)}(\+90[${SC}.\-]?|0[${SC}.\-]?)?\(?([2-58]${D}{2})\)?[${SC}.\-]?` +
|
|
542
|
+
String.raw`(${D}{3})[${SC}.\-]?` +
|
|
543
|
+
String.raw`(${D}{2})[${SC}.\-]?(${D}{2})${SUF}(?!${W})`, has_digit,
|
|
544
|
+
);
|
|
545
|
+
const _PHONE_444_RE = re.compile(
|
|
546
|
+
String.raw`(?=4)${nlb(WC)}(444)[${SC}.\-]?(${D})[${SC}.\-]?(${D}{3})(?!${D})`, (t) => t.includes('444'),
|
|
547
|
+
);
|
|
548
|
+
|
|
549
|
+
function _pass_phones(text) {
|
|
550
|
+
const repl = (m) => {
|
|
551
|
+
const [raw, prefix, area, a, b, c, suf] = m;
|
|
552
|
+
// without a 0/+90 prefix, require separators so plain 10-digit ids stay ids
|
|
553
|
+
if (!prefix && !re.search(`[${SC}.()\\-]`, raw)) return raw;
|
|
554
|
+
let parts;
|
|
555
|
+
if (prefix && prefix.startsWith('+')) parts = ['artı doksan', number_to_words(py_int(area))];
|
|
556
|
+
else if (prefix) parts = [`sıfır ${number_to_words(py_int(area))}`];
|
|
557
|
+
else parts = [number_to_words(py_int(area))];
|
|
558
|
+
parts.push(read_digit_group(a), read_digit_group(b), _suffix(read_digit_group(c), suf));
|
|
559
|
+
return parts.join(', ');
|
|
560
|
+
};
|
|
561
|
+
|
|
562
|
+
text = _sub(_PHONE_RE, repl, text);
|
|
563
|
+
return _sub(
|
|
564
|
+
_PHONE_444_RE,
|
|
565
|
+
(m) => `dört yüz kırk dört, ${read_digit_group(m[2])}, ${read_digit_group(m[3])}`,
|
|
566
|
+
text,
|
|
567
|
+
);
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
const _IBAN_RE = re.compile(
|
|
571
|
+
String.raw`(?=T)${B}TR${S}?(${D}{2})((?:${S}?${D}{4}){5})${S}?(${D}{2})${B}`, (t) => t.includes('TR'),
|
|
572
|
+
);
|
|
573
|
+
const _FOUR_DIGITS_RE = re.compile(`${D}{4}`);
|
|
574
|
+
|
|
575
|
+
function _pass_iban(text) {
|
|
576
|
+
const repl = (m) => {
|
|
577
|
+
// check digits and the last pair as numbers, the four-digit groups digit by digit (easy to write down)
|
|
578
|
+
const groups = [read_digit_group(m[1])];
|
|
579
|
+
for (const g of _FOUR_DIGITS_RE.findall(m[2])) groups.push(digits_to_words(g));
|
|
580
|
+
groups.push(read_digit_group(m[3]));
|
|
581
|
+
return `te re ${groups.join(', ')}`;
|
|
582
|
+
};
|
|
583
|
+
|
|
584
|
+
return _sub(_IBAN_RE, repl, text);
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
const _PLATE_RE = re.compile(
|
|
588
|
+
String.raw`(?=[0-8])${nlb(WC)}(0[1-9]|[1-7]${D}|8[01])${S}([${UP}]{1,3})${S}(${D}{2,5})${SUF}(?!${W})`, has_digit,
|
|
589
|
+
);
|
|
590
|
+
|
|
591
|
+
function _pass_plates(text) {
|
|
592
|
+
const repl = (m) => {
|
|
593
|
+
const [, city, letters, num, suf] = m;
|
|
594
|
+
let tail;
|
|
595
|
+
if (plen(num) === 4) {
|
|
596
|
+
const d = [...num];
|
|
597
|
+
tail = `${read_digit_group(d.slice(0, 2).join(''))} ${read_digit_group(d.slice(2).join(''))}`;
|
|
598
|
+
} else {
|
|
599
|
+
tail = read_digit_group(num);
|
|
600
|
+
}
|
|
601
|
+
return _suffix(`${read_digit_group(city)} ${_letters(letters)} ${tail}`, suf);
|
|
602
|
+
};
|
|
603
|
+
|
|
604
|
+
return _sub(_PLATE_RE, repl, text);
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
function _date_words(d, mo, y) {
|
|
608
|
+
let out = `${number_to_words(d)} ${L.MONTHS[mo]}`;
|
|
609
|
+
if (y) out += ` ${number_to_words(py_int(y))}`;
|
|
610
|
+
return out;
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
const _DATE_RE = re.compile(
|
|
614
|
+
String.raw`${NBD}(${D}{1,2})([./\-])(${D}{1,2})\2(${D}{4}|${D}{2})${SUF}(?!${W}|[.,]${D})`, has_digit,
|
|
615
|
+
);
|
|
616
|
+
const _MONTH_YEAR_RE = re.compile(
|
|
617
|
+
String.raw`${NBD}(0?[1-9]|1[0-2])[./]((?:19|20)${D}{2})${SUF}(?!${W}|[.,]${D})`, has_digit,
|
|
618
|
+
);
|
|
619
|
+
const _ISO_DATE_RE = re.compile(String.raw`${NBD}(${D}{4})-(${D}{2})-(${D}{2})${SUF}(?!${W}|[.,]${D})`, has_digit);
|
|
620
|
+
// year first with slashes or dots: 2026/12/31, 2026/1/5, 2026.12.31
|
|
621
|
+
const _YMD_RE = re.compile(
|
|
622
|
+
String.raw`${NBD}((?:1[89]|20)${D}{2})(?:/(${D}{1,2})/(${D}{1,2})|\.(${D}{2})\.(${D}{2}))${SUF}(?!${W}|[.,/]${D})`,
|
|
623
|
+
has_digit,
|
|
624
|
+
);
|
|
625
|
+
|
|
626
|
+
function _pass_dates(text) {
|
|
627
|
+
const iso = (m) => {
|
|
628
|
+
const [, y, mo, d, suf] = m;
|
|
629
|
+
if (int(mo) >= 1 && int(mo) <= 12 && int(d) >= 1 && int(d) <= 31) {
|
|
630
|
+
return _suffix(_date_words(int(d), int(mo), y), suf);
|
|
631
|
+
}
|
|
632
|
+
return m[0];
|
|
633
|
+
};
|
|
634
|
+
|
|
635
|
+
const ymd = (m) => {
|
|
636
|
+
const [, y, mo1, d1, mo2, d2, suf] = m;
|
|
637
|
+
const mo = int(mo1 || mo2);
|
|
638
|
+
const d = int(d1 || d2);
|
|
639
|
+
if (mo >= 1 && mo <= 12 && d >= 1 && d <= 31) return _suffix(_date_words(d, mo, y), suf);
|
|
640
|
+
return m[0];
|
|
641
|
+
};
|
|
642
|
+
|
|
643
|
+
text = _sub(_YMD_RE, ymd, text);
|
|
644
|
+
|
|
645
|
+
const dmy = (m) => {
|
|
646
|
+
const [, d, , mo, y, suf] = m;
|
|
647
|
+
if (int(mo) >= 1 && int(mo) <= 12 && int(d) >= 1 && int(d) <= 31) {
|
|
648
|
+
return _suffix(_date_words(int(d), int(mo), y), suf);
|
|
649
|
+
}
|
|
650
|
+
return m[0];
|
|
651
|
+
};
|
|
652
|
+
|
|
653
|
+
text = _sub(_DATE_RE, dmy, _sub(_ISO_DATE_RE, iso, text));
|
|
654
|
+
|
|
655
|
+
const month_year = (m) => _suffix(`${L.MONTHS[int(m[1])]} ${number_to_words(py_int(m[2]))}`, m[3]);
|
|
656
|
+
|
|
657
|
+
return _sub(_MONTH_YEAR_RE, month_year, text);
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
function _time_words(h, mi, s = null) {
|
|
661
|
+
let out = number_to_words(py_int(h));
|
|
662
|
+
if (s != null) return `${out} ${read_digit_group(mi)} ${read_digit_group(s)}`;
|
|
663
|
+
if (mi !== '00') out += ` ${read_digit_group(mi)}`;
|
|
664
|
+
return out;
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
const _TIME_COLON_RE = re.compile(
|
|
668
|
+
String.raw`${NBD}([01]?${D}|2[0-4]):([0-5]${D})(?::([0-5]${D}))?${SUF}(?!${W}|[.,]${D})`, has_digit,
|
|
669
|
+
);
|
|
670
|
+
const _TIME_DOT_RE = re.compile(String.raw`${NBD}([01]?${D}|2[0-4])\.([0-5]${D})${SUF}(?!${W}|[.,]${D})`, has_digit);
|
|
671
|
+
const _TIME_RANGE_RE = re.compile(
|
|
672
|
+
String.raw`${NBD}([01]?${D}|2[0-4])[:.]([0-5]${D})${S}?-${S}?([01]?${D}|2[0-4])[:.]([0-5]${D})${SUF}` +
|
|
673
|
+
String.raw`(?!${W}|[.,]${D})`,
|
|
674
|
+
has_digit,
|
|
675
|
+
);
|
|
676
|
+
|
|
677
|
+
const _SIGN_BEFORE_RE = re.compile(`[%‰₺$€£]${S}?${END}`);
|
|
678
|
+
const _TIME_WORD_AFTER_RE = re.compile(`${S}+(?:${L.TIME_CONTEXT.join('|')})`);
|
|
679
|
+
|
|
680
|
+
function _pass_times(text) {
|
|
681
|
+
text = _sub(
|
|
682
|
+
_TIME_RANGE_RE,
|
|
683
|
+
(m) => _suffix(`${_time_words(m[1], m[2])} ${_ila(text, end(m))} ${_time_words(m[3], m[4])}`, m[5]),
|
|
684
|
+
text,
|
|
685
|
+
);
|
|
686
|
+
text = _sub(_TIME_COLON_RE, (m) => _suffix(_time_words(m[1], m[2], m[3]), m[4]), text);
|
|
687
|
+
|
|
688
|
+
const dot = (m) => {
|
|
689
|
+
const [, h, mi, suf] = m;
|
|
690
|
+
if (_SIGN_BEFORE_RE.search(text.slice(cp_back(text, m.index, 2), m.index))) return m[0];
|
|
691
|
+
const before = _context_before(text, m.index);
|
|
692
|
+
const after = tr_lower(text.slice(end(m), cp_fwd(text, end(m), 16)));
|
|
693
|
+
let timey = Boolean(suf) || _has_word(before, L.TIME_CONTEXT) || mi === '00' || mi === '30';
|
|
694
|
+
timey = timey || Boolean(_TIME_WORD_AFTER_RE.match(after));
|
|
695
|
+
if (!timey) return m[0];
|
|
696
|
+
return _suffix(_time_words(h, mi), suf);
|
|
697
|
+
};
|
|
698
|
+
|
|
699
|
+
return _sub(_TIME_DOT_RE, dot, text);
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
const _CUR_SYMBOLS = String.raw`₺|\$|€|£|¥|₽`;
|
|
703
|
+
const _CUR_CODES = 'TL|TRY|USD|EUR|GBP|JPY|RUB|CHF';
|
|
704
|
+
const _CUR_PREFIX_RE = re.compile(
|
|
705
|
+
`(?=[₺$€£¥₽])${nlb(WC)}(${_CUR_SYMBOLS})${S}?(${NUMU})${SUF}(?!${W}|[.,]${D})`, has('₺$€£¥₽'),
|
|
706
|
+
);
|
|
707
|
+
const _CUR_POSTFIX_RE = re.compile(
|
|
708
|
+
`${NBD}(${NUMU})(?:${S}?-${S}?(${NUMU}))?${S}?(${_CUR_SYMBOLS}|(?:${_CUR_CODES})(?!${W}))${SUF}(?!${W}|[.,]${D})`,
|
|
709
|
+
has_digit,
|
|
710
|
+
);
|
|
711
|
+
const _SCALE_WORDS = /bin|milyon|milyar|trilyon/;
|
|
712
|
+
const _SCALE_CUR_RE = re.compile(
|
|
713
|
+
`(?=[bmt])${B}(bin|milyon|milyar|trilyon)${S}(${_CUR_SYMBOLS}|${_CUR_CODES})(?!${W})${SUF}`,
|
|
714
|
+
(t) => _SCALE_WORDS.test(t),
|
|
715
|
+
);
|
|
716
|
+
const _AMOUNT_RE = re.compile(`(${INT_TH}|${D}+)[.,](${D}{2})`);
|
|
717
|
+
|
|
718
|
+
function _money_words(amount, cur_word) {
|
|
719
|
+
const sub = L.CURRENCY_SUBUNIT.get(cur_word);
|
|
720
|
+
const m = _AMOUNT_RE.fullmatch(amount);
|
|
721
|
+
if (sub && m && !re.fullmatch(INT_TH, amount)) {
|
|
722
|
+
const major = number_to_words(py_int(m[1].replaceAll('.', '')));
|
|
723
|
+
const minor = int(m[2]);
|
|
724
|
+
return `${major} ${cur_word}${minor ? ` ${number_to_words(minor)} ${sub}` : ''}`;
|
|
725
|
+
}
|
|
726
|
+
return `${read_number(amount, 'virgül')} ${cur_word}`;
|
|
727
|
+
}
|
|
728
|
+
|
|
729
|
+
function _pass_money(text) {
|
|
730
|
+
const prefix = (m) => {
|
|
731
|
+
const [, sym, amount, suf] = m;
|
|
732
|
+
return _suffix(_money_words(amount, L.CURRENCY_WORDS.get(sym)), suf);
|
|
733
|
+
};
|
|
734
|
+
|
|
735
|
+
const postfix = (m) => {
|
|
736
|
+
const [, a, b, cur, suf] = m;
|
|
737
|
+
const word = L.CURRENCY_WORDS.get(cur);
|
|
738
|
+
if (b) return _suffix(`${read_number(a, 'virgül')} ${_ila(text, end(m))} ${_money_words(b, word)}`, suf);
|
|
739
|
+
return _suffix(_money_words(a, word), suf);
|
|
740
|
+
};
|
|
741
|
+
|
|
742
|
+
text = _sub(_CUR_PREFIX_RE, prefix, text);
|
|
743
|
+
text = _sub(_CUR_POSTFIX_RE, postfix, text);
|
|
744
|
+
return _sub(_SCALE_CUR_RE, (m) => _suffix(`${m[1]} ${L.CURRENCY_WORDS.get(m[2])}`, m[3]), text);
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
const _PCT_PREFIX_RE = re.compile(
|
|
748
|
+
`(?=[%‰])${nlb(WC)}([%‰])${S}?(${NUMU})(?:${S}?-${S}?%?${S}?(${NUMU}))?${SUF}(?!${W}|[.,]${D})`, has('%‰'),
|
|
749
|
+
);
|
|
750
|
+
const _PCT_POSTFIX_RE = re.compile(`${NBD}(${NUMU})${S}?%${SUF}(?!${W})`, has('%'));
|
|
751
|
+
|
|
752
|
+
function _pass_percent(text) {
|
|
753
|
+
const pre = (m) => {
|
|
754
|
+
const [, sym, a, b, suf] = m;
|
|
755
|
+
const word = sym === '%' ? 'yüzde' : 'binde';
|
|
756
|
+
const body = !b ? read_number(a, 'virgül') : _read_range(a, b, 'virgül', _ila(text, end(m)));
|
|
757
|
+
return _suffix(`${word} ${body}`, suf);
|
|
758
|
+
};
|
|
759
|
+
|
|
760
|
+
text = _sub(_PCT_PREFIX_RE, pre, text);
|
|
761
|
+
return _sub(_PCT_POSTFIX_RE, (m) => _suffix(`yüzde ${read_number(m[1], 'virgül')}`, m[2]), text);
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
const _DEG_RE = re.compile(`${NBS}(-?${NUMU})${S}?°${S}?([CF])?(?![${LET}])${SUF}`, has('°'));
|
|
765
|
+
|
|
766
|
+
function _pass_degrees(text) {
|
|
767
|
+
const repl = (m) => {
|
|
768
|
+
const [, num, scale, suf] = m;
|
|
769
|
+
let out = `${read_number(num, 'virgül')} derece`;
|
|
770
|
+
if (scale === 'F') out += ' fahrenhayt';
|
|
771
|
+
return _suffix(out, suf);
|
|
772
|
+
};
|
|
773
|
+
|
|
774
|
+
return _sub(_DEG_RE, repl, text);
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
const _by_length = (keys) => [...keys].sort((a, b) => plen(b) - plen(a)); // sorted(key=len, reverse=True)
|
|
778
|
+
|
|
779
|
+
function _build_unit_re() {
|
|
780
|
+
const alt = _by_length(L.UNITS.keys()).map(re.escape).join('|');
|
|
781
|
+
return re.compile(`${NBS}(-?${NUMU})(?:${S}?-${S}?(${NUMU}))?${S}?(${alt})(?![${WC}²³/])${SUF}`, has_digit);
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
const _UNIT_RE = _build_unit_re();
|
|
785
|
+
const _SPEED_RE = re.compile(
|
|
786
|
+
`${NBD}(${NUMU})(?:${S}?-${S}?(${NUMU}))?${S}?(${_by_length(L.SPEED_UNITS.keys()).map(re.escape).join('|')})` +
|
|
787
|
+
`(?!${W})${SUF}`, has_digit,
|
|
788
|
+
);
|
|
789
|
+
|
|
790
|
+
function _pass_units(text) {
|
|
791
|
+
const speed = (m) => {
|
|
792
|
+
const [, a, b, unit, suf] = m;
|
|
793
|
+
const [per, word] = L.SPEED_UNITS.get(unit);
|
|
794
|
+
const num = !b ? read_number(a, 'virgül') : _read_range(a, b, 'virgül', _ila(text, end(m)));
|
|
795
|
+
return _suffix(`${per} ${num} ${word}`, suf);
|
|
796
|
+
};
|
|
797
|
+
|
|
798
|
+
const unit = (m) => {
|
|
799
|
+
const [, a, b, u, suf] = m;
|
|
800
|
+
const num = !b ? read_number(a, 'virgül') : _read_range(a, b, 'virgül', _ila(text, end(m)));
|
|
801
|
+
return _suffix(`${num} ${L.UNITS.get(u)}`, suf);
|
|
802
|
+
};
|
|
803
|
+
|
|
804
|
+
text = _sub(_SPEED_RE, speed, text);
|
|
805
|
+
return _sub(_UNIT_RE, unit, text);
|
|
806
|
+
}
|
|
807
|
+
|
|
808
|
+
// tokens mixing letters and digits: TR42648163, A4, 5G, MP3, H2O, COVID-19, F-16
|
|
809
|
+
// ...with a decimal version at the end: GPT-3.5
|
|
810
|
+
// (?<![\w'])(?=[LET\d-]*\d)(?=[LET\d-]*[UP])([LET\d]+(?:-[LET\d]+)*(?:(?<=\d)[.,]\d+)?)SUF(?![\w]|[.,]\d)
|
|
811
|
+
// The two lookaheads scan the run of [LET\d-] from each start: on a long run without a digit or a capital ("a-a-a...",
|
|
812
|
+
// "-----") Python tries every position of it, O(n²). finditer runs the pattern without them and tests them on what it
|
|
813
|
+
// matches: they hold for the same run from a later start only if they hold from this one (its rest of the run is
|
|
814
|
+
// shorter), so when they fail, no match starts before the run's end. A text without "-" has only one start per run
|
|
815
|
+
// (the others follow a letter or digit), so there the pattern as Python writes it is linear and runs as it is.
|
|
816
|
+
const _ALNUM_SRC = `${nlb(`${WC}'`)}(?=[${LET}${D}\\-]*${D})(?=[${LET}${D}\\-]*[${UP}])` +
|
|
817
|
+
`([${LET}${D}]+(?:-[${LET}${D}]+)*(?:${lb(D)}[.,]${D}+)?)${SUF}(?!${W}|[.,]${D})`;
|
|
818
|
+
const _HAS_CAPITAL = new RegExp(`[${UP}]`);
|
|
819
|
+
const _ALNUM_RE = {
|
|
820
|
+
plain: re.compile(`(?=[${LET}${D}])${_ALNUM_SRC}`),
|
|
821
|
+
rx: new RegExp(
|
|
822
|
+
`(?=[${LET}${D}])${nlb(`${WC}'`)}([${LET}${D}]+(?:-[${LET}${D}]+)*(?:${lb(D)}[.,]${D}+)?)${SUF}` +
|
|
823
|
+
`(?!${W}|[.,]${D})`, 'gu',
|
|
824
|
+
),
|
|
825
|
+
digit: new RegExp(`[${LET}${D}\\-]*${D}`, 'uy'),
|
|
826
|
+
capital: new RegExp(`[${LET}${D}\\-]*[${UP}]`, 'uy'),
|
|
827
|
+
run: new RegExp(`[${LET}${D}\\-]*`, 'uy'),
|
|
828
|
+
*finditer(text) {
|
|
829
|
+
if (!has_digit(text) || !_HAS_CAPITAL.test(text)) return; // every match has both
|
|
830
|
+
if (!text.includes('-')) {
|
|
831
|
+
yield* this.plain.finditer(text);
|
|
832
|
+
return;
|
|
833
|
+
}
|
|
834
|
+
let pos = 0;
|
|
835
|
+
for (;;) {
|
|
836
|
+
this.rx.lastIndex = pos;
|
|
837
|
+
const m = this.rx.exec(text);
|
|
838
|
+
if (!m) return;
|
|
839
|
+
this.digit.lastIndex = this.capital.lastIndex = this.run.lastIndex = m.index;
|
|
840
|
+
if (this.digit.test(text) && this.capital.test(text)) {
|
|
841
|
+
yield m;
|
|
842
|
+
pos = end(m);
|
|
843
|
+
} else {
|
|
844
|
+
this.run.exec(text);
|
|
845
|
+
pos = this.run.lastIndex;
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
},
|
|
849
|
+
};
|
|
850
|
+
|
|
851
|
+
const _ALNUM_PIECE_RE = re.compile(`[${LET}]+|${D}+(?:[.,]${D}+)?`);
|
|
852
|
+
|
|
853
|
+
function _read_alnum(tok) {
|
|
854
|
+
const out = [];
|
|
855
|
+
for (const piece of _ALNUM_PIECE_RE.findall(tok)) {
|
|
856
|
+
if (!isdigit(piece) && isdigit(first_cp(piece))) out.push(read_number(piece)); // GPT-3.5
|
|
857
|
+
else if (isdigit(piece)) out.push(plen(piece) >= 5 ? digits_to_words(piece) : read_digit_group(piece));
|
|
858
|
+
else if (L.ACRONYMS.has(piece)) out.push(L.ACRONYMS.get(piece));
|
|
859
|
+
else if (is_all_caps(piece) || plen(piece) === 1) out.push(_pronounceable(piece) ? tr_lower(piece) : _letters(piece));
|
|
860
|
+
else out.push(piece);
|
|
861
|
+
}
|
|
862
|
+
return out.join(' ');
|
|
863
|
+
}
|
|
864
|
+
|
|
865
|
+
function _pass_alnum(text) {
|
|
866
|
+
// plain words with a trailing number like "Ankara2023" are rare; require caps letters
|
|
867
|
+
const repl = (m) => _suffix(_read_alnum(m[1]), m[2]);
|
|
868
|
+
|
|
869
|
+
return _sub(_ALNUM_RE, repl, text);
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
// either end may be negative: "-10 - 5 derece", "-5 - -2 gün" (a sign only after a space or the start)
|
|
873
|
+
const _RANGE_RE = re.compile(
|
|
874
|
+
`(?=[\\-${D}])(?:${nlb(`${WC}.,\\-`)}(-)|${NB})(${NUM})${S}?-${S}?(-?)(${NUM})${SUF}(?!${W}|[.,]${D})`,
|
|
875
|
+
(t) => t.includes('-') && has_digit(t),
|
|
876
|
+
);
|
|
877
|
+
|
|
878
|
+
const _SCORE_AFTER_RE = re.compile(`${S}*(?:${L.SCORE_CONTEXT.join('|')})`);
|
|
879
|
+
|
|
880
|
+
function _pass_ranges(text) {
|
|
881
|
+
const repl = (m) => {
|
|
882
|
+
const [, sa, a, sb, b, suf] = m;
|
|
883
|
+
const before = _context_before(text, m.index, 40);
|
|
884
|
+
const after = tr_lower(text.slice(end(m), cp_fwd(text, end(m), 30)));
|
|
885
|
+
let av = _int_value(a.split(',')[0]);
|
|
886
|
+
let bv = _int_value(b.split(',')[0]);
|
|
887
|
+
if (sa && av !== null) av = -av;
|
|
888
|
+
if (sb && bv !== null) bv = -bv;
|
|
889
|
+
const first = read_number((sa || '') + a);
|
|
890
|
+
|
|
891
|
+
const last = (v) => {
|
|
892
|
+
if (v !== null && suf && !sb) return _attach_number_suffix(v, read_number(b), suf);
|
|
893
|
+
return _suffix(read_number(sb + b), suf);
|
|
894
|
+
};
|
|
895
|
+
|
|
896
|
+
const descending = av !== null && bv !== null && bv <= av; // ranges go up; "3-1", "6-6", "2023-24" don't
|
|
897
|
+
if (descending || _has_word(before, L.SCORE_CONTEXT) || _SCORE_AFTER_RE.match(after)) {
|
|
898
|
+
return `${first} ${last(bv)}`;
|
|
899
|
+
}
|
|
900
|
+
return `${first} ${_ila(text, end(m))} ${last(_int_value(b))}`;
|
|
901
|
+
};
|
|
902
|
+
|
|
903
|
+
return _sub(_RANGE_RE, repl, text);
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
// N/M is a building / apartment number: after "No", "numara", "kapı", "daire"... or before "nolu".
|
|
907
|
+
const _LET_WORD_RE = re.compile(`[${LET}]+`);
|
|
908
|
+
const _NOLU_RE = re.compile(`${S}*(?:nolu|no'lu|numaralı)${B}`);
|
|
909
|
+
function _address_number(text, start, end_) {
|
|
910
|
+
const words = _LET_WORD_RE.findall(_context_before(text, start, 24));
|
|
911
|
+
if (words.length && L.ADDRESS_NUMBER_CONTEXT.has(words[words.length - 1])) return true;
|
|
912
|
+
return _NOLU_RE.match(tr_lower(text.slice(end_, cp_fwd(text, end_, 16)))) !== null;
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
const _has_slash = (t) => t.includes('/');
|
|
916
|
+
const _FRACTION_RE = re.compile(
|
|
917
|
+
`(?=${D})${nlb(`${WC}./,`)}(${D}{1,3})/(${D}{1,3})${SUF}(?![${WC}/]|[.,]${D})`, _has_slash,
|
|
918
|
+
);
|
|
919
|
+
const _RATIO_RE = re.compile(`${NBD}(${D}+,${D}+)/(${D}{1,3})${SUF}(?![${WC}/]|[.,]${D})`, _has_slash);
|
|
920
|
+
|
|
921
|
+
function _pass_fractions(text) {
|
|
922
|
+
const repl = (m) => {
|
|
923
|
+
const a = int(m[1]);
|
|
924
|
+
const b = int(m[2]);
|
|
925
|
+
const suf = m[3];
|
|
926
|
+
if (a === 24 && b === 7) return _suffix('yirmi dört yedi', suf);
|
|
927
|
+
if (_address_number(text, m.index, end(m))) { // "No: 12/4", "Daire 3/1", "10/2 nolu": never a date
|
|
928
|
+
return _suffix(`${number_to_words(a)} bölü ${number_to_words(b)}`, suf);
|
|
929
|
+
}
|
|
930
|
+
if (b === 0) return m[0];
|
|
931
|
+
if (a >= b && a >= 1 && a <= 31 && b >= 1 && b <= 12) return _suffix(_date_words(a, b, null), suf);
|
|
932
|
+
if (a < b) return _suffix(`${attach_suffix(number_to_words(b), 'de')} ${number_to_words(a)}`, suf);
|
|
933
|
+
return _suffix(`${number_to_words(a)} bölü ${number_to_words(b)}`, suf);
|
|
934
|
+
};
|
|
935
|
+
|
|
936
|
+
const ratio = (m) => _suffix(`${read_number(m[1])} bölü ${number_to_words(py_int(m[2]))}`, m[3]);
|
|
937
|
+
|
|
938
|
+
text = _sub(_RATIO_RE, ratio, text);
|
|
939
|
+
return _sub(_FRACTION_RE, repl, text);
|
|
940
|
+
}
|
|
941
|
+
|
|
942
|
+
const _ROMAN = 'M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})';
|
|
943
|
+
const _ROMAN_RE = re.compile(`(?=[MDCLXVI.])${nlb(`${WC}.`)}(${_ROMAN})\\.(?=${S}+([${LET}][${LET}']*))`, has('.'));
|
|
944
|
+
const _ROMAN_VALUES = { I: 1, V: 5, X: 10, L: 50, C: 100, D: 500, M: 1000 };
|
|
945
|
+
|
|
946
|
+
function _roman_to_int(s) {
|
|
947
|
+
let total = 0;
|
|
948
|
+
for (let i = 0; i < s.length; i++) {
|
|
949
|
+
const v = _ROMAN_VALUES[s[i]];
|
|
950
|
+
if (i + 1 < s.length && _ROMAN_VALUES[s[i + 1]] > v) total -= v;
|
|
951
|
+
else total += v;
|
|
952
|
+
}
|
|
953
|
+
return total;
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
const _ROMAN_SUFFIX_RE = re.compile(`(?=[IVXLC])${nlb(`${WC}.`)}([IVXLC]{2,}|[IVX])'([${LOW}]+)(?!${W})`, has("'"));
|
|
957
|
+
|
|
958
|
+
// A capitalized word after which "N." / "IV." is an ordinal: "Lig", "Murat", "Yüzyılda" (also "LİG").
|
|
959
|
+
function _ordinal_noun(word) {
|
|
960
|
+
if (is_all_caps(word)) word = tr_capitalize(tr_lower(word));
|
|
961
|
+
return L.ORDINAL_CAP_NOUNS.has(word.split("'")[0]) || L.ORDINAL_CAP_PREFIXES.some((p) => word.startsWith(p));
|
|
962
|
+
}
|
|
963
|
+
|
|
964
|
+
const _WORD_BEFORE_RE = re.compile(`([${LET}]+)[ \\t]+${END}`);
|
|
965
|
+
// a title ("Dr.", "doktor", "Sayın") or another initial ("J. I. Smith") right before a lone I. / X.
|
|
966
|
+
// (?i:...) is spelled out by ci()
|
|
967
|
+
const _TITLE_BEFORE_RE = re.compile(
|
|
968
|
+
`(?:${nlb(WC)}(?:${L.INITIAL_TITLES.map(ci).join('|')})\\.?|${nlb(`${WC}.`)}[${UP}]\\.)[ \\t]+${END}`,
|
|
969
|
+
);
|
|
970
|
+
|
|
971
|
+
function _pass_roman(text) {
|
|
972
|
+
const suffixed = (m) => {
|
|
973
|
+
const [, tok, suf] = m;
|
|
974
|
+
if (!re.fullmatch(_ROMAN, tok)) return m[0];
|
|
975
|
+
if (plen(tok) === 1) {
|
|
976
|
+
// the written suffix tells which reading the writer had in mind: "V'te" (beşte) and "Ek I'de"
|
|
977
|
+
// (birde) are numerals, "X'te" (ikste) and "V'ye" (veye) letters; when it fits both or neither,
|
|
978
|
+
// a regnal name or "Madde" before it makes it the numeral ("Louis X'un", "Madde V'e")
|
|
979
|
+
const s = tr_lower(suf);
|
|
980
|
+
const n = _roman_to_int(tok);
|
|
981
|
+
const as_number = harmonize_suffix(number_to_words(n), s) === s;
|
|
982
|
+
const as_letter = harmonize_suffix(L.LETTER_NAMES.get(tok), s) === s;
|
|
983
|
+
if (as_letter && !as_number) return m[0];
|
|
984
|
+
if (as_letter === as_number) {
|
|
985
|
+
const prev = _WORD_BEFORE_RE.search(text, cp_back(text, m.index, 24), m.index);
|
|
986
|
+
if (!(prev && _ordinal_noun(prev[1]))) return m[0];
|
|
987
|
+
}
|
|
988
|
+
}
|
|
989
|
+
const n = _roman_to_int(tok);
|
|
990
|
+
return _attach_number_suffix(n, number_to_words(n), suf);
|
|
991
|
+
};
|
|
992
|
+
|
|
993
|
+
text = _sub(_ROMAN_SUFFIX_RE, suffixed, text);
|
|
994
|
+
|
|
995
|
+
const repl = (m) => {
|
|
996
|
+
const tok = m[1];
|
|
997
|
+
const nxt = m[2];
|
|
998
|
+
if (!tok || (plen(tok) === 1 && !'IVX'.includes(tok))) return m[0];
|
|
999
|
+
// before a capitalized word that is not a known noun, a lone V. is an initial ("V. Öztürk", but
|
|
1000
|
+
// "V. Murat"); I. and X. are ordinals ("I. Balkan Savaşı", "I. TBMM") unless a title or another
|
|
1001
|
+
// initial comes before them ("Dr. I. Yılmaz")
|
|
1002
|
+
if (plen(tok) === 1 && !(islower(first_cp(nxt)) || _ordinal_noun(nxt))) {
|
|
1003
|
+
if (tok === 'V' || _TITLE_BEFORE_RE.search(text, cp_back(text, m.index, 24), m.index)) return m[0];
|
|
1004
|
+
}
|
|
1005
|
+
return ordinal_to_words(_roman_to_int(tok));
|
|
1006
|
+
};
|
|
1007
|
+
|
|
1008
|
+
return _sub(_ROMAN_RE, repl, text);
|
|
1009
|
+
}
|
|
1010
|
+
|
|
1011
|
+
const _ORDINAL_DOT_RE = re.compile(`${NBD}(${D}{1,4})\\.(?=${S}+([${LET}][${LET}']*))`, has_digit);
|
|
1012
|
+
|
|
1013
|
+
function _pass_ordinals(text) {
|
|
1014
|
+
const repl = (m) => {
|
|
1015
|
+
const n = py_int(m[1]);
|
|
1016
|
+
const nxt = m[2];
|
|
1017
|
+
if (islower(first_cp(nxt)) || _ordinal_noun(nxt)) return ordinal_to_words(n);
|
|
1018
|
+
return m[0];
|
|
1019
|
+
};
|
|
1020
|
+
|
|
1021
|
+
return _sub(_ORDINAL_DOT_RE, repl, text);
|
|
1022
|
+
}
|
|
1023
|
+
|
|
1024
|
+
const _NUMBER_RE = re.compile(
|
|
1025
|
+
`${NBS}(-)?((?:${INT_TH}|${D}+)(?:,${D}+|\\.${D}+)?)(?:'([${LET}]+)|([${LOW}]+))?(?!${W})`, has_digit,
|
|
1026
|
+
);
|
|
1027
|
+
const _LIST_RE = re.compile(`${NBD}${D}+(?:,${D}+){2,}(?![${WC}.,])`, (t) => t.includes(',') && has_digit(t));
|
|
1028
|
+
|
|
1029
|
+
// dotted versions and addresses: 1.2.3, 10.15.7, 192.168.1.1 (thousands groups like 1.250.000 are numbers)
|
|
1030
|
+
const _VERSION_RE = re.compile(`${NBD}${D}+(?:\\.${D}+){2,}(?!${W}|[.,]${D})`, (t) => t.includes('.') && has_digit(t));
|
|
1031
|
+
// a version wildcard: 3.x, 2.7.x
|
|
1032
|
+
const _VERSION_X_RE = re.compile(`${NBD}(${D}+(?:\\.${D}+)*)\\.[xX](?!${W})`, has('xX'));
|
|
1033
|
+
// a word, a hyphen, a number: "klod-3.5", "covid-19" (capitalized ones are read by _pass_alnum)
|
|
1034
|
+
const _WORD_HYPHEN_NUM_RE = re.compile(`(?<=[${LET}]{2})-(?=${D})`, (t) => t.includes('-'));
|
|
1035
|
+
|
|
1036
|
+
function _dotted(s) {
|
|
1037
|
+
return s.split('.').map((p) => (plen(p) <= 3 ? read_digit_group(p) : digits_to_words(p))).join(' nokta ');
|
|
1038
|
+
}
|
|
1039
|
+
|
|
1040
|
+
const _NUMBER_WORDS = new Set(split(
|
|
1041
|
+
'sıfır bir iki üç dört beş altı yedi sekiz dokuz on yirmi otuz kırk elli altmış yetmiş seksen doksan ' +
|
|
1042
|
+
'yüz bin milyon milyar trilyon katrilyon',
|
|
1043
|
+
));
|
|
1044
|
+
|
|
1045
|
+
// re.search(rf"([{LET}]+)\.?$", head).group(1), or '' when it does not match. Searching from the start, as Python does,
|
|
1046
|
+
// costs the whole text before every postcode candidate (O(n²) on an address list); the match can only be the letters
|
|
1047
|
+
// that end head, before an optional "." and the final newline $ allows, so they are read from the end.
|
|
1048
|
+
function _letters_at_end(head) {
|
|
1049
|
+
let r = head.length;
|
|
1050
|
+
if (head.endsWith('\n')) r--;
|
|
1051
|
+
if (head[r - 1] === '.') r--;
|
|
1052
|
+
let a = r;
|
|
1053
|
+
while (a > 0 && _LET_CHAR.test(head[a - 1])) a--;
|
|
1054
|
+
return head.slice(a, r);
|
|
1055
|
+
}
|
|
1056
|
+
const _LET_CHAR = new RegExp(`[${LET}]`); // LET is all BMP: one UTF-16 unit each
|
|
1057
|
+
|
|
1058
|
+
// A bare 5-digit Turkish postcode inside an address, before the place name: "..., 34728 Kadıköy".
|
|
1059
|
+
//
|
|
1060
|
+
// The address is named by a whole address word (ADDRESS_CONTEXT: "Mah.", "sokak", "No"; not "katılan",
|
|
1061
|
+
// "mahallede"), and the number comes right after a comma, slash, colon, address word or door number
|
|
1062
|
+
// ("No:5 D:3 34728 Kadıköy"), never after an ordinary word ("Mahallede 15000 Suriyeli yaşıyor").
|
|
1063
|
+
// In ALL CAPS only a door number counts: "İSTİKLAL CADDESİ 50000 KİŞİYLE DOLDU" is a count.
|
|
1064
|
+
const _NEXT_NAME_RE = re.compile(`[ \\t]+([${UP}][${LET}]*)`);
|
|
1065
|
+
const _ADDRESS_WORD_RE = re.compile(`[${LOW}]+(?:'[${LOW}]+)?`);
|
|
1066
|
+
function _postcode(text, m, num) {
|
|
1067
|
+
const nxt = _NEXT_NAME_RE.match(text, end(m)); // re.match(..., text[m.end():])
|
|
1068
|
+
const d = [...num];
|
|
1069
|
+
// the first two characters against ASCII bounds: UTF-16 order agrees with code point order there
|
|
1070
|
+
const first2 = d.slice(0, 2).join('');
|
|
1071
|
+
if (d.length !== 5 || !(first2 >= '01' && first2 <= '81') || !nxt) return false;
|
|
1072
|
+
const words = _ADDRESS_WORD_RE.findall(_context_before(text, m.index, 100));
|
|
1073
|
+
if (!words.some((w) => L.ADDRESS_CONTEXT.has(w))) return false;
|
|
1074
|
+
const head = rstrip(text.slice(0, m.index), ' \t');
|
|
1075
|
+
const prev = tr_lower(_letters_at_end(head));
|
|
1076
|
+
const door = isdecimal(last_cp(head)) || _NUMBER_WORDS.has(prev);
|
|
1077
|
+
if (is_all_caps(nxt[1])) return door;
|
|
1078
|
+
return door || !head || ',/:;\n'.includes(last_cp(head)) || L.ADDRESS_CONTEXT.has(prev);
|
|
1079
|
+
}
|
|
1080
|
+
|
|
1081
|
+
const _ID_WORD_AFTER_RE = re.compile(`${S}*(?:numara|no${B}|nolu|kodu|kodlu)`);
|
|
1082
|
+
const _SCALE_AFTER_RE = re.compile(`${S}+(?:bin|milyon|milyar|trilyon)${B}`);
|
|
1083
|
+
const _UNIT_WORD_AFTER_RE = re.compile(`${S}+(?:${_UNIT_WORDS_RE})`);
|
|
1084
|
+
|
|
1085
|
+
function _pass_numbers(text) {
|
|
1086
|
+
text = _sub(_LIST_RE, (m) => m[0].split(',').map((x) => number_to_words(py_int(x))).join(', '), text);
|
|
1087
|
+
text = _sub(_VERSION_X_RE, (m) => `${_dotted(m[1])} nokta iks`, text);
|
|
1088
|
+
text = _sub(_VERSION_RE, (m) => (re.fullmatch(INT_TH, m[0]) ? m[0] : _dotted(m[0])), text);
|
|
1089
|
+
text = _WORD_HYPHEN_NUM_RE.sub(' ', text);
|
|
1090
|
+
|
|
1091
|
+
const spoken = (m, sign, num, suf) => {
|
|
1092
|
+
const neg = Boolean(sign) && (m.index === 0 || !isalnum(char_before(text, m.index)));
|
|
1093
|
+
const prefix = neg ? 'eksi ' : sign ? '-' : '';
|
|
1094
|
+
const after = tr_lower(text.slice(end(m), cp_fwd(text, end(m), 24)));
|
|
1095
|
+
if (isdigit(num)) {
|
|
1096
|
+
const ctx = _context_before(text, m.index);
|
|
1097
|
+
const count = _quantity(ctx, after);
|
|
1098
|
+
const len = plen(num);
|
|
1099
|
+
const idlike = len >= 5 && (
|
|
1100
|
+
num.startsWith('0')
|
|
1101
|
+
|| len >= 10
|
|
1102
|
+
|| (!count && _has_cue(ctx, L.ID_CONTEXT, 3))
|
|
1103
|
+
|| _ID_WORD_AFTER_RE.match(after) !== null
|
|
1104
|
+
|| _postcode(text, m, num)
|
|
1105
|
+
);
|
|
1106
|
+
const codelike = len >= 4 && !count && _has_cue(ctx, L.CODE_CONTEXT, 2);
|
|
1107
|
+
if (idlike || codelike) {
|
|
1108
|
+
const words = len <= 8 ? digits_to_words(num) : read_long_digits(num);
|
|
1109
|
+
return prefix + _suffix(words, suf);
|
|
1110
|
+
}
|
|
1111
|
+
if (num.startsWith('0') && len > 1) return prefix + _suffix(read_digit_group(num), suf);
|
|
1112
|
+
const n = py_int(num);
|
|
1113
|
+
return prefix + _attach_number_suffix(n, number_to_words(n), suf);
|
|
1114
|
+
}
|
|
1115
|
+
const ctx_before = _context_before(text, m.index, 24);
|
|
1116
|
+
const dot_word = (
|
|
1117
|
+
_SCALE_AFTER_RE.match(after)
|
|
1118
|
+
|| _UNIT_WORD_AFTER_RE.match(after)
|
|
1119
|
+
|| _has_word(ctx_before, ['yüzde', 'oran', 'faiz', 'puan', 'not', 'ortalama'])
|
|
1120
|
+
) ? 'virgül' : 'nokta';
|
|
1121
|
+
const words = read_number(num, dot_word);
|
|
1122
|
+
const val = !num.includes(',') && re.fullmatch(INT_TH, num) ? _int_value(num) : null;
|
|
1123
|
+
if (val !== null) return prefix + _attach_number_suffix(val, words, suf);
|
|
1124
|
+
return prefix + _suffix(words, suf);
|
|
1125
|
+
};
|
|
1126
|
+
|
|
1127
|
+
const repl = (m) => {
|
|
1128
|
+
const [, sign, num, suf, bare] = m;
|
|
1129
|
+
if (bare !== undefined && !L.BARE_NUMBER_SUFFIXES.has(tr_lower(bare))) {
|
|
1130
|
+
// not a suffix ("4o", "2x", "1975yılında"): the number, then the letters as a word of their own
|
|
1131
|
+
return `${spoken(m, sign, num, null)} ${bare === 'x' ? 'iks' : bare}`;
|
|
1132
|
+
}
|
|
1133
|
+
return spoken(m, sign, num, bare === undefined ? suf : bare);
|
|
1134
|
+
};
|
|
1135
|
+
|
|
1136
|
+
return _sub(_NUMBER_RE, repl, text);
|
|
1137
|
+
}
|
|
1138
|
+
|
|
1139
|
+
// Safety net for "no digit is left": a digit run no pass recognized is read plainly ("on iki.31" never reaches TTS).
|
|
1140
|
+
//
|
|
1141
|
+
// A period after a number word is a decimal point ("on iki.31" -> on iki nokta otuz bir); after any other word
|
|
1142
|
+
// it ends an abbreviation ("Tablo.3'te" -> Tablo üçte, "md.5" -> md beş). A suffix stays on its number.
|
|
1143
|
+
function _pass_leftover_digits(text) {
|
|
1144
|
+
text = _WORD_DOT_DIGIT_RE.sub((m) => m[1] + (_NUMBER_WORDS.has(tr_lower(m[1])) ? ' nokta ' : ' '), text);
|
|
1145
|
+
|
|
1146
|
+
const read = (m) => {
|
|
1147
|
+
const [, digits, suf] = m;
|
|
1148
|
+
if (plen(digits) > 21) return ` ${_suffix(digits_to_words(digits), suf)} `;
|
|
1149
|
+
const words = read_digit_group(digits);
|
|
1150
|
+
if (digits.startsWith('0')) return ` ${_suffix(words, suf)} `; // "05" keeps its zero: no ordinal
|
|
1151
|
+
return ` ${_attach_number_suffix(py_int(digits), words, suf)} `;
|
|
1152
|
+
};
|
|
1153
|
+
|
|
1154
|
+
return _DIGITS_RE.sub(read, text);
|
|
1155
|
+
}
|
|
1156
|
+
// (?<![LET]): a match takes the whole run of letters before the "."; Python, failing at the start of a long run of
|
|
1157
|
+
// letters without one, tries again from every letter of it (O(n²)), and every one of those fails the same way
|
|
1158
|
+
const _WORD_DOT_DIGIT_RE = re.compile(`(?<![${LET}])([${LET}]+)\\.(?=${D})`, (t) => t.includes('.') && has_digit(t));
|
|
1159
|
+
const _DIGITS_RE = re.compile(`(${D}+)(?:'([${LET}]+))?`, has_digit);
|
|
1160
|
+
|
|
1161
|
+
const _POWER_RE = re.compile(`(?:${nlb(WC)}([A-Za-z]{1,2}|${D}+(?:[.,]${D}+)?)|(\\)))([²³])(?!${W})`, has('²³'));
|
|
1162
|
+
|
|
1163
|
+
// x² -> iks kare, a³ -> a küp, 10² -> 10 kare, (a+b)² -> (a+b) kare, m² (no number before it) -> metrekare.
|
|
1164
|
+
function _pass_powers(text) {
|
|
1165
|
+
const repl = (m) => {
|
|
1166
|
+
const [, b, paren, sup] = m;
|
|
1167
|
+
let base = b || paren;
|
|
1168
|
+
if (L.UNITS.has(base + sup)) return L.UNITS.get(base + sup);
|
|
1169
|
+
const power = sup === '²' ? 'kare' : 'küp';
|
|
1170
|
+
if (isalpha(base)) {
|
|
1171
|
+
if (plen(base) > 1) return m[0];
|
|
1172
|
+
base = L.LETTER_NAMES.get(tr_upper(base)) ?? base;
|
|
1173
|
+
}
|
|
1174
|
+
return `${base} ${power}`;
|
|
1175
|
+
};
|
|
1176
|
+
|
|
1177
|
+
return _sub(_POWER_RE, repl, text);
|
|
1178
|
+
}
|
|
1179
|
+
|
|
1180
|
+
const _ACRONYM_RE = re.compile(`(?=[${UP}])${nlb(`${WC}'`)}([${UP}][${UP}]+|PhD)(?:'([${LET}]+))?(?!${W})`);
|
|
1181
|
+
const _SINGLE_LETTER_RE = re.compile(`(?=[A-NPR-ZÇĞİÖŞÜ])${nlb(`${WC}'.`)}([A-NPR-ZÇĞİÖŞÜ])(?=${S}+[${LOW}])`);
|
|
1182
|
+
// a capital letter with a suffix: "X'te" -> ikste, "A'dan Z'ye" -> adan zeye (not the pronoun "O'nun"); the
|
|
1183
|
+
// suffix is lowercase or ALL CAPS ("X'TE"), so a name with an elided article stays: "D'Artagnan", "L'Occitane"
|
|
1184
|
+
const _SINGLE_LETTER_SUFFIX_RE = re.compile(
|
|
1185
|
+
`(?=[A-NPR-ZÇĞİÖŞÜ])${nlb(`${WC}'.`)}([A-NPR-ZÇĞİÖŞÜ])'([${LOW}]+|[${UP}]+)(?![${WC}'])`, has("'"),
|
|
1186
|
+
);
|
|
1187
|
+
// an initial before a name: "V. Öztürk" -> "ve. Öztürk", "J. K. Rowling"; the period stays (it may end a sentence)
|
|
1188
|
+
const _INITIAL_RE = re.compile(`(?=[${UP}])${nlb(`${WC}'.`)}([${UP}])\\.(?=${S}+[${UP}])`, has('.'));
|
|
1189
|
+
const _SENTENCE_SPLIT_RE = re.compile(`(?<=[.!?])${S}+`, null, false);
|
|
1190
|
+
|
|
1191
|
+
// lowercase words the number passes wrote ("on dört otuzda", "elli bin", "yüzde", "lira"): in "SAAT 14:30'DA
|
|
1192
|
+
// BAŞLAYACAK DÖRT ETKİNLİK" they do not make the headline a mixed-case sentence
|
|
1193
|
+
const _EXPANSION_STEMS = [...new Set([..._NUMBER_WORDS, 'dörd', 'nokta', 'virgül', 'buçuk', 'yüzde', 'binde', 'eksi',
|
|
1194
|
+
'artı', 'çarpı', 'bölü', 'ila', 'lira', 'kuruş', 'dolar', 'avro', 'sent', 'sterlin', 'derece'])].sort();
|
|
1195
|
+
|
|
1196
|
+
// A word of [LET] letters that is_all_caps: two or more, all capitals. A sentence needs three for a headline.
|
|
1197
|
+
const _CAPS_WORD_RE = new RegExp(`(?<![${LET}])[${UP}]{2,}(?![${LET}])`, 'g');
|
|
1198
|
+
const _three_caps_words = (sent) => {
|
|
1199
|
+
_CAPS_WORD_RE.lastIndex = 0;
|
|
1200
|
+
return _CAPS_WORD_RE.test(sent) && _CAPS_WORD_RE.test(sent) && _CAPS_WORD_RE.test(sent);
|
|
1201
|
+
};
|
|
1202
|
+
|
|
1203
|
+
// Lowercase ALL-CAPS sentences, keeping known acronyms for the acronym pass.
|
|
1204
|
+
function _lower_caps_headlines(text) {
|
|
1205
|
+
const out = [];
|
|
1206
|
+
for (let sent of _SENTENCE_SPLIT_RE.split(text)) {
|
|
1207
|
+
// caps_words below is the [LET] words that are two or more capitals: fewer than three, and it is no headline
|
|
1208
|
+
if (!_three_caps_words(sent)) {
|
|
1209
|
+
out.push(sent);
|
|
1210
|
+
continue;
|
|
1211
|
+
}
|
|
1212
|
+
const words = _LET_WORD_RE.findall(sent)
|
|
1213
|
+
.filter((w) => !(islower(w) && _EXPANSION_STEMS.some((p) => w.startsWith(p))));
|
|
1214
|
+
const letters = words.flatMap((w) => [...w]);
|
|
1215
|
+
const caps_words = words.filter((w) => is_all_caps(w));
|
|
1216
|
+
const upper = letters.filter((c) => isupper(c)).length;
|
|
1217
|
+
if (letters.length >= 12 && upper / letters.length > 0.8 && caps_words.length >= 3) {
|
|
1218
|
+
sent = _LET_WORD_RE.sub(
|
|
1219
|
+
(m) => (L.ACRONYMS.has(m[0]) || !_pronounceable_or_word(m[0]) ? m[0] : tr_lower(m[0])), sent,
|
|
1220
|
+
);
|
|
1221
|
+
}
|
|
1222
|
+
out.push(sent);
|
|
1223
|
+
}
|
|
1224
|
+
return out.join(' ');
|
|
1225
|
+
}
|
|
1226
|
+
|
|
1227
|
+
// consonant clusters a Turkish word (or a loanword) can start or end with: "KRİZ", "SPOR", "TREN", "TÜRK",
|
|
1228
|
+
// "DÖRT", "GENÇ", "HALK", "FİLM", "ÜST"
|
|
1229
|
+
const _ONSETS = new Set(['bl', 'br', 'dr', 'fl', 'fr', 'gl', 'gr', 'kl', 'kr', 'pl', 'pr', 'sf', 'sk', 'sl', 'sm', 'sn',
|
|
1230
|
+
'sp', 'st', 'şp', 'şt', 'tr', 'skr', 'spr', 'str']);
|
|
1231
|
+
const _CODAS = new Set(['rk', 'rt', 'rç', 'rs', 'rş', 'rz', 'rp', 'rf', 'rm', 'rn', 'rd', 'rb', 'rg', 'nk', 'nç', 'nt',
|
|
1232
|
+
'ns', 'nd', 'ng', 'nz', 'lk', 'lt', 'lç', 'lp', 'lm', 'lf', 'ls', 'şk', 'şt', 'st', 'sk', 'kt', 'ks', 'ft', 'pt',
|
|
1233
|
+
'ps', 'mp']);
|
|
1234
|
+
const _CONSONANT_RUN_RE = re.compile('[^aeıioöuüâîû]+');
|
|
1235
|
+
|
|
1236
|
+
// Spelled like a Turkish word or loanword: a vowel, a known cluster at each end, at most four consonants in a
|
|
1237
|
+
// row inside. "KONTROL", "TÜRK", "KRİZ" are; "AVM" (vm), "İETT" (tt), "DHA" (dh), "AİHM" (hm), "TBMM" are not.
|
|
1238
|
+
function _word_shaped(w) {
|
|
1239
|
+
const low = tr_lower(w);
|
|
1240
|
+
if (![...low].some((c) => VOWELS.has(c))) return false;
|
|
1241
|
+
const lead = re.match('[^aeıioöuüâîû]*', low)[0];
|
|
1242
|
+
const trail = re.search(`[^aeıioöuüâîû]*${END}`, low)[0];
|
|
1243
|
+
if ((plen(lead) > 1 && !_ONSETS.has(lead)) || (plen(trail) > 1 && !_CODAS.has(trail))) return false;
|
|
1244
|
+
return _CONSONANT_RUN_RE.findall(strip(low, 'bcçdfgğhjklmnprsştvyzqwx')).every((r) => plen(r) <= 4);
|
|
1245
|
+
}
|
|
1246
|
+
|
|
1247
|
+
// In an ALL-CAPS headline: read as a word ("DÖRT KİŞİ", "KRİZ", "KONTROL") unless it is an acronym.
|
|
1248
|
+
function _pronounceable_or_word(w) {
|
|
1249
|
+
if (L.CAPS_WORDS.has(w) || _pronounceable(w) || plen(w) === 1) return true;
|
|
1250
|
+
return !L.EVAL_ACRONYMS.has(w) && _word_shaped(w);
|
|
1251
|
+
}
|
|
1252
|
+
|
|
1253
|
+
function _pass_acronyms(text) {
|
|
1254
|
+
const repl = (m) => {
|
|
1255
|
+
const [, tok, suf] = m;
|
|
1256
|
+
let spoken;
|
|
1257
|
+
if (L.ACRONYMS.has(tok)) {
|
|
1258
|
+
spoken = L.ACRONYMS.get(tok);
|
|
1259
|
+
} else if (L.CAPS_WORDS.has(tok)) {
|
|
1260
|
+
return tr_lower(tok) + (suf ? `'${tr_lower(suf)}` : '');
|
|
1261
|
+
} else if (_pronounceable(tok)) {
|
|
1262
|
+
return tr_lower(tok) + (suf ? `'${tr_lower(suf)}` : '');
|
|
1263
|
+
} else {
|
|
1264
|
+
spoken = _letters(tok);
|
|
1265
|
+
}
|
|
1266
|
+
if (!suf) return spoken;
|
|
1267
|
+
if (!spoken.includes(' ')) return `${spoken}'${tr_lower(suf)}`; // read as a word: keep the apostrophe
|
|
1268
|
+
return _suffix(spoken, suf);
|
|
1269
|
+
};
|
|
1270
|
+
|
|
1271
|
+
text = _sub(_ACRONYM_RE, repl, text);
|
|
1272
|
+
// lone capital letters used as labels: "C vitamini", "B planı", "A blok" (not the pronoun "O")
|
|
1273
|
+
return _sub(_SINGLE_LETTER_RE, (m) => L.LETTER_NAMES.get(m[1]) ?? m[1], text);
|
|
1274
|
+
}
|
|
1275
|
+
|
|
1276
|
+
// A capital letter with a suffix or as an initial is read by its name: "X'te" -> ikste, "V. Öztürk" -> ve. Öztürk.
|
|
1277
|
+
function _pass_letter_names(text) {
|
|
1278
|
+
text = _sub(_SINGLE_LETTER_SUFFIX_RE, (m) => _suffix(L.LETTER_NAMES.get(m[1]), m[2]), text);
|
|
1279
|
+
return _INITIAL_RE.sub((m) => `${L.LETTER_NAMES.get(m[1]) ?? tr_lower(m[1])}.`, text);
|
|
1280
|
+
}
|
|
1281
|
+
|
|
1282
|
+
// _pass_symbols' patterns, each with the one character it cannot match without
|
|
1283
|
+
const _SYMBOLS = [
|
|
1284
|
+
[re.compile(`(?=[${SC}&])(?:${S}&${S}|${lb(WC)}&(?=${W}))`, has('&')), ' ve '],
|
|
1285
|
+
[re.compile(`(?=[${SC}x×])${lb(D)}${S}?[x×]${S}?(?=${D})`, has('x×')), ' çarpı '],
|
|
1286
|
+
[re.compile(`${S}?=${S}?`, has('=')), ' eşittir '],
|
|
1287
|
+
[re.compile(`(?=[${SC}+])(?:${lb(WC)}${S}\\+${S}(?=${W})|${lb(D)}\\+(?=${D}))`, has('+')), ' artı '],
|
|
1288
|
+
[re.compile('±', has('±')), ' artı eksi '],
|
|
1289
|
+
[re.compile(`[~≈]${S}?(?=${W})`, has('~≈')), 'yaklaşık '],
|
|
1290
|
+
[re.compile(`(?=#)${nlb(WC)}#(?=${W})`, has('#')), ''],
|
|
1291
|
+
[re.compile(`(?=@)${nlb(WC)}@(?=${W})`, has('@')), ''],
|
|
1292
|
+
[re.compile(`§${S}?`, has('§')), 'paragraf '],
|
|
1293
|
+
[re.compile(`№${S}?`, has('№')), 'numara '],
|
|
1294
|
+
[re.compile('[*_|•→←▪►✓✔]', has('*_|•→←▪►✓✔')), ' '],
|
|
1295
|
+
];
|
|
1296
|
+
// Kadıköy/İstanbul -> Kadıköy, İstanbul
|
|
1297
|
+
const _CITY_SLASH_RE = re.compile(`(?=[${UP}])${B}([${UP}][${LOW}]+)/(?=[${UP}][${LOW}])`, _has_slash);
|
|
1298
|
+
const _WORD_SLASH_RE = re.compile(`(?=/)${lb(WC)}/(?=${W})`, _has_slash);
|
|
1299
|
+
const _REPEATED_MARK_RE = re.compile(String.raw`([!?])\1+`, has('!?'));
|
|
1300
|
+
|
|
1301
|
+
function _pass_symbols(text) {
|
|
1302
|
+
for (const [rx, word] of _SYMBOLS) text = rx.sub(word, text);
|
|
1303
|
+
if (U_PAIR_BUG && U_PAIR_RE.test(text)) throw _engine_error(text); // a "#" or "@" between them is gone
|
|
1304
|
+
text = _CITY_SLASH_RE.sub((m) => `${m[1]}, `, text);
|
|
1305
|
+
text = _WORD_SLASH_RE.sub(' ', text);
|
|
1306
|
+
text = _REPEATED_MARK_RE.sub((m) => m[1], text);
|
|
1307
|
+
return text;
|
|
1308
|
+
}
|
|
1309
|
+
|
|
1310
|
+
// (?<!\s): the match takes a whole run of spaces; Python retries a run that no punctuation ends from each of its
|
|
1311
|
+
// characters (O(n²) on a long run of newlines), and all of them fail. All BMP: no u flag (Safari 16.4 is O(n²) with it)
|
|
1312
|
+
const _SPACE_PUNCT_RE = re.compile(`(?<![${SC}])${S}+([,.;:!?)])`, null, false);
|
|
1313
|
+
const _PAREN_SPACE_RE = re.compile(`([(])${S}+`, has('('), false);
|
|
1314
|
+
const _COMMAS_RE = re.compile(`,${S}*,`, has(','), false);
|
|
1315
|
+
|
|
1316
|
+
function _cleanup(text) {
|
|
1317
|
+
text = _SPACE_PUNCT_RE.sub((m) => m[1], text);
|
|
1318
|
+
text = _PAREN_SPACE_RE.sub((m) => m[1], text);
|
|
1319
|
+
text = _COMMAS_RE.sub(',', text);
|
|
1320
|
+
text = _BLANKS_RE.sub(' ', text);
|
|
1321
|
+
return strip(text);
|
|
1322
|
+
}
|
|
1323
|
+
|
|
1324
|
+
const _PASSES = [
|
|
1325
|
+
_pass_names,
|
|
1326
|
+
_pass_web,
|
|
1327
|
+
_pass_abbreviations,
|
|
1328
|
+
_pass_products,
|
|
1329
|
+
_pass_iban,
|
|
1330
|
+
_pass_phones,
|
|
1331
|
+
_pass_plates,
|
|
1332
|
+
_pass_dates,
|
|
1333
|
+
_pass_times,
|
|
1334
|
+
_pass_money,
|
|
1335
|
+
_pass_percent,
|
|
1336
|
+
_pass_degrees,
|
|
1337
|
+
_pass_units,
|
|
1338
|
+
_pass_powers,
|
|
1339
|
+
_pass_alnum,
|
|
1340
|
+
_pass_ranges,
|
|
1341
|
+
_pass_fractions,
|
|
1342
|
+
_pass_roman,
|
|
1343
|
+
_pass_ordinals,
|
|
1344
|
+
_pass_numbers,
|
|
1345
|
+
_pass_leftover_digits,
|
|
1346
|
+
_lower_caps_headlines,
|
|
1347
|
+
_pass_acronyms,
|
|
1348
|
+
_pass_letter_names,
|
|
1349
|
+
_pass_symbols,
|
|
1350
|
+
_pass_clitics,
|
|
1351
|
+
_cleanup,
|
|
1352
|
+
];
|
|
1353
|
+
|
|
1354
|
+
// Python (CPython 3.12 in Pyodide) knows Unicode 15.0: to it a later character is unassigned, no letter, digit or
|
|
1355
|
+
// space, without case or composition. A newer JS engine knows such characters (Unicode 16 added letters, digits and
|
|
1356
|
+
// marks), so they are swapped for a noncharacter, which every engine treats as Python treats them, and swapped back
|
|
1357
|
+
// at the end. No pass drops, adds or reorders one (only a URL path takes one in, and keeps it), so the n-th
|
|
1358
|
+
// placeholder out is the n-th character in; the placeholder itself is in the class, so one in the input round-trips.
|
|
1359
|
+
const _NONCHAR = '\u{fdd0}';
|
|
1360
|
+
// A lone surrogate is a code point of its own to Python, and stays one when a pass deletes what stood between it and
|
|
1361
|
+
// another ("\ud83d\u200d\ude00" loses its ZWJ, "5\ud800\ue000\udc00" its mark); a JS string would then hold one astral
|
|
1362
|
+
// character there, a letter to \w if it is one. So lone surrogates get a noncharacter of their own for the passes
|
|
1363
|
+
// (no letter, digit, space or cased character, as a surrogate is to Python) and come back at the end. U+FDD1 is
|
|
1364
|
+
// unassigned, so one in the input is swapped for U+FDD0 with the rest: every U+FDD1 in the output is a placeholder.
|
|
1365
|
+
const _SURROGATE = '\u{fdd1}';
|
|
1366
|
+
const _PLACEHOLDER_RE = /[\ufdd0\ufdd1]/g;
|
|
1367
|
+
// Both swaps are one pass of a pattern without the u flag, before any u-flag pattern sees the text: Safari 26
|
|
1368
|
+
// (WebKit 2215 to 2311) reads a lone high surrogate and a following U+FC00..U+FFFF as one code point under the u flag
|
|
1369
|
+
// (py.js U_PAIR_BUG; a U+FEFF after one was not removed). The pass takes every surrogate pair (swapped when it is
|
|
1370
|
+
// unassigned), every other surrogate (a lone one) and every unassigned BMP character: UNASSIGNED split into a BMP
|
|
1371
|
+
// class and astral ranges.
|
|
1372
|
+
const [_BMP_UNASSIGNED, _ASTRAL_UNASSIGNED] = (() => {
|
|
1373
|
+
const hex = (n) => `\\u${n.toString(16).padStart(4, '0')}`;
|
|
1374
|
+
let bmp = '';
|
|
1375
|
+
const astral = [];
|
|
1376
|
+
for (const [, a, b = a] of UNASSIGNED.matchAll(/\\u\{([0-9a-f]+)\}(?:-\\u\{([0-9a-f]+)\})?/g)) {
|
|
1377
|
+
const lo = parseInt(a, 16);
|
|
1378
|
+
const hi = parseInt(b, 16);
|
|
1379
|
+
if (lo <= 0xffff) bmp += `${hex(lo)}-${hex(Math.min(hi, 0xffff))}`;
|
|
1380
|
+
if (hi > 0xffff) astral.push(Math.max(lo, 0x10000), hi);
|
|
1381
|
+
}
|
|
1382
|
+
return [bmp, Uint32Array.from(astral)];
|
|
1383
|
+
})();
|
|
1384
|
+
const _PIN_RE = new RegExp(`[\\ud800-\\udbff][\\udc00-\\udfff]|[\\ud800-\\udfff]|[${_BMP_UNASSIGNED}]`, 'g');
|
|
1385
|
+
|
|
1386
|
+
// [text with the placeholders, the unassigned characters they replace, the lone surrogates]
|
|
1387
|
+
function _pin(text) {
|
|
1388
|
+
const unassigned = [];
|
|
1389
|
+
const lone = [];
|
|
1390
|
+
const pinned = text.replace(_PIN_RE, (c) => {
|
|
1391
|
+
if (c.length === 2) {
|
|
1392
|
+
if (!in_table(_ASTRAL_UNASSIGNED, c.codePointAt(0))) return c;
|
|
1393
|
+
unassigned.push(c);
|
|
1394
|
+
return _NONCHAR;
|
|
1395
|
+
}
|
|
1396
|
+
if (c >= '\ud800' && c <= '\udfff') {
|
|
1397
|
+
lone.push(c);
|
|
1398
|
+
return _SURROGATE;
|
|
1399
|
+
}
|
|
1400
|
+
unassigned.push(c);
|
|
1401
|
+
return _NONCHAR;
|
|
1402
|
+
});
|
|
1403
|
+
return [pinned, unassigned, lone];
|
|
1404
|
+
}
|
|
1405
|
+
|
|
1406
|
+
// The placeholders of out back to the characters they stand for, in order; drop: leave out the surrogates. null when
|
|
1407
|
+
// a count changed (no pass is known to do that), as the placeholders would land in the wrong places.
|
|
1408
|
+
function _unpin(out, unassigned, lone, drop) {
|
|
1409
|
+
let i = 0;
|
|
1410
|
+
let j = 0;
|
|
1411
|
+
const back = out.replace(_PLACEHOLDER_RE, (c) => {
|
|
1412
|
+
if (c === _NONCHAR) return unassigned[i++] ?? c;
|
|
1413
|
+
j++;
|
|
1414
|
+
return drop ? '' : (lone[j - 1] ?? c);
|
|
1415
|
+
});
|
|
1416
|
+
return i === unassigned.length && j === lone.length ? back : null;
|
|
1417
|
+
}
|
|
1418
|
+
|
|
1419
|
+
// [spoken, the spoken form without Python's surrogates]: normalize(text), and what ahenk_front's frontend() reads of
|
|
1420
|
+
// it (it starts with encode("utf-8", "ignore"), which drops every surrogate code point; two that a pass brought
|
|
1421
|
+
// together are one astral character in the JS string, so frontend() cannot tell them apart any more).
|
|
1422
|
+
function _normalize(text) {
|
|
1423
|
+
if (typeof text !== 'string') throw type_error(`normalize() argument must be str, not ${typeof text}`);
|
|
1424
|
+
const [pinned, unassigned, lone] = _pin(text);
|
|
1425
|
+
if (unassigned.length || lone.length) {
|
|
1426
|
+
const out = _run(pinned, _PASSES);
|
|
1427
|
+
const spoken = _unpin(out, unassigned, lone, false);
|
|
1428
|
+
if (spoken !== null) return [spoken, lone.length ? _unpin(out, unassigned, lone, true) : spoken];
|
|
1429
|
+
}
|
|
1430
|
+
const out = _run(text, _PASSES);
|
|
1431
|
+
return [out, out];
|
|
1432
|
+
}
|
|
1433
|
+
|
|
1434
|
+
// for itn.js, which has the same need
|
|
1435
|
+
export { _pin as pin_newer, _unpin as unpin_newer };
|
|
1436
|
+
|
|
1437
|
+
// For tools/parity/matchers_check.mjs: each linear matcher with the pattern it stands for, as Python writes it
|
|
1438
|
+
export const _testing = {
|
|
1439
|
+
url: [_URL_RE, _URL_SRC],
|
|
1440
|
+
email: [_EMAIL_RE, _EMAIL_SRC],
|
|
1441
|
+
alnum: [_ALNUM_RE, _ALNUM_SRC],
|
|
1442
|
+
leftover: [re.compile(`(?<![${LET}])([${LET}]+)\\.(?=${D})`), `([${LET}]+)\\.(?=${D})`],
|
|
1443
|
+
cleanup: [_SPACE_PUNCT_RE, `${S}+([,.;:!?)])`],
|
|
1444
|
+
letters_at_end: [_letters_at_end, `([${LET}]+)\\.?${END}`],
|
|
1445
|
+
preclean: _preclean,
|
|
1446
|
+
passes: _PASSES,
|
|
1447
|
+
};
|
|
1448
|
+
|
|
1449
|
+
// Convert written Turkish into its spoken form (for TTS input).
|
|
1450
|
+
//
|
|
1451
|
+
// Case, punctuation and apostrophes on proper nouns are kept, because TTS models use them for prosody.
|
|
1452
|
+
// Throws where Python raises (a ValueError for a number of more than 4300 digits).
|
|
1453
|
+
export function normalize(text) {
|
|
1454
|
+
return _normalize(text)[0];
|
|
1455
|
+
}
|
|
1456
|
+
|
|
1457
|
+
// For ahenk_front.prepare(): [normalize(text), the text its frontend() reads], the second without the surrogate code
|
|
1458
|
+
// points the Python string holds (frontend() drops them first thing).
|
|
1459
|
+
export function normalize_for_frontend(text) {
|
|
1460
|
+
return _normalize(text);
|
|
1461
|
+
}
|
|
1462
|
+
|
|
1463
|
+
function _run(text, passes) {
|
|
1464
|
+
text = _preclean(text);
|
|
1465
|
+
for (const p of passes) text = p(text);
|
|
1466
|
+
return text;
|
|
1467
|
+
}
|
|
1468
|
+
|
|
1469
|
+
// ---- normalize_for_eval ------------------------------------------------------------------------------------------
|
|
1470
|
+
|
|
1471
|
+
// The eval form keeps foreign names and single letters as written: ASR output and references spell "iPhone", "John
|
|
1472
|
+
// F. Kennedy", "X'te", and lowercase ASR output ("x'te", "f") must still match them.
|
|
1473
|
+
const _EVAL_PASSES = _PASSES.filter((p) => p !== _pass_names && p !== _pass_letter_names);
|
|
1474
|
+
const _CIRCUMFLEX = { â: 'a', î: 'i', û: 'u', Â: 'a', Î: 'i', Û: 'u' };
|
|
1475
|
+
const _CIRCUMFLEX_RE = /[âîûÂÎÛ]/g;
|
|
1476
|
+
|
|
1477
|
+
const _LOWER_ACRONYM_RE = re.compile(`${nlb(`${WC}'`)}([${LOW}]{2,6})(?:'([${LOW}]+))?(?!${W})`);
|
|
1478
|
+
|
|
1479
|
+
// Lowercase ASR output and references write "abd", "tbmm"; treat them like "ABD", "TBMM".
|
|
1480
|
+
function _uppercase_known_acronyms(text) {
|
|
1481
|
+
return _LOWER_ACRONYM_RE.sub((m) => {
|
|
1482
|
+
const up = tr_upper(m[1]);
|
|
1483
|
+
if (!L.EVAL_ACRONYMS.has(up)) return m[0];
|
|
1484
|
+
return up + (m[2] ? `'${m[2]}` : '');
|
|
1485
|
+
}, text);
|
|
1486
|
+
}
|
|
1487
|
+
|
|
1488
|
+
const _BUCUK_RE = re.compile(`${B}buçuk${B}`, has('ç'));
|
|
1489
|
+
const _INNER_APOSTROPHE_RE = re.compile(`${lb(WC)}'(?=${W})`, has("'"));
|
|
1490
|
+
const _NOT_WORD_RE = re.compile(`[^${WC}${SC}]|_`);
|
|
1491
|
+
|
|
1492
|
+
// Canonical form for WER/CER: spoken form, Turkish lowercase, no punctuation. Apply the same function to references
|
|
1493
|
+
// and hypotheses.
|
|
1494
|
+
//
|
|
1495
|
+
// The passes run on the text with its unassigned characters and lone surrogates swapped for placeholders, as in
|
|
1496
|
+
// normalize(); none of them survives the last steps (to Python they are neither \w nor \s, so each becomes a space,
|
|
1497
|
+
// as the placeholder does), so nothing is swapped back.
|
|
1498
|
+
export function normalize_for_eval(text) {
|
|
1499
|
+
if (typeof text !== 'string') throw type_error(`expected string or bytes-like object, got '${typeof text}'`);
|
|
1500
|
+
const [pinned, unassigned, lone] = _pin(text);
|
|
1501
|
+
let out = _run(_uppercase_known_acronyms(pinned), _EVAL_PASSES);
|
|
1502
|
+
if ((unassigned.length || lone.length) && _unpin(out, unassigned, lone, false) === null) {
|
|
1503
|
+
out = _run(_uppercase_known_acronyms(text), _EVAL_PASSES); // a count changed (no pass is known to do that)
|
|
1504
|
+
}
|
|
1505
|
+
out = tr_lower(out).replace(_CIRCUMFLEX_RE, (c) => _CIRCUMFLEX[c]);
|
|
1506
|
+
// "bir buçuk" and "bir virgül beş" are the same quantity
|
|
1507
|
+
out = _BUCUK_RE.sub('virgül beş', out);
|
|
1508
|
+
out = _INNER_APOSTROPHE_RE.sub('', out);
|
|
1509
|
+
// the apostrophe gone, two characters meet that a buggy engine reads as one (py.js U_PAIR_BUG)
|
|
1510
|
+
if (U_PAIR_BUG && U_PAIR_RE.test(out)) throw _engine_error(out);
|
|
1511
|
+
out = _NOT_WORD_RE.sub(' ', out);
|
|
1512
|
+
return split(out).join(' ');
|
|
1513
|
+
}
|