@ozgurguler/trnorm 0.0.0-stage → 0.1.0

package/src/py.js

@@ -0,0 +1,438 @@
+// The Python semantics the trnorm port relies on, where JavaScript differs (no Python file of its own: str, re and
+// int() as CPython 3.12 defines them, the Python Pyodide runs).
+//  - re: \w \d \b are ASCII-only in JS even with the u flag; \s and trim() use another whitespace set; $ also matches
+//    before a final newline in Python; pattern.match / fullmatch / search take pos and endpos.
+//  - str: len() and indices count code points (JS counts UTF-16 units); split() and strip() use Python's
+//    whitespace; split(sep, 1) splits once (JS's limit truncates); isupper(), isdecimal()... as Python defines them;
+//    lower() and upper() as Unicode 15.0 maps case, final sigma included (case_tables.js).
+//  - int(): unbounded (BigInt), any Unicode decimal digits, and str <-> int conversion refuses more than 4300 digits
+//    (ValueError): normalize() raises there in Python, so it throws here.
+
+import { LOWERCASE, SIGMA_CASED, SIGMA_IGNORABLE, TITLECASE, UPPERCASE } from './case_tables.js';
+import { UNASSIGNED } from './unassigned.js';
+
+// ---- regex fragments --------------------------------------------------------------------------------------------------
+// Every pattern is compiled with the u flag. Python's \w (str patterns) is str.isalnum() or "_", i.e. letters and
+// everything with a numeric value: \p{L}\p{N}_. \d is str.isdecimal(): \p{Nd}. \s is str.isspace(): Zs and the bidi
+// classes WS, B, S, so \x1c-\x1f and \x85 more than JS's \s, and \u{feff} not. Checked over every code point against
+// CPython 3.12 (Unicode 15.0): they differ only on characters Unicode 15.0 does not assign, which normalize() hides
+// from the engine (normalizer.js, unassigned.js).
+export const WC = String.raw`\p{L}\p{N}_`; // \w inside a class
+export const W = `[${WC}]`;
+export const D = String.raw`\p{Nd}`; // also usable inside a class
+export const SC = String.raw`\t-\r\x1c-\x20\x85\xa0\u{1680}\u{2000}-\u{200a}\u{2028}\u{2029}\u{202f}\u{205f}\u{3000}`;
+export const S = `[${SC}]`;
+// Safari 26.0 to 26.5 (WebKit 2215 to 2311) decode a character of U+D800..U+DBFF or U+F800..U+FBFF followed by one of
+// U+FC00..U+FFFF as one code point in a u-flag pattern (its surrogate test masks with 0xDC00, not 0xFC00): /\uff15/u
+// does not find U+FF15 in "\ufb01\uff15", so "fi５" kept its digit unread. True in such an engine. normalizer.js
+// leaves no lone surrogate to a u-flag pattern; a text that still holds such a pair is left to the Python there.
+export const U_PAIR_BUG = !/\uff15/u.test('\ufb01\uff15');
+export const U_PAIR_RE = /[\uf800-\ufbff][\ufc00-\uffff]/;
+
+// (?<=[cls]) and (?<![cls]) for a class with astral members (\w, \d): the class is tested by a lookahead on the one
+// character the lookbehind steps back over. JavaScriptCore before WebKit 2370 (Safari 16.4 to 26.5) does not match a
+// class or a \p{} escape backwards over a surrogate pair, so (?<![\p{L}])5 matched in "\u{1D465}5"; stepping back
+// with "." and looking ahead works in every engine, and V8 runs it as fast. cls must not hold a line terminator,
+// which "." does not step over.
+export const lb = (cls) => `(?<=(?=[${cls}]).)`;
+export const nlb = (cls) => `(?<!(?=[${cls}]).)`;
+export const B = `(?:${lb(WC)}(?!${W})|${nlb(WC)}(?=${W}))`; // \b
+export const END = String.raw`(?=\n?$)`; // $ (no MULTILINE): the end, or before a newline that ends the text
+export const ANY = String.raw`[^\n]`; // . (no DOTALL)
+
+// Python's re.IGNORECASE matches a character whose simple lowercase is the pattern's, plus fixed equivalents (i ~ ı,
+// s ~ ſ); JS's i flag folds case instead, so İ and ı would not match i. ci(word) spells a lowercase literal the way
+// Python matches it under re.I; the classes for i, ı, k and s are Python's, checked over all code points.
+const CI = { i: 'Iiİı', ı: 'Iiİı', k: 'Kk\u{212a}', s: 'Ssſ' };
+export function ci(word) {
+  let out = '';
+  for (const c of word) {
+    const up = c.toUpperCase();
+    out += CI[c] ? `[${CI[c]}]` : up !== c ? `[${up}${c}]` : escape(c);
+  }
+  return out;
+}
+export const CI_A_Z = 'A-Za-zİıſ\u{212a}'; // [a-z] under re.I, inside a class
+
+// re.escape for a literal: only what is special in a u-flag pattern (Python's re.escape also escapes "-", " ", "#",
+// "&", "~", which a u-flag pattern rejects)
+function escape(s) {
+  return s.replace(/[\\^$.*+?()[\]{}|/]/gu, '\\$&');
+}
+
+// A compiled pattern. Matches are JS exec arrays: m[0], m[n] (undefined where Python has None), m.index; positions are
+// UTF-16 indices (cp_back / cp_fwd do Python's character arithmetic on them). needs(text), when given, is false only
+// for a text the pattern cannot match anywhere (it lacks a character every match has): finditer and sub skip the
+// scan then, which keeps the result and saves most passes on a paragraph without digits, "%", "/"...
+//
+// unicode false: compiled without the u flag, for a pattern whose characters are all BMP and that has no negated class
+// or "." (so it matches the same): Safari 16.4 runs a u-flag pattern with a lookbehind O(n²) on a long run of spaces.
+class Pattern {
+  constructor(source, needs = null, unicode = true) {
+    this.source = unicode ? source : source.replace(/\\u\{([0-9a-f]{1,4})\}/gi, (m, h) => `\\u${h.padStart(4, '0')}`);
+    this.u = unicode ? 'u' : '';
+    this.rx = new RegExp(this.source, `g${this.u}`);
+    this.needs = needs;
+    this._y = null;
+    this._full = null;
+  }
+
+  // pattern.match(text, pos): a match that starts at pos
+  match(text, pos = 0) {
+    const rx = this._y || (this._y = new RegExp(this.source, `y${this.u}`));
+    rx.lastIndex = pos;
+    return rx.exec(text);
+  }
+
+  // pattern.fullmatch(text, pos): from pos to the end
+  fullmatch(text, pos = 0) {
+    const rx = this._full || (this._full = new RegExp(`(?:${this.source})$`, `y${this.u}`));
+    rx.lastIndex = pos;
+    return rx.exec(text);
+  }
+
+  // pattern.search(text, pos, endpos): the text ends at endpos ($, lookaheads), lookbehinds still see before pos
+  search(text, pos = 0, endpos = text.length) {
+    const rx = this.rx;
+    rx.lastIndex = pos;
+    return rx.exec(endpos < text.length ? text.slice(0, endpos) : text);
+  }
+
+  // lastIndex is set before every exec, so a callback may use the same pattern in between. No pattern in trnorm
+  // matches the empty string; one that did would step past an empty match the way Python did before 3.7.
+  *finditer(text) {
+    if (this.needs && !this.needs(text)) return;
+    const rx = this.rx;
+    let pos = 0;
+    while (pos <= text.length) {
+      rx.lastIndex = pos;
+      const m = rx.exec(text);
+      if (!m) return;
+      yield m;
+      pos = m.index + m[0].length;
+      if (!m[0]) pos += cp_len_at(text, pos);
+    }
+  }
+
+  // findall for a pattern without groups
+  findall(text) {
+    return text.match(this.rx) || [];
+  }
+
+  // sub with a function of the match, or a literal string (no backreferences)
+  sub(repl, text) {
+    if (this.needs && !this.needs(text)) return text;
+    let out = '';
+    let pos = 0;
+    for (const m of this.finditer(text)) {
+      out += text.slice(pos, m.index) + (typeof repl === 'string' ? repl : repl(m));
+      pos = m.index + m[0].length;
+    }
+    return pos ? out + text.slice(pos) : text;
+  }
+
+  split(text) {
+    return text.split(this.rx);
+  }
+}
+
+// The re module: re.compile(pattern), and re.sub / match / fullmatch / search / findall / split taking a pattern
+// source, compiled once and cached (as Python's re caches), so the port can keep Python's inline calls.
+const cache = new Map();
+function compile(source, needs = null, unicode = true) {
+  let p = cache.get(source);
+  if (!p) cache.set(source, (p = new Pattern(source, needs, unicode)));
+  return p;
+}
+// needs() helpers: the text holds one of these (BMP) characters / a decimal digit. No u flag where none is needed: a
+// plain class scan is the fastest test, and the first decimal digit outside ASCII is U+0660.
+export const has = (chars) => {
+  if (chars.length === 1) return (text) => text.includes(chars);
+  const rx = new RegExp(`[${chars.replace(/[\\\]^-]/g, '\\$&')}]`);
+  return (text) => rx.test(text);
+};
+const ASCII_DIGIT = /[0-9]/;
+const FROM_0660 = /[^\0-\u065f]/;
+const ANY_DIGIT = /\p{Nd}/u;
+export const has_digit = (text) => ASCII_DIGIT.test(text) || (FROM_0660.test(text) && ANY_DIGIT.test(text));
+export const re = {
+  compile,
+  escape,
+  sub: (source, repl, text) => compile(source).sub(repl, text),
+  match: (source, text) => compile(source).match(text),
+  fullmatch: (source, text) => compile(source).fullmatch(text),
+  search: (source, text) => compile(source).search(text),
+  findall: (source, text) => compile(source).findall(text),
+  split: (source, text) => compile(source).split(text),
+};
+// match.end()
+export const end = (m) => m.index + m[0].length;
+
+// ---- code points ------------------------------------------------------------------------------------------------------
+const isHigh = (c) => c >= 0xd800 && c <= 0xdbff;
+const isLow = (c) => c >= 0xdc00 && c <= 0xdfff;
+const cp_len_at = (s, i) => (isHigh(s.charCodeAt(i)) && isLow(s.charCodeAt(i + 1)) ? 2 : 1);
+
+// len(s)
+export function plen(s) {
+  let n = s.length;
+  for (let i = 0; i < s.length - 1; i++) {
+    if (isHigh(s.charCodeAt(i)) && isLow(s.charCodeAt(i + 1))) {
+      n--;
+      i++;
+    }
+  }
+  return n;
+}
+
+// the index n code points before / after index i: Python's text[i - n:i] is s.slice(cp_back(s, i, n), i)
+export function cp_back(s, i, n) {
+  for (; n > 0 && i > 0; n--) i -= i >= 2 && isLow(s.charCodeAt(i - 1)) && isHigh(s.charCodeAt(i - 2)) ? 2 : 1;
+  return i;
+}
+export function cp_fwd(s, i, n) {
+  for (; n > 0 && i < s.length; n--) i += cp_len_at(s, i);
+  return i;
+}
+// s[0] and s[-1] ('' for an empty string, where Python's s[:1] and s[-1:] are '')
+export const first_cp = (s) => (s ? String.fromCodePoint(s.codePointAt(0)) : '');
+export const last_cp = (s) => (s ? s.slice(cp_back(s, s.length, 1)) : '');
+// the character that ends at index i (Python's text[i - 1]) and the one that starts there ('' past either end)
+export const char_before = (s, i) => s.slice(cp_back(s, i, 1), i);
+export const char_at = (s, i) => s.slice(i, cp_fwd(s, i, 1));
+
+// ---- str methods ------------------------------------------------------------------------------------------------------
+const SPACE = new Set();
+for (const [a, b] of [[0x09, 0x0d], [0x1c, 0x20], [0x85, 0x85], [0xa0, 0xa0], [0x1680, 0x1680], [0x2000, 0x200a],
+  [0x2028, 0x2029], [0x202f, 0x202f], [0x205f, 0x205f], [0x3000, 0x3000]]) {
+  for (let c = a; c <= b; c++) SPACE.add(c);
+}
+// Python's whitespace is all in the BMP, so a UTF-16 unit test is a code point test.
+const isspace_unit = (s, i) => SPACE.has(s.charCodeAt(i));
+
+// s.strip() / s.strip(chars) and the one-sided forms; chars: BMP characters
+function stripper(s, chars, left, right) {
+  const has = chars === undefined ? (i) => isspace_unit(s, i) : (i) => chars.includes(s[i]);
+  let a = 0;
+  let b = s.length;
+  if (left) while (a < b && has(a)) a++;
+  if (right) while (b > a && has(b - 1)) b--;
+  return a === 0 && b === s.length ? s : s.slice(a, b);
+}
+export const strip = (s, chars) => stripper(s, chars, true, true);
+export const lstrip = (s, chars) => stripper(s, chars, true, false);
+export const rstrip = (s, chars) => stripper(s, chars, false, true);
+
+// s.split() (no separator): runs of whitespace, no empty strings
+export function split(s) {
+  const out = [];
+  const n = s.length;
+  let i = 0;
+  while (i < n) {
+    while (i < n && isspace_unit(s, i)) i++;
+    if (i >= n) break;
+    let j = i + 1;
+    while (j < n && !isspace_unit(s, j)) j++;
+    out.push(s.slice(i, j));
+    i = j;
+  }
+  return out;
+}
+
+// s.split(sep, 1)
+export function split1(s, sep) {
+  const i = s.indexOf(sep);
+  return i < 0 ? [s] : [s.slice(0, i), s.slice(i + sep.length)];
+}
+
+// A character Unicode 15.0 does not have: newer than Python's data (case, letters, digits). Every one is above
+// U+0377, so a text below that (Turkish and Latin text) skips the big class.
+const NEWER = new RegExp(`[${UNASSIGNED}]`, 'u');
+const ABOVE_0377 = /[^\0-\u0377]/;
+const has_newer = (s) => ABOVE_0377.test(s) && NEWER.test(s);
+const ALPHA = /^\p{L}+$/u;
+const DECIMAL = /^\p{Nd}+$/u;
+// str.isdigit(): decimals plus the other Numeric_Type=Digit characters (superscripts, circled digits...), from
+// CPython 3.12's unicodedata (the same 128 in 3.13)
+const DIGIT = new RegExp(String.raw`^[\p{Nd}\u{b2}-\u{b3}\u{b9}\u{1369}-\u{1371}\u{19da}\u{2070}\u{2074}-\u{2079}` +
+  String.raw`\u{2080}-\u{2089}\u{2460}-\u{2468}\u{2474}-\u{247c}\u{2488}-\u{2490}\u{24ea}\u{24f5}-\u{24fd}\u{24ff}` +
+  String.raw`\u{2776}-\u{277e}\u{2780}-\u{2788}\u{278a}-\u{2792}\u{10a40}-\u{10a43}\u{10e60}-\u{10e68}` +
+  String.raw`\u{11052}-\u{1105a}\u{1f100}-\u{1f10a}]+$`, 'u');
+const ALNUM = /^[\p{L}\p{N}]+$/u; // isalpha() or a numeric value
+
+// A character newer than Unicode 15.0 is none of these to Python (normalize() never asks about one: it hides them).
+// one BMP character, asked often (letter by letter): answers kept, 0 unknown, 1 no, 2 yes
+const alpha_bmp = new Uint8Array(0x10000);
+export function isalpha(s) {
+  if (s.length === 1) {
+    const c = s.charCodeAt(0);
+    return (alpha_bmp[c] ||= ALPHA.test(s) && !has_newer(s) ? 2 : 1) === 2;
+  }
+  return ALPHA.test(s) && !has_newer(s);
+}
+export const isdecimal = (s) => DECIMAL.test(s) && !has_newer(s);
+export const isdigit = (s) => DIGIT.test(s) && !has_newer(s);
+export const isalnum = (s) => ALNUM.test(s) && !has_newer(s);
+
+// ---- case -------------------------------------------------------------------------------------------------------------
+// Python's case properties, not the engine's: a newer Unicode moved some assigned characters (U+0295 is no longer
+// lowercase in Unicode 17, and Safari 26's \p{Lowercase} already says so). case_tables.js holds Python's ranges.
+const tables = {};
+function table(name, encoded) {
+  let t = tables[name];
+  if (!t) {
+    const nums = encoded.split(' ');
+    t = tables[name] = new Uint32Array(nums.length);
+    let prev = -1;
+    for (let i = 0; i < nums.length; i += 2) {
+      t[i] = prev + 1 + parseInt(nums[i], 36);
+      t[i + 1] = prev = t[i] + parseInt(nums[i + 1], 36);
+    }
+  }
+  return t;
+}
+export function in_table(t, cp) {
+  let lo = 0;
+  let hi = (t.length >> 1) - 1;
+  while (lo <= hi) {
+    const mid = (lo + hi) >> 1;
+    if (cp < t[2 * mid]) hi = mid - 1;
+    else if (cp > t[2 * mid + 1]) lo = mid + 1;
+    else return true;
+  }
+  return false;
+}
+const is_lower_cp = (cp) => (cp < 0x80 ? cp >= 0x61 && cp <= 0x7a : in_table(table('l', LOWERCASE), cp));
+const is_upper_cp = (cp) => (cp < 0x80 ? cp >= 0x41 && cp <= 0x5a : in_table(table('u', UPPERCASE), cp));
+const is_title_cp = (cp) => cp >= 0x1c5 && in_table(table('t', TITLECASE), cp);
+
+// at least one cased character, and no character of the other case (or titlecase)
+export function isupper(s) {
+  let cased = false;
+  for (const c of s) {
+    const cp = c.codePointAt(0);
+    if (is_lower_cp(cp) || is_title_cp(cp)) return false;
+    if (!cased && is_upper_cp(cp)) cased = true;
+  }
+  return cased;
+}
+export function islower(s) {
+  let cased = false;
+  for (const c of s) {
+    const cp = c.codePointAt(0);
+    if (is_upper_cp(cp) || is_title_cp(cp)) return false;
+    if (!cased && is_lower_cp(cp)) cased = true;
+  }
+  return cased;
+}
+
+
+// The engine's case mapping of one character, unless it is newer than Unicode 15.0 or maps to a newer one: that
+// mapping came later (case pairs never change once made; a later version only pairs an old character with a new one,
+// as Unicode 16 did ƛ with Ƛ U+A7DC), so Python leaves the character as it is. Checked over every code point against
+// CPython 3.12 in V8, SpiderMonkey and JavaScriptCore.
+function map_char(c, mapped) {
+  return mapped === c || (!NEWER.test(c) && !NEWER.test(mapped)) ? mapped : c;
+}
+
+// str.lower()'s Final_Sigma rule (CPython's handle_capital_sigma), on the code points cps around Σ at i: final when a
+// cased character comes before it, case-ignorable ones skipped, and none after it
+function final_sigma(cps, i) {
+  const ign = table('i', SIGMA_IGNORABLE);
+  const cased = table('c', SIGMA_CASED);
+  let j = i - 1;
+  while (j >= 0 && in_table(ign, cps[j].codePointAt(0))) j--;
+  if (j < 0 || !in_table(cased, cps[j].codePointAt(0))) return false;
+  for (j = i + 1; j < cps.length && in_table(ign, cps[j].codePointAt(0)); j++);
+  return j === cps.length || !in_table(cased, cps[j].codePointAt(0));
+}
+
+// str.lower(): toLowerCase() is the same full mapping, but it decides σ or ς with the engine's Case_Ignorable and
+// Cased, which changed after 15.0 (U+1171E is no longer case-ignorable), and it knows newer case pairs
+export function lower(s) {
+  const out = s.toLowerCase();
+  if (!s.includes('\u03a3') && !has_newer(out) && !has_newer(s)) return out;
+  const cps = Array.from(s);
+  let r = '';
+  for (let i = 0; i < cps.length; i++) {
+    const c = cps[i];
+    r += c === '\u03a3' ? (final_sigma(cps, i) ? '\u03c2' : '\u03c3') : map_char(c, c.toLowerCase());
+  }
+  return r;
+}
+
+// str.upper() (no context rule: each character on its own)
+export function upper(s) {
+  const out = s.toUpperCase();
+  if (!has_newer(out) && !has_newer(s)) return out;
+  let r = '';
+  for (const c of s) r += map_char(c, c.toUpperCase());
+  return r;
+}
+
+// ---- int() ------------------------------------------------------------------------------------------------------------
+// Python's exceptions where the port raises one on purpose (python: true; see front-native.js)
+export class ValueError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = 'ValueError';
+    this.python = true;
+  }
+}
+export const type_error = (message) => Object.assign(new TypeError(message), { python: true });
+const MAX_STR_DIGITS = 4300; // sys.get_int_max_str_digits() default
+
+// The value of a decimal digit (\p{Nd}): Unicode encodes them in runs of ten, zero to nine.
+const ND = /^\p{Nd}$/u;
+const digitCache = new Map();
+export function digit_value(ch) {
+  const cp = ch.codePointAt(0);
+  if (cp >= 0x30 && cp <= 0x39) return cp - 0x30;
+  let v = digitCache.get(cp);
+  if (v === undefined) {
+    let zero = cp;
+    while (ND.test(String.fromCodePoint(zero - 1))) zero--;
+    v = (cp - zero) % 10;
+    digitCache.set(cp, v);
+  }
+  return v;
+}
+
+// int(s) -> BigInt: surrounding whitespace, a sign, decimal digits of any script, single underscores between digits
+export function py_int(s) {
+  const t = strip(s);
+  let i = 0;
+  let neg = false;
+  if (t[0] === '+' || t[0] === '-') {
+    neg = t[0] === '-';
+    i = 1;
+  }
+  let ascii = '';
+  let underscore = false;
+  for (const c of t.slice(i)) {
+    if (c === '_' && ascii && !underscore) {
+      underscore = true;
+      continue;
+    }
+    if (!ND.test(c)) throw new ValueError(`invalid literal for int() with base 10: '${s}'`);
+    ascii += digit_value(c);
+    underscore = false;
+  }
+  if (!ascii || underscore) throw new ValueError(`invalid literal for int() with base 10: '${s}'`);
+  if (ascii.length > MAX_STR_DIGITS) {
+    throw new ValueError(`Exceeds the limit (${MAX_STR_DIGITS} digits) for integer string conversion: value has ` +
+      `${ascii.length} digits; use sys.set_int_max_str_digits() to increase the limit`);
+  }
+  const n = BigInt(ascii);
+  return neg ? -n : n;
+}
+
+// str(n) for an int
+export function int_str(n) {
+  const s = BigInt(n).toString();
+  if (s.replace('-', '').length > MAX_STR_DIGITS) {
+    throw new ValueError(`Exceeds the limit (${MAX_STR_DIGITS} digits) for integer string conversion; use ` +
+      'sys.set_int_max_str_digits() to increase the limit');
+  }
+  return s;
+}

package/src/unassigned.js

@@ -0,0 +1,100 @@
+// Generated by tools/parity/gen_unassigned.py from CPython 3.12.3 (unicodedata 15.0.0); do not edit.
+// The code points that Unicode version leaves unassigned (category Cn), outside the emoji ranges, as the inside
+// of a u-flag regex class.
+export const UNICODE_VERSION = '15.0.0';
+export const UNASSIGNED =
+  String.raw`\u{378}-\u{379}\u{380}-\u{383}\u{38b}\u{38d}\u{3a2}\u{530}\u{557}-\u{558}\u{58b}-\u{58c}\u{590}\u{5c8}-\u{5cf}` +
+  String.raw`\u{5eb}-\u{5ee}\u{5f5}-\u{5ff}\u{70e}\u{74b}-\u{74c}\u{7b2}-\u{7bf}\u{7fb}-\u{7fc}\u{82e}-\u{82f}\u{83f}` +
+  String.raw`\u{85c}-\u{85d}\u{85f}\u{86b}-\u{86f}\u{88f}\u{892}-\u{897}\u{984}\u{98d}-\u{98e}\u{991}-\u{992}\u{9a9}\u{9b1}` +
+  String.raw`\u{9b3}-\u{9b5}\u{9ba}-\u{9bb}\u{9c5}-\u{9c6}\u{9c9}-\u{9ca}\u{9cf}-\u{9d6}\u{9d8}-\u{9db}\u{9de}` +
+  String.raw`\u{9e4}-\u{9e5}\u{9ff}-\u{a00}\u{a04}\u{a0b}-\u{a0e}\u{a11}-\u{a12}\u{a29}\u{a31}\u{a34}\u{a37}\u{a3a}-\u{a3b}` +
+  String.raw`\u{a3d}\u{a43}-\u{a46}\u{a49}-\u{a4a}\u{a4e}-\u{a50}\u{a52}-\u{a58}\u{a5d}\u{a5f}-\u{a65}\u{a77}-\u{a80}` +
+  String.raw`\u{a84}\u{a8e}\u{a92}\u{aa9}\u{ab1}\u{ab4}\u{aba}-\u{abb}\u{ac6}\u{aca}\u{ace}-\u{acf}\u{ad1}-\u{adf}` +
+  String.raw`\u{ae4}-\u{ae5}\u{af2}-\u{af8}\u{b00}\u{b04}\u{b0d}-\u{b0e}\u{b11}-\u{b12}\u{b29}\u{b31}\u{b34}\u{b3a}-\u{b3b}` +
+  String.raw`\u{b45}-\u{b46}\u{b49}-\u{b4a}\u{b4e}-\u{b54}\u{b58}-\u{b5b}\u{b5e}\u{b64}-\u{b65}\u{b78}-\u{b81}\u{b84}` +
+  String.raw`\u{b8b}-\u{b8d}\u{b91}\u{b96}-\u{b98}\u{b9b}\u{b9d}\u{ba0}-\u{ba2}\u{ba5}-\u{ba7}\u{bab}-\u{bad}` +
+  String.raw`\u{bba}-\u{bbd}\u{bc3}-\u{bc5}\u{bc9}\u{bce}-\u{bcf}\u{bd1}-\u{bd6}\u{bd8}-\u{be5}\u{bfb}-\u{bff}\u{c0d}` +
+  String.raw`\u{c11}\u{c29}\u{c3a}-\u{c3b}\u{c45}\u{c49}\u{c4e}-\u{c54}\u{c57}\u{c5b}-\u{c5c}\u{c5e}-\u{c5f}\u{c64}-\u{c65}` +
+  String.raw`\u{c70}-\u{c76}\u{c8d}\u{c91}\u{ca9}\u{cb4}\u{cba}-\u{cbb}\u{cc5}\u{cc9}\u{cce}-\u{cd4}\u{cd7}-\u{cdc}\u{cdf}` +
+  String.raw`\u{ce4}-\u{ce5}\u{cf0}\u{cf4}-\u{cff}\u{d0d}\u{d11}\u{d45}\u{d49}\u{d50}-\u{d53}\u{d64}-\u{d65}\u{d80}\u{d84}` +
+  String.raw`\u{d97}-\u{d99}\u{db2}\u{dbc}\u{dbe}-\u{dbf}\u{dc7}-\u{dc9}\u{dcb}-\u{dce}\u{dd5}\u{dd7}\u{de0}-\u{de5}` +
+  String.raw`\u{df0}-\u{df1}\u{df5}-\u{e00}\u{e3b}-\u{e3e}\u{e5c}-\u{e80}\u{e83}\u{e85}\u{e8b}\u{ea4}\u{ea6}\u{ebe}-\u{ebf}` +
+  String.raw`\u{ec5}\u{ec7}\u{ecf}\u{eda}-\u{edb}\u{ee0}-\u{eff}\u{f48}\u{f6d}-\u{f70}\u{f98}\u{fbd}\u{fcd}\u{fdb}-\u{fff}` +
+  String.raw`\u{10c6}\u{10c8}-\u{10cc}\u{10ce}-\u{10cf}\u{1249}\u{124e}-\u{124f}\u{1257}\u{1259}\u{125e}-\u{125f}\u{1289}` +
+  String.raw`\u{128e}-\u{128f}\u{12b1}\u{12b6}-\u{12b7}\u{12bf}\u{12c1}\u{12c6}-\u{12c7}\u{12d7}\u{1311}\u{1316}-\u{1317}` +
+  String.raw`\u{135b}-\u{135c}\u{137d}-\u{137f}\u{139a}-\u{139f}\u{13f6}-\u{13f7}\u{13fe}-\u{13ff}\u{169d}-\u{169f}` +
+  String.raw`\u{16f9}-\u{16ff}\u{1716}-\u{171e}\u{1737}-\u{173f}\u{1754}-\u{175f}\u{176d}\u{1771}\u{1774}-\u{177f}` +
+  String.raw`\u{17de}-\u{17df}\u{17ea}-\u{17ef}\u{17fa}-\u{17ff}\u{181a}-\u{181f}\u{1879}-\u{187f}\u{18ab}-\u{18af}` +
+  String.raw`\u{18f6}-\u{18ff}\u{191f}\u{192c}-\u{192f}\u{193c}-\u{193f}\u{1941}-\u{1943}\u{196e}-\u{196f}\u{1975}-\u{197f}` +
+  String.raw`\u{19ac}-\u{19af}\u{19ca}-\u{19cf}\u{19db}-\u{19dd}\u{1a1c}-\u{1a1d}\u{1a5f}\u{1a7d}-\u{1a7e}\u{1a8a}-\u{1a8f}` +
+  String.raw`\u{1a9a}-\u{1a9f}\u{1aae}-\u{1aaf}\u{1acf}-\u{1aff}\u{1b4d}-\u{1b4f}\u{1b7f}\u{1bf4}-\u{1bfb}\u{1c38}-\u{1c3a}` +
+  String.raw`\u{1c4a}-\u{1c4c}\u{1c89}-\u{1c8f}\u{1cbb}-\u{1cbc}\u{1cc8}-\u{1ccf}\u{1cfb}-\u{1cff}\u{1f16}-\u{1f17}` +
+  String.raw`\u{1f1e}-\u{1f1f}\u{1f46}-\u{1f47}\u{1f4e}-\u{1f4f}\u{1f58}\u{1f5a}\u{1f5c}\u{1f5e}\u{1f7e}-\u{1f7f}\u{1fb5}` +
+  String.raw`\u{1fc5}\u{1fd4}-\u{1fd5}\u{1fdc}\u{1ff0}-\u{1ff1}\u{1ff5}\u{1fff}\u{2065}\u{2072}-\u{2073}\u{208f}` +
+  String.raw`\u{209d}-\u{209f}\u{20c1}-\u{20cf}\u{20f1}-\u{20ff}\u{218c}-\u{218f}\u{2427}-\u{243f}\u{244b}-\u{245f}` +
+  String.raw`\u{2cf4}-\u{2cf8}\u{2d26}\u{2d28}-\u{2d2c}\u{2d2e}-\u{2d2f}\u{2d68}-\u{2d6e}\u{2d71}-\u{2d7e}\u{2d97}-\u{2d9f}` +
+  String.raw`\u{2da7}\u{2daf}\u{2db7}\u{2dbf}\u{2dc7}\u{2dcf}\u{2dd7}\u{2ddf}\u{2e5e}-\u{2e7f}\u{2e9a}\u{2ef4}-\u{2eff}` +
+  String.raw`\u{2fd6}-\u{2fef}\u{2ffc}-\u{2fff}\u{3040}\u{3097}-\u{3098}\u{3100}-\u{3104}\u{3130}\u{318f}\u{31e4}-\u{31ef}` +
+  String.raw`\u{321f}\u{a48d}-\u{a48f}\u{a4c7}-\u{a4cf}\u{a62c}-\u{a63f}\u{a6f8}-\u{a6ff}\u{a7cb}-\u{a7cf}\u{a7d2}\u{a7d4}` +
+  String.raw`\u{a7da}-\u{a7f1}\u{a82d}-\u{a82f}\u{a83a}-\u{a83f}\u{a878}-\u{a87f}\u{a8c6}-\u{a8cd}\u{a8da}-\u{a8df}` +
+  String.raw`\u{a954}-\u{a95e}\u{a97d}-\u{a97f}\u{a9ce}\u{a9da}-\u{a9dd}\u{a9ff}\u{aa37}-\u{aa3f}\u{aa4e}-\u{aa4f}` +
+  String.raw`\u{aa5a}-\u{aa5b}\u{aac3}-\u{aada}\u{aaf7}-\u{ab00}\u{ab07}-\u{ab08}\u{ab0f}-\u{ab10}\u{ab17}-\u{ab1f}\u{ab27}` +
+  String.raw`\u{ab2f}\u{ab6c}-\u{ab6f}\u{abee}-\u{abef}\u{abfa}-\u{abff}\u{d7a4}-\u{d7af}\u{d7c7}-\u{d7ca}\u{d7fc}-\u{d7ff}` +
+  String.raw`\u{fa6e}-\u{fa6f}\u{fada}-\u{faff}\u{fb07}-\u{fb12}\u{fb18}-\u{fb1c}\u{fb37}\u{fb3d}\u{fb3f}\u{fb42}\u{fb45}` +
+  String.raw`\u{fbc3}-\u{fbd2}\u{fd90}-\u{fd91}\u{fdc8}-\u{fdce}\u{fdd0}-\u{fdef}\u{fe1a}-\u{fe1f}\u{fe53}\u{fe67}` +
+  String.raw`\u{fe6c}-\u{fe6f}\u{fe75}\u{fefd}-\u{fefe}\u{ff00}\u{ffbf}-\u{ffc1}\u{ffc8}-\u{ffc9}\u{ffd0}-\u{ffd1}` +
+  String.raw`\u{ffd8}-\u{ffd9}\u{ffdd}-\u{ffdf}\u{ffe7}\u{ffef}-\u{fff8}\u{fffe}-\u{ffff}\u{1000c}\u{10027}\u{1003b}` +
+  String.raw`\u{1003e}\u{1004e}-\u{1004f}\u{1005e}-\u{1007f}\u{100fb}-\u{100ff}\u{10103}-\u{10106}\u{10134}-\u{10136}` +
+  String.raw`\u{1018f}\u{1019d}-\u{1019f}\u{101a1}-\u{101cf}\u{101fe}-\u{1027f}\u{1029d}-\u{1029f}\u{102d1}-\u{102df}` +
+  String.raw`\u{102fc}-\u{102ff}\u{10324}-\u{1032c}\u{1034b}-\u{1034f}\u{1037b}-\u{1037f}\u{1039e}\u{103c4}-\u{103c7}` +
+  String.raw`\u{103d6}-\u{103ff}\u{1049e}-\u{1049f}\u{104aa}-\u{104af}\u{104d4}-\u{104d7}\u{104fc}-\u{104ff}` +
+  String.raw`\u{10528}-\u{1052f}\u{10564}-\u{1056e}\u{1057b}\u{1058b}\u{10593}\u{10596}\u{105a2}\u{105b2}\u{105ba}` +
+  String.raw`\u{105bd}-\u{105ff}\u{10737}-\u{1073f}\u{10756}-\u{1075f}\u{10768}-\u{1077f}\u{10786}\u{107b1}` +
+  String.raw`\u{107bb}-\u{107ff}\u{10806}-\u{10807}\u{10809}\u{10836}\u{10839}-\u{1083b}\u{1083d}-\u{1083e}\u{10856}` +
+  String.raw`\u{1089f}-\u{108a6}\u{108b0}-\u{108df}\u{108f3}\u{108f6}-\u{108fa}\u{1091c}-\u{1091e}\u{1093a}-\u{1093e}` +
+  String.raw`\u{10940}-\u{1097f}\u{109b8}-\u{109bb}\u{109d0}-\u{109d1}\u{10a04}\u{10a07}-\u{10a0b}\u{10a14}\u{10a18}` +
+  String.raw`\u{10a36}-\u{10a37}\u{10a3b}-\u{10a3e}\u{10a49}-\u{10a4f}\u{10a59}-\u{10a5f}\u{10aa0}-\u{10abf}` +
+  String.raw`\u{10ae7}-\u{10aea}\u{10af7}-\u{10aff}\u{10b36}-\u{10b38}\u{10b56}-\u{10b57}\u{10b73}-\u{10b77}` +
+  String.raw`\u{10b92}-\u{10b98}\u{10b9d}-\u{10ba8}\u{10bb0}-\u{10bff}\u{10c49}-\u{10c7f}\u{10cb3}-\u{10cbf}` +
+  String.raw`\u{10cf3}-\u{10cf9}\u{10d28}-\u{10d2f}\u{10d3a}-\u{10e5f}\u{10e7f}\u{10eaa}\u{10eae}-\u{10eaf}` +
+  String.raw`\u{10eb2}-\u{10efc}\u{10f28}-\u{10f2f}\u{10f5a}-\u{10f6f}\u{10f8a}-\u{10faf}\u{10fcc}-\u{10fdf}` +
+  String.raw`\u{10ff7}-\u{10fff}\u{1104e}-\u{11051}\u{11076}-\u{1107e}\u{110c3}-\u{110cc}\u{110ce}-\u{110cf}` +
+  String.raw`\u{110e9}-\u{110ef}\u{110fa}-\u{110ff}\u{11135}\u{11148}-\u{1114f}\u{11177}-\u{1117f}\u{111e0}` +
+  String.raw`\u{111f5}-\u{111ff}\u{11212}\u{11242}-\u{1127f}\u{11287}\u{11289}\u{1128e}\u{1129e}\u{112aa}-\u{112af}` +
+  String.raw`\u{112eb}-\u{112ef}\u{112fa}-\u{112ff}\u{11304}\u{1130d}-\u{1130e}\u{11311}-\u{11312}\u{11329}\u{11331}` +
+  String.raw`\u{11334}\u{1133a}\u{11345}-\u{11346}\u{11349}-\u{1134a}\u{1134e}-\u{1134f}\u{11351}-\u{11356}` +
+  String.raw`\u{11358}-\u{1135c}\u{11364}-\u{11365}\u{1136d}-\u{1136f}\u{11375}-\u{113ff}\u{1145c}\u{11462}-\u{1147f}` +
+  String.raw`\u{114c8}-\u{114cf}\u{114da}-\u{1157f}\u{115b6}-\u{115b7}\u{115de}-\u{115ff}\u{11645}-\u{1164f}` +
+  String.raw`\u{1165a}-\u{1165f}\u{1166d}-\u{1167f}\u{116ba}-\u{116bf}\u{116ca}-\u{116ff}\u{1171b}-\u{1171c}` +
+  String.raw`\u{1172c}-\u{1172f}\u{11747}-\u{117ff}\u{1183c}-\u{1189f}\u{118f3}-\u{118fe}\u{11907}-\u{11908}` +
+  String.raw`\u{1190a}-\u{1190b}\u{11914}\u{11917}\u{11936}\u{11939}-\u{1193a}\u{11947}-\u{1194f}\u{1195a}-\u{1199f}` +
+  String.raw`\u{119a8}-\u{119a9}\u{119d8}-\u{119d9}\u{119e5}-\u{119ff}\u{11a48}-\u{11a4f}\u{11aa3}-\u{11aaf}` +
+  String.raw`\u{11af9}-\u{11aff}\u{11b0a}-\u{11bff}\u{11c09}\u{11c37}\u{11c46}-\u{11c4f}\u{11c6d}-\u{11c6f}` +
+  String.raw`\u{11c90}-\u{11c91}\u{11ca8}\u{11cb7}-\u{11cff}\u{11d07}\u{11d0a}\u{11d37}-\u{11d39}\u{11d3b}\u{11d3e}` +
+  String.raw`\u{11d48}-\u{11d4f}\u{11d5a}-\u{11d5f}\u{11d66}\u{11d69}\u{11d8f}\u{11d92}\u{11d99}-\u{11d9f}` +
+  String.raw`\u{11daa}-\u{11edf}\u{11ef9}-\u{11eff}\u{11f11}\u{11f3b}-\u{11f3d}\u{11f5a}-\u{11faf}\u{11fb1}-\u{11fbf}` +
+  String.raw`\u{11ff2}-\u{11ffe}\u{1239a}-\u{123ff}\u{1246f}\u{12475}-\u{1247f}\u{12544}-\u{12f8f}\u{12ff3}-\u{12fff}` +
+  String.raw`\u{13456}-\u{143ff}\u{14647}-\u{167ff}\u{16a39}-\u{16a3f}\u{16a5f}\u{16a6a}-\u{16a6d}\u{16abf}` +
+  String.raw`\u{16aca}-\u{16acf}\u{16aee}-\u{16aef}\u{16af6}-\u{16aff}\u{16b46}-\u{16b4f}\u{16b5a}\u{16b62}` +
+  String.raw`\u{16b78}-\u{16b7c}\u{16b90}-\u{16e3f}\u{16e9b}-\u{16eff}\u{16f4b}-\u{16f4e}\u{16f88}-\u{16f8e}` +
+  String.raw`\u{16fa0}-\u{16fdf}\u{16fe5}-\u{16fef}\u{16ff2}-\u{16fff}\u{187f8}-\u{187ff}\u{18cd6}-\u{18cff}` +
+  String.raw`\u{18d09}-\u{1afef}\u{1aff4}\u{1affc}\u{1afff}\u{1b123}-\u{1b131}\u{1b133}-\u{1b14f}\u{1b153}-\u{1b154}` +
+  String.raw`\u{1b156}-\u{1b163}\u{1b168}-\u{1b16f}\u{1b2fc}-\u{1bbff}\u{1bc6b}-\u{1bc6f}\u{1bc7d}-\u{1bc7f}` +
+  String.raw`\u{1bc89}-\u{1bc8f}\u{1bc9a}-\u{1bc9b}\u{1bca4}-\u{1ceff}\u{1cf2e}-\u{1cf2f}\u{1cf47}-\u{1cf4f}` +
+  String.raw`\u{1cfc4}-\u{1cfff}\u{1d0f6}-\u{1d0ff}\u{1d127}-\u{1d128}\u{1d1eb}-\u{1d1ff}\u{1d246}-\u{1d2bf}` +
+  String.raw`\u{1d2d4}-\u{1d2df}\u{1d2f4}-\u{1d2ff}\u{1d357}-\u{1d35f}\u{1d379}-\u{1d3ff}\u{1d455}\u{1d49d}` +
+  String.raw`\u{1d4a0}-\u{1d4a1}\u{1d4a3}-\u{1d4a4}\u{1d4a7}-\u{1d4a8}\u{1d4ad}\u{1d4ba}\u{1d4bc}\u{1d4c4}\u{1d506}` +
+  String.raw`\u{1d50b}-\u{1d50c}\u{1d515}\u{1d51d}\u{1d53a}\u{1d53f}\u{1d545}\u{1d547}-\u{1d549}\u{1d551}` +
+  String.raw`\u{1d6a6}-\u{1d6a7}\u{1d7cc}-\u{1d7cd}\u{1da8c}-\u{1da9a}\u{1daa0}\u{1dab0}-\u{1deff}\u{1df1f}-\u{1df24}` +
+  String.raw`\u{1df2b}-\u{1dfff}\u{1e007}\u{1e019}-\u{1e01a}\u{1e022}\u{1e025}\u{1e02b}-\u{1e02f}\u{1e06e}-\u{1e08e}` +
+  String.raw`\u{1e090}-\u{1e0ff}\u{1e12d}-\u{1e12f}\u{1e13e}-\u{1e13f}\u{1e14a}-\u{1e14d}\u{1e150}-\u{1e28f}` +
+  String.raw`\u{1e2af}-\u{1e2bf}\u{1e2fa}-\u{1e2fe}\u{1e300}-\u{1e4cf}\u{1e4fa}-\u{1e7df}\u{1e7e7}\u{1e7ec}\u{1e7ef}` +
+  String.raw`\u{1e7ff}\u{1e8c5}-\u{1e8c6}\u{1e8d7}-\u{1e8ff}\u{1e94c}-\u{1e94f}\u{1e95a}-\u{1e95d}\u{1e960}-\u{1ec70}` +
+  String.raw`\u{1ecb5}-\u{1ed00}\u{1ed3e}-\u{1edff}\u{1ee04}\u{1ee20}\u{1ee23}\u{1ee25}-\u{1ee26}\u{1ee28}\u{1ee33}` +
+  String.raw`\u{1ee38}\u{1ee3a}\u{1ee3c}-\u{1ee41}\u{1ee43}-\u{1ee46}\u{1ee48}\u{1ee4a}\u{1ee4c}\u{1ee50}\u{1ee53}` +
+  String.raw`\u{1ee55}-\u{1ee56}\u{1ee58}\u{1ee5a}\u{1ee5c}\u{1ee5e}\u{1ee60}\u{1ee63}\u{1ee65}-\u{1ee66}\u{1ee6b}\u{1ee73}` +
+  String.raw`\u{1ee78}\u{1ee7d}\u{1ee7f}\u{1ee8a}\u{1ee9c}-\u{1eea0}\u{1eea4}\u{1eeaa}\u{1eebc}-\u{1eeef}` +
+  String.raw`\u{1eef2}-\u{1efff}\u{1fb93}\u{1fbcb}-\u{1fbef}\u{1fbfa}-\u{1fbff}\u{1fffe}-\u{1ffff}\u{2a6e0}-\u{2a6ff}` +
+  String.raw`\u{2b73a}-\u{2b73f}\u{2b81e}-\u{2b81f}\u{2cea2}-\u{2ceaf}\u{2ebe1}-\u{2f7ff}\u{2fa1e}-\u{2ffff}` +
+  String.raw`\u{3134b}-\u{3134f}\u{323b0}-\u{e0000}\u{e0002}-\u{e001f}\u{e0080}-\u{e00ff}\u{e01f0}-\u{effff}` +
+  String.raw`\u{ffffe}-\u{fffff}\u{10fffe}-\u{10ffff}`;