polytypo 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +109 -0
- data/lib/polytypo/data/README.md +20 -0
- data/lib/polytypo/data/UNICODE +1 -0
- data/lib/polytypo/data/VERSION +1 -0
- data/lib/polytypo/data/fixtures/de-CH.json +501 -0
- data/lib/polytypo/data/fixtures/de-DE.json +547 -0
- data/lib/polytypo/data/fixtures/el.json +239 -0
- data/lib/polytypo/data/fixtures/en-GB.json +1274 -0
- data/lib/polytypo/data/fixtures/en-US.json +1807 -0
- data/lib/polytypo/data/fixtures/fi.json +1306 -0
- data/lib/polytypo/data/fixtures/fr-CA.json +268 -0
- data/lib/polytypo/data/fixtures/fr.json +603 -0
- data/lib/polytypo/data/fixtures/locale-resolution.json +209 -0
- data/lib/polytypo/data/fixtures/ru.json +688 -0
- data/lib/polytypo/data/fixtures/sv.json +1290 -0
- data/lib/polytypo/data/locales/de-CH.json +77 -0
- data/lib/polytypo/data/locales/de-DE.json +76 -0
- data/lib/polytypo/data/locales/el.json +90 -0
- data/lib/polytypo/data/locales/en-GB.json +115 -0
- data/lib/polytypo/data/locales/en-US.json +133 -0
- data/lib/polytypo/data/locales/fi.json +136 -0
- data/lib/polytypo/data/locales/fr-CA.json +78 -0
- data/lib/polytypo/data/locales/fr.json +84 -0
- data/lib/polytypo/data/locales/registry.json +9 -0
- data/lib/polytypo/data/locales/ru.json +112 -0
- data/lib/polytypo/data/locales/sv.json +124 -0
- data/lib/polytypo/data/rules/dashes.md +1238 -0
- data/lib/polytypo/data/rules/order.json +78 -0
- data/lib/polytypo/data/schema/fixtures.schema.json +79 -0
- data/lib/polytypo/data/schema/locale.schema.json +235 -0
- data/lib/polytypo/data/schema/registry.schema.json +29 -0
- data/lib/polytypo/data/schema/resolution.schema.json +50 -0
- data/lib/polytypo/engine/codepoints.rb +24 -0
- data/lib/polytypo/engine/edits.rb +64 -0
- data/lib/polytypo/engine/locale.rb +138 -0
- data/lib/polytypo/engine/pipeline.rb +61 -0
- data/lib/polytypo/engine/registry.rb +47 -0
- data/lib/polytypo/engine/rules/apostrophe.rb +127 -0
- data/lib/polytypo/engine/rules/dash_shared.rb +342 -0
- data/lib/polytypo/engine/rules/dashes.rb +125 -0
- data/lib/polytypo/engine/rules/ellipsis.rb +100 -0
- data/lib/polytypo/engine/rules/hyphen.rb +207 -0
- data/lib/polytypo/engine/rules/nbsp.rb +616 -0
- data/lib/polytypo/engine/rules/quote_ambiguity.rb +241 -0
- data/lib/polytypo/engine/rules/quotes.rb +420 -0
- data/lib/polytypo/engine/rules/ranges.rb +124 -0
- data/lib/polytypo/engine/rules/spaces.rb +232 -0
- data/lib/polytypo/engine/rules/symbols.rb +291 -0
- data/lib/polytypo/engine/rules.rb +19 -0
- data/lib/polytypo/engine/sentinels.rb +23 -0
- data/lib/polytypo/engine/unicode_util.rb +390 -0
- data/lib/polytypo/errors.rb +24 -0
- data/lib/polytypo/modes/html.rb +233 -0
- data/lib/polytypo/modes/markdown.rb +187 -0
- data/lib/polytypo/modes/parse_error.rb +19 -0
- data/lib/polytypo/modes/runner.rb +57 -0
- data/lib/polytypo/modes/spans.rb +132 -0
- data/lib/polytypo/version.rb +5 -0
- data/lib/polytypo.rb +91 -0
- data/polytypo.gemspec +37 -0
- metadata +122 -0
|
@@ -0,0 +1,390 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Polytypo
|
|
4
|
+
module Engine
|
|
5
|
+
# Shared Unicode general-category data for the rule engine, ported verbatim from the JS
|
|
6
|
+
# reference implementation's src/engine/unicode.ts -- a DERIVED artifact generated from the
|
|
7
|
+
# Node/V8 UCD (Unicode 17.0, matching spec/UNICODE), not from Ruby's own Onigmo Unicode
|
|
8
|
+
# tables. Using the same frozen table as JS (rather than a Ruby-native \\p{} Regexp property
|
|
9
|
+
# escape or another gem's own bundled UCD) keeps this port on the exact same Unicode version
|
|
10
|
+
# as the canonical reference implementation, deterministically, regardless of which Onigmo
|
|
11
|
+
# version a given Ruby install ships -- and keeps regex out of core rule logic entirely,
|
|
12
|
+
# consistent with every other port (ARCHITECTURE.md section 4.1 exists for Go's RE2, but the
|
|
13
|
+
# portability argument -- one deterministic table, not five interpreters' own Unicode data --
|
|
14
|
+
# applies here too).
|
|
15
|
+
module UnicodeUtil
|
|
16
|
+
# Lu, Ll, Lt, Lm, Lo, Mn, Mc, Me as inclusive code-point ranges, flattened to
|
|
17
|
+
# [start, end, start, end, ...] and sorted ascending.
|
|
18
|
+
LETTER_RANGES = [
|
|
19
|
+
0x41, 0x5a, 0x61, 0x7a, 0xaa, 0xaa, 0xb5, 0xb5, 0xba, 0xba, 0xc0, 0xd6,
|
|
20
|
+
0xd8, 0xf6, 0xf8, 0x2c1, 0x2c6, 0x2d1, 0x2e0, 0x2e4, 0x2ec, 0x2ec, 0x2ee, 0x2ee,
|
|
21
|
+
0x300, 0x374, 0x376, 0x377, 0x37a, 0x37d, 0x37f, 0x37f, 0x386, 0x386, 0x388, 0x38a,
|
|
22
|
+
0x38c, 0x38c, 0x38e, 0x3a1, 0x3a3, 0x3f5, 0x3f7, 0x481, 0x483, 0x52f, 0x531, 0x556,
|
|
23
|
+
0x559, 0x559, 0x560, 0x588, 0x591, 0x5bd, 0x5bf, 0x5bf, 0x5c1, 0x5c2, 0x5c4, 0x5c5,
|
|
24
|
+
0x5c7, 0x5c7, 0x5d0, 0x5ea, 0x5ef, 0x5f2, 0x610, 0x61a, 0x620, 0x65f, 0x66e, 0x6d3,
|
|
25
|
+
0x6d5, 0x6dc, 0x6df, 0x6e8, 0x6ea, 0x6ef, 0x6fa, 0x6fc, 0x6ff, 0x6ff, 0x710, 0x74a,
|
|
26
|
+
0x74d, 0x7b1, 0x7ca, 0x7f5, 0x7fa, 0x7fa, 0x7fd, 0x7fd, 0x800, 0x82d, 0x840, 0x85b,
|
|
27
|
+
0x860, 0x86a, 0x870, 0x887, 0x889, 0x88f, 0x897, 0x8e1, 0x8e3, 0x963, 0x971, 0x983,
|
|
28
|
+
0x985, 0x98c, 0x98f, 0x990, 0x993, 0x9a8, 0x9aa, 0x9b0, 0x9b2, 0x9b2, 0x9b6, 0x9b9,
|
|
29
|
+
0x9bc, 0x9c4, 0x9c7, 0x9c8, 0x9cb, 0x9ce, 0x9d7, 0x9d7, 0x9dc, 0x9dd, 0x9df, 0x9e3,
|
|
30
|
+
0x9f0, 0x9f1, 0x9fc, 0x9fc, 0x9fe, 0x9fe, 0xa01, 0xa03, 0xa05, 0xa0a, 0xa0f, 0xa10,
|
|
31
|
+
0xa13, 0xa28, 0xa2a, 0xa30, 0xa32, 0xa33, 0xa35, 0xa36, 0xa38, 0xa39, 0xa3c, 0xa3c,
|
|
32
|
+
0xa3e, 0xa42, 0xa47, 0xa48, 0xa4b, 0xa4d, 0xa51, 0xa51, 0xa59, 0xa5c, 0xa5e, 0xa5e,
|
|
33
|
+
0xa70, 0xa75, 0xa81, 0xa83, 0xa85, 0xa8d, 0xa8f, 0xa91, 0xa93, 0xaa8, 0xaaa, 0xab0,
|
|
34
|
+
0xab2, 0xab3, 0xab5, 0xab9, 0xabc, 0xac5, 0xac7, 0xac9, 0xacb, 0xacd, 0xad0, 0xad0,
|
|
35
|
+
0xae0, 0xae3, 0xaf9, 0xaff, 0xb01, 0xb03, 0xb05, 0xb0c, 0xb0f, 0xb10, 0xb13, 0xb28,
|
|
36
|
+
0xb2a, 0xb30, 0xb32, 0xb33, 0xb35, 0xb39, 0xb3c, 0xb44, 0xb47, 0xb48, 0xb4b, 0xb4d,
|
|
37
|
+
0xb55, 0xb57, 0xb5c, 0xb5d, 0xb5f, 0xb63, 0xb71, 0xb71, 0xb82, 0xb83, 0xb85, 0xb8a,
|
|
38
|
+
0xb8e, 0xb90, 0xb92, 0xb95, 0xb99, 0xb9a, 0xb9c, 0xb9c, 0xb9e, 0xb9f, 0xba3, 0xba4,
|
|
39
|
+
0xba8, 0xbaa, 0xbae, 0xbb9, 0xbbe, 0xbc2, 0xbc6, 0xbc8, 0xbca, 0xbcd, 0xbd0, 0xbd0,
|
|
40
|
+
0xbd7, 0xbd7, 0xc00, 0xc0c, 0xc0e, 0xc10, 0xc12, 0xc28, 0xc2a, 0xc39, 0xc3c, 0xc44,
|
|
41
|
+
0xc46, 0xc48, 0xc4a, 0xc4d, 0xc55, 0xc56, 0xc58, 0xc5a, 0xc5c, 0xc5d, 0xc60, 0xc63,
|
|
42
|
+
0xc80, 0xc83, 0xc85, 0xc8c, 0xc8e, 0xc90, 0xc92, 0xca8, 0xcaa, 0xcb3, 0xcb5, 0xcb9,
|
|
43
|
+
0xcbc, 0xcc4, 0xcc6, 0xcc8, 0xcca, 0xccd, 0xcd5, 0xcd6, 0xcdc, 0xcde, 0xce0, 0xce3,
|
|
44
|
+
0xcf1, 0xcf3, 0xd00, 0xd0c, 0xd0e, 0xd10, 0xd12, 0xd44, 0xd46, 0xd48, 0xd4a, 0xd4e,
|
|
45
|
+
0xd54, 0xd57, 0xd5f, 0xd63, 0xd7a, 0xd7f, 0xd81, 0xd83, 0xd85, 0xd96, 0xd9a, 0xdb1,
|
|
46
|
+
0xdb3, 0xdbb, 0xdbd, 0xdbd, 0xdc0, 0xdc6, 0xdca, 0xdca, 0xdcf, 0xdd4, 0xdd6, 0xdd6,
|
|
47
|
+
0xdd8, 0xddf, 0xdf2, 0xdf3, 0xe01, 0xe3a, 0xe40, 0xe4e, 0xe81, 0xe82, 0xe84, 0xe84,
|
|
48
|
+
0xe86, 0xe8a, 0xe8c, 0xea3, 0xea5, 0xea5, 0xea7, 0xebd, 0xec0, 0xec4, 0xec6, 0xec6,
|
|
49
|
+
0xec8, 0xece, 0xedc, 0xedf, 0xf00, 0xf00, 0xf18, 0xf19, 0xf35, 0xf35, 0xf37, 0xf37,
|
|
50
|
+
0xf39, 0xf39, 0xf3e, 0xf47, 0xf49, 0xf6c, 0xf71, 0xf84, 0xf86, 0xf97, 0xf99, 0xfbc,
|
|
51
|
+
0xfc6, 0xfc6, 0x1000, 0x103f, 0x1050, 0x108f, 0x109a, 0x109d, 0x10a0, 0x10c5, 0x10c7, 0x10c7,
|
|
52
|
+
0x10cd, 0x10cd, 0x10d0, 0x10fa, 0x10fc, 0x1248, 0x124a, 0x124d, 0x1250, 0x1256, 0x1258, 0x1258,
|
|
53
|
+
0x125a, 0x125d, 0x1260, 0x1288, 0x128a, 0x128d, 0x1290, 0x12b0, 0x12b2, 0x12b5, 0x12b8, 0x12be,
|
|
54
|
+
0x12c0, 0x12c0, 0x12c2, 0x12c5, 0x12c8, 0x12d6, 0x12d8, 0x1310, 0x1312, 0x1315, 0x1318, 0x135a,
|
|
55
|
+
0x135d, 0x135f, 0x1380, 0x138f, 0x13a0, 0x13f5, 0x13f8, 0x13fd, 0x1401, 0x166c, 0x166f, 0x167f,
|
|
56
|
+
0x1681, 0x169a, 0x16a0, 0x16ea, 0x16f1, 0x16f8, 0x1700, 0x1715, 0x171f, 0x1734, 0x1740, 0x1753,
|
|
57
|
+
0x1760, 0x176c, 0x176e, 0x1770, 0x1772, 0x1773, 0x1780, 0x17d3, 0x17d7, 0x17d7, 0x17dc, 0x17dd,
|
|
58
|
+
0x180b, 0x180d, 0x180f, 0x180f, 0x1820, 0x1878, 0x1880, 0x18aa, 0x18b0, 0x18f5, 0x1900, 0x191e,
|
|
59
|
+
0x1920, 0x192b, 0x1930, 0x193b, 0x1950, 0x196d, 0x1970, 0x1974, 0x1980, 0x19ab, 0x19b0, 0x19c9,
|
|
60
|
+
0x1a00, 0x1a1b, 0x1a20, 0x1a5e, 0x1a60, 0x1a7c, 0x1a7f, 0x1a7f, 0x1aa7, 0x1aa7, 0x1ab0, 0x1add,
|
|
61
|
+
0x1ae0, 0x1aeb, 0x1b00, 0x1b4c, 0x1b6b, 0x1b73, 0x1b80, 0x1baf, 0x1bba, 0x1bf3, 0x1c00, 0x1c37,
|
|
62
|
+
0x1c4d, 0x1c4f, 0x1c5a, 0x1c7d, 0x1c80, 0x1c8a, 0x1c90, 0x1cba, 0x1cbd, 0x1cbf, 0x1cd0, 0x1cd2,
|
|
63
|
+
0x1cd4, 0x1cfa, 0x1d00, 0x1f15, 0x1f18, 0x1f1d, 0x1f20, 0x1f45, 0x1f48, 0x1f4d, 0x1f50, 0x1f57,
|
|
64
|
+
0x1f59, 0x1f59, 0x1f5b, 0x1f5b, 0x1f5d, 0x1f5d, 0x1f5f, 0x1f7d, 0x1f80, 0x1fb4, 0x1fb6, 0x1fbc,
|
|
65
|
+
0x1fbe, 0x1fbe, 0x1fc2, 0x1fc4, 0x1fc6, 0x1fcc, 0x1fd0, 0x1fd3, 0x1fd6, 0x1fdb, 0x1fe0, 0x1fec,
|
|
66
|
+
0x1ff2, 0x1ff4, 0x1ff6, 0x1ffc, 0x2071, 0x2071, 0x207f, 0x207f, 0x2090, 0x209c, 0x20d0, 0x20f0,
|
|
67
|
+
0x2102, 0x2102, 0x2107, 0x2107, 0x210a, 0x2113, 0x2115, 0x2115, 0x2119, 0x211d, 0x2124, 0x2124,
|
|
68
|
+
0x2126, 0x2126, 0x2128, 0x2128, 0x212a, 0x212d, 0x212f, 0x2139, 0x213c, 0x213f, 0x2145, 0x2149,
|
|
69
|
+
0x214e, 0x214e, 0x2183, 0x2184, 0x2c00, 0x2ce4, 0x2ceb, 0x2cf3, 0x2d00, 0x2d25, 0x2d27, 0x2d27,
|
|
70
|
+
0x2d2d, 0x2d2d, 0x2d30, 0x2d67, 0x2d6f, 0x2d6f, 0x2d7f, 0x2d96, 0x2da0, 0x2da6, 0x2da8, 0x2dae,
|
|
71
|
+
0x2db0, 0x2db6, 0x2db8, 0x2dbe, 0x2dc0, 0x2dc6, 0x2dc8, 0x2dce, 0x2dd0, 0x2dd6, 0x2dd8, 0x2dde,
|
|
72
|
+
0x2de0, 0x2dff, 0x2e2f, 0x2e2f, 0x3005, 0x3006, 0x302a, 0x302f, 0x3031, 0x3035, 0x303b, 0x303c,
|
|
73
|
+
0x3041, 0x3096, 0x3099, 0x309a, 0x309d, 0x309f, 0x30a1, 0x30fa, 0x30fc, 0x30ff, 0x3105, 0x312f,
|
|
74
|
+
0x3131, 0x318e, 0x31a0, 0x31bf, 0x31f0, 0x31ff, 0x3400, 0x4dbf, 0x4e00, 0xa48c, 0xa4d0, 0xa4fd,
|
|
75
|
+
0xa500, 0xa60c, 0xa610, 0xa61f, 0xa62a, 0xa62b, 0xa640, 0xa672, 0xa674, 0xa67d, 0xa67f, 0xa6e5,
|
|
76
|
+
0xa6f0, 0xa6f1, 0xa717, 0xa71f, 0xa722, 0xa788, 0xa78b, 0xa7dc, 0xa7f1, 0xa827, 0xa82c, 0xa82c,
|
|
77
|
+
0xa840, 0xa873, 0xa880, 0xa8c5, 0xa8e0, 0xa8f7, 0xa8fb, 0xa8fb, 0xa8fd, 0xa8ff, 0xa90a, 0xa92d,
|
|
78
|
+
0xa930, 0xa953, 0xa960, 0xa97c, 0xa980, 0xa9c0, 0xa9cf, 0xa9cf, 0xa9e0, 0xa9ef, 0xa9fa, 0xa9fe,
|
|
79
|
+
0xaa00, 0xaa36, 0xaa40, 0xaa4d, 0xaa60, 0xaa76, 0xaa7a, 0xaac2, 0xaadb, 0xaadd, 0xaae0, 0xaaef,
|
|
80
|
+
0xaaf2, 0xaaf6, 0xab01, 0xab06, 0xab09, 0xab0e, 0xab11, 0xab16, 0xab20, 0xab26, 0xab28, 0xab2e,
|
|
81
|
+
0xab30, 0xab5a, 0xab5c, 0xab69, 0xab70, 0xabea, 0xabec, 0xabed, 0xac00, 0xd7a3, 0xd7b0, 0xd7c6,
|
|
82
|
+
0xd7cb, 0xd7fb, 0xf900, 0xfa6d, 0xfa70, 0xfad9, 0xfb00, 0xfb06, 0xfb13, 0xfb17, 0xfb1d, 0xfb28,
|
|
83
|
+
0xfb2a, 0xfb36, 0xfb38, 0xfb3c, 0xfb3e, 0xfb3e, 0xfb40, 0xfb41, 0xfb43, 0xfb44, 0xfb46, 0xfbb1,
|
|
84
|
+
0xfbd3, 0xfd3d, 0xfd50, 0xfd8f, 0xfd92, 0xfdc7, 0xfdf0, 0xfdfb, 0xfe00, 0xfe0f, 0xfe20, 0xfe2f,
|
|
85
|
+
0xfe70, 0xfe74, 0xfe76, 0xfefc, 0xff21, 0xff3a, 0xff41, 0xff5a, 0xff66, 0xffbe, 0xffc2, 0xffc7,
|
|
86
|
+
0xffca, 0xffcf, 0xffd2, 0xffd7, 0xffda, 0xffdc, 0x10000, 0x1000b, 0x1000d, 0x10026, 0x10028, 0x1003a,
|
|
87
|
+
0x1003c, 0x1003d, 0x1003f, 0x1004d, 0x10050, 0x1005d, 0x10080, 0x100fa, 0x101fd, 0x101fd, 0x10280, 0x1029c,
|
|
88
|
+
0x102a0, 0x102d0, 0x102e0, 0x102e0, 0x10300, 0x1031f, 0x1032d, 0x10340, 0x10342, 0x10349, 0x10350, 0x1037a,
|
|
89
|
+
0x10380, 0x1039d, 0x103a0, 0x103c3, 0x103c8, 0x103cf, 0x10400, 0x1049d, 0x104b0, 0x104d3, 0x104d8, 0x104fb,
|
|
90
|
+
0x10500, 0x10527, 0x10530, 0x10563, 0x10570, 0x1057a, 0x1057c, 0x1058a, 0x1058c, 0x10592, 0x10594, 0x10595,
|
|
91
|
+
0x10597, 0x105a1, 0x105a3, 0x105b1, 0x105b3, 0x105b9, 0x105bb, 0x105bc, 0x105c0, 0x105f3, 0x10600, 0x10736,
|
|
92
|
+
0x10740, 0x10755, 0x10760, 0x10767, 0x10780, 0x10785, 0x10787, 0x107b0, 0x107b2, 0x107ba, 0x10800, 0x10805,
|
|
93
|
+
0x10808, 0x10808, 0x1080a, 0x10835, 0x10837, 0x10838, 0x1083c, 0x1083c, 0x1083f, 0x10855, 0x10860, 0x10876,
|
|
94
|
+
0x10880, 0x1089e, 0x108e0, 0x108f2, 0x108f4, 0x108f5, 0x10900, 0x10915, 0x10920, 0x10939, 0x10940, 0x10959,
|
|
95
|
+
0x10980, 0x109b7, 0x109be, 0x109bf, 0x10a00, 0x10a03, 0x10a05, 0x10a06, 0x10a0c, 0x10a13, 0x10a15, 0x10a17,
|
|
96
|
+
0x10a19, 0x10a35, 0x10a38, 0x10a3a, 0x10a3f, 0x10a3f, 0x10a60, 0x10a7c, 0x10a80, 0x10a9c, 0x10ac0, 0x10ac7,
|
|
97
|
+
0x10ac9, 0x10ae6, 0x10b00, 0x10b35, 0x10b40, 0x10b55, 0x10b60, 0x10b72, 0x10b80, 0x10b91, 0x10c00, 0x10c48,
|
|
98
|
+
0x10c80, 0x10cb2, 0x10cc0, 0x10cf2, 0x10d00, 0x10d27, 0x10d4a, 0x10d65, 0x10d69, 0x10d6d, 0x10d6f, 0x10d85,
|
|
99
|
+
0x10e80, 0x10ea9, 0x10eab, 0x10eac, 0x10eb0, 0x10eb1, 0x10ec2, 0x10ec7, 0x10efa, 0x10f1c, 0x10f27, 0x10f27,
|
|
100
|
+
0x10f30, 0x10f50, 0x10f70, 0x10f85, 0x10fb0, 0x10fc4, 0x10fe0, 0x10ff6, 0x11000, 0x11046, 0x11070, 0x11075,
|
|
101
|
+
0x1107f, 0x110ba, 0x110c2, 0x110c2, 0x110d0, 0x110e8, 0x11100, 0x11134, 0x11144, 0x11147, 0x11150, 0x11173,
|
|
102
|
+
0x11176, 0x11176, 0x11180, 0x111c4, 0x111c9, 0x111cc, 0x111ce, 0x111cf, 0x111da, 0x111da, 0x111dc, 0x111dc,
|
|
103
|
+
0x11200, 0x11211, 0x11213, 0x11237, 0x1123e, 0x11241, 0x11280, 0x11286, 0x11288, 0x11288, 0x1128a, 0x1128d,
|
|
104
|
+
0x1128f, 0x1129d, 0x1129f, 0x112a8, 0x112b0, 0x112ea, 0x11300, 0x11303, 0x11305, 0x1130c, 0x1130f, 0x11310,
|
|
105
|
+
0x11313, 0x11328, 0x1132a, 0x11330, 0x11332, 0x11333, 0x11335, 0x11339, 0x1133b, 0x11344, 0x11347, 0x11348,
|
|
106
|
+
0x1134b, 0x1134d, 0x11350, 0x11350, 0x11357, 0x11357, 0x1135d, 0x11363, 0x11366, 0x1136c, 0x11370, 0x11374,
|
|
107
|
+
0x11380, 0x11389, 0x1138b, 0x1138b, 0x1138e, 0x1138e, 0x11390, 0x113b5, 0x113b7, 0x113c0, 0x113c2, 0x113c2,
|
|
108
|
+
0x113c5, 0x113c5, 0x113c7, 0x113ca, 0x113cc, 0x113d3, 0x113e1, 0x113e2, 0x11400, 0x1144a, 0x1145e, 0x11461,
|
|
109
|
+
0x11480, 0x114c5, 0x114c7, 0x114c7, 0x11580, 0x115b5, 0x115b8, 0x115c0, 0x115d8, 0x115dd, 0x11600, 0x11640,
|
|
110
|
+
0x11644, 0x11644, 0x11680, 0x116b8, 0x11700, 0x1171a, 0x1171d, 0x1172b, 0x11740, 0x11746, 0x11800, 0x1183a,
|
|
111
|
+
0x118a0, 0x118df, 0x118ff, 0x11906, 0x11909, 0x11909, 0x1190c, 0x11913, 0x11915, 0x11916, 0x11918, 0x11935,
|
|
112
|
+
0x11937, 0x11938, 0x1193b, 0x11943, 0x119a0, 0x119a7, 0x119aa, 0x119d7, 0x119da, 0x119e1, 0x119e3, 0x119e4,
|
|
113
|
+
0x11a00, 0x11a3e, 0x11a47, 0x11a47, 0x11a50, 0x11a99, 0x11a9d, 0x11a9d, 0x11ab0, 0x11af8, 0x11b60, 0x11b67,
|
|
114
|
+
0x11bc0, 0x11be0, 0x11c00, 0x11c08, 0x11c0a, 0x11c36, 0x11c38, 0x11c40, 0x11c72, 0x11c8f, 0x11c92, 0x11ca7,
|
|
115
|
+
0x11ca9, 0x11cb6, 0x11d00, 0x11d06, 0x11d08, 0x11d09, 0x11d0b, 0x11d36, 0x11d3a, 0x11d3a, 0x11d3c, 0x11d3d,
|
|
116
|
+
0x11d3f, 0x11d47, 0x11d60, 0x11d65, 0x11d67, 0x11d68, 0x11d6a, 0x11d8e, 0x11d90, 0x11d91, 0x11d93, 0x11d98,
|
|
117
|
+
0x11db0, 0x11ddb, 0x11ee0, 0x11ef6, 0x11f00, 0x11f10, 0x11f12, 0x11f3a, 0x11f3e, 0x11f42, 0x11f5a, 0x11f5a,
|
|
118
|
+
0x11fb0, 0x11fb0, 0x12000, 0x12399, 0x12480, 0x12543, 0x12f90, 0x12ff0, 0x13000, 0x1342f, 0x13440, 0x13455,
|
|
119
|
+
0x13460, 0x143fa, 0x14400, 0x14646, 0x16100, 0x1612f, 0x16800, 0x16a38, 0x16a40, 0x16a5e, 0x16a70, 0x16abe,
|
|
120
|
+
0x16ad0, 0x16aed, 0x16af0, 0x16af4, 0x16b00, 0x16b36, 0x16b40, 0x16b43, 0x16b63, 0x16b77, 0x16b7d, 0x16b8f,
|
|
121
|
+
0x16d40, 0x16d6c, 0x16e40, 0x16e7f, 0x16ea0, 0x16eb8, 0x16ebb, 0x16ed3, 0x16f00, 0x16f4a, 0x16f4f, 0x16f87,
|
|
122
|
+
0x16f8f, 0x16f9f, 0x16fe0, 0x16fe1, 0x16fe3, 0x16fe4, 0x16ff0, 0x16ff3, 0x17000, 0x18cd5, 0x18cff, 0x18d1e,
|
|
123
|
+
0x18d80, 0x18df2, 0x1aff0, 0x1aff3, 0x1aff5, 0x1affb, 0x1affd, 0x1affe, 0x1b000, 0x1b122, 0x1b132, 0x1b132,
|
|
124
|
+
0x1b150, 0x1b152, 0x1b155, 0x1b155, 0x1b164, 0x1b167, 0x1b170, 0x1b2fb, 0x1bc00, 0x1bc6a, 0x1bc70, 0x1bc7c,
|
|
125
|
+
0x1bc80, 0x1bc88, 0x1bc90, 0x1bc99, 0x1bc9d, 0x1bc9e, 0x1cf00, 0x1cf2d, 0x1cf30, 0x1cf46, 0x1d165, 0x1d169,
|
|
126
|
+
0x1d16d, 0x1d172, 0x1d17b, 0x1d182, 0x1d185, 0x1d18b, 0x1d1aa, 0x1d1ad, 0x1d242, 0x1d244, 0x1d400, 0x1d454,
|
|
127
|
+
0x1d456, 0x1d49c, 0x1d49e, 0x1d49f, 0x1d4a2, 0x1d4a2, 0x1d4a5, 0x1d4a6, 0x1d4a9, 0x1d4ac, 0x1d4ae, 0x1d4b9,
|
|
128
|
+
0x1d4bb, 0x1d4bb, 0x1d4bd, 0x1d4c3, 0x1d4c5, 0x1d505, 0x1d507, 0x1d50a, 0x1d50d, 0x1d514, 0x1d516, 0x1d51c,
|
|
129
|
+
0x1d51e, 0x1d539, 0x1d53b, 0x1d53e, 0x1d540, 0x1d544, 0x1d546, 0x1d546, 0x1d54a, 0x1d550, 0x1d552, 0x1d6a5,
|
|
130
|
+
0x1d6a8, 0x1d6c0, 0x1d6c2, 0x1d6da, 0x1d6dc, 0x1d6fa, 0x1d6fc, 0x1d714, 0x1d716, 0x1d734, 0x1d736, 0x1d74e,
|
|
131
|
+
0x1d750, 0x1d76e, 0x1d770, 0x1d788, 0x1d78a, 0x1d7a8, 0x1d7aa, 0x1d7c2, 0x1d7c4, 0x1d7cb, 0x1da00, 0x1da36,
|
|
132
|
+
0x1da3b, 0x1da6c, 0x1da75, 0x1da75, 0x1da84, 0x1da84, 0x1da9b, 0x1da9f, 0x1daa1, 0x1daaf, 0x1df00, 0x1df1e,
|
|
133
|
+
0x1df25, 0x1df2a, 0x1e000, 0x1e006, 0x1e008, 0x1e018, 0x1e01b, 0x1e021, 0x1e023, 0x1e024, 0x1e026, 0x1e02a,
|
|
134
|
+
0x1e030, 0x1e06d, 0x1e08f, 0x1e08f, 0x1e100, 0x1e12c, 0x1e130, 0x1e13d, 0x1e14e, 0x1e14e, 0x1e290, 0x1e2ae,
|
|
135
|
+
0x1e2c0, 0x1e2ef, 0x1e4d0, 0x1e4ef, 0x1e5d0, 0x1e5f0, 0x1e6c0, 0x1e6de, 0x1e6e0, 0x1e6f5, 0x1e6fe, 0x1e6ff,
|
|
136
|
+
0x1e7e0, 0x1e7e6, 0x1e7e8, 0x1e7eb, 0x1e7ed, 0x1e7ee, 0x1e7f0, 0x1e7fe, 0x1e800, 0x1e8c4, 0x1e8d0, 0x1e8d6,
|
|
137
|
+
0x1e900, 0x1e94b, 0x1ee00, 0x1ee03, 0x1ee05, 0x1ee1f, 0x1ee21, 0x1ee22, 0x1ee24, 0x1ee24, 0x1ee27, 0x1ee27,
|
|
138
|
+
0x1ee29, 0x1ee32, 0x1ee34, 0x1ee37, 0x1ee39, 0x1ee39, 0x1ee3b, 0x1ee3b, 0x1ee42, 0x1ee42, 0x1ee47, 0x1ee47,
|
|
139
|
+
0x1ee49, 0x1ee49, 0x1ee4b, 0x1ee4b, 0x1ee4d, 0x1ee4f, 0x1ee51, 0x1ee52, 0x1ee54, 0x1ee54, 0x1ee57, 0x1ee57,
|
|
140
|
+
0x1ee59, 0x1ee59, 0x1ee5b, 0x1ee5b, 0x1ee5d, 0x1ee5d, 0x1ee5f, 0x1ee5f, 0x1ee61, 0x1ee62, 0x1ee64, 0x1ee64,
|
|
141
|
+
0x1ee67, 0x1ee6a, 0x1ee6c, 0x1ee72, 0x1ee74, 0x1ee77, 0x1ee79, 0x1ee7c, 0x1ee7e, 0x1ee7e, 0x1ee80, 0x1ee89,
|
|
142
|
+
0x1ee8b, 0x1ee9b, 0x1eea1, 0x1eea3, 0x1eea5, 0x1eea9, 0x1eeab, 0x1eebb, 0x20000, 0x2a6df, 0x2a700, 0x2b81d,
|
|
143
|
+
0x2b820, 0x2cead, 0x2ceb0, 0x2ebe0, 0x2ebf0, 0x2ee5d, 0x2f800, 0x2fa1d, 0x30000, 0x3134a, 0x31350, 0x33479,
|
|
144
|
+
0xe0100, 0xe01ef,
|
|
145
|
+
].freeze
|
|
146
|
+
|
|
147
|
+
# Lu and Lt as inclusive code-point ranges, same flattened shape -- spec 3.1 UPPER, read by
|
|
148
|
+
# nbsp's N7.
|
|
149
|
+
UPPER_RANGES = [
|
|
150
|
+
0x41, 0x5a, 0xc0, 0xd6, 0xd8, 0xde, 0x100, 0x100, 0x102, 0x102, 0x104, 0x104,
|
|
151
|
+
0x106, 0x106, 0x108, 0x108, 0x10a, 0x10a, 0x10c, 0x10c, 0x10e, 0x10e, 0x110, 0x110,
|
|
152
|
+
0x112, 0x112, 0x114, 0x114, 0x116, 0x116, 0x118, 0x118, 0x11a, 0x11a, 0x11c, 0x11c,
|
|
153
|
+
0x11e, 0x11e, 0x120, 0x120, 0x122, 0x122, 0x124, 0x124, 0x126, 0x126, 0x128, 0x128,
|
|
154
|
+
0x12a, 0x12a, 0x12c, 0x12c, 0x12e, 0x12e, 0x130, 0x130, 0x132, 0x132, 0x134, 0x134,
|
|
155
|
+
0x136, 0x136, 0x139, 0x139, 0x13b, 0x13b, 0x13d, 0x13d, 0x13f, 0x13f, 0x141, 0x141,
|
|
156
|
+
0x143, 0x143, 0x145, 0x145, 0x147, 0x147, 0x14a, 0x14a, 0x14c, 0x14c, 0x14e, 0x14e,
|
|
157
|
+
0x150, 0x150, 0x152, 0x152, 0x154, 0x154, 0x156, 0x156, 0x158, 0x158, 0x15a, 0x15a,
|
|
158
|
+
0x15c, 0x15c, 0x15e, 0x15e, 0x160, 0x160, 0x162, 0x162, 0x164, 0x164, 0x166, 0x166,
|
|
159
|
+
0x168, 0x168, 0x16a, 0x16a, 0x16c, 0x16c, 0x16e, 0x16e, 0x170, 0x170, 0x172, 0x172,
|
|
160
|
+
0x174, 0x174, 0x176, 0x176, 0x178, 0x179, 0x17b, 0x17b, 0x17d, 0x17d, 0x181, 0x182,
|
|
161
|
+
0x184, 0x184, 0x186, 0x187, 0x189, 0x18b, 0x18e, 0x191, 0x193, 0x194, 0x196, 0x198,
|
|
162
|
+
0x19c, 0x19d, 0x19f, 0x1a0, 0x1a2, 0x1a2, 0x1a4, 0x1a4, 0x1a6, 0x1a7, 0x1a9, 0x1a9,
|
|
163
|
+
0x1ac, 0x1ac, 0x1ae, 0x1af, 0x1b1, 0x1b3, 0x1b5, 0x1b5, 0x1b7, 0x1b8, 0x1bc, 0x1bc,
|
|
164
|
+
0x1c4, 0x1c5, 0x1c7, 0x1c8, 0x1ca, 0x1cb, 0x1cd, 0x1cd, 0x1cf, 0x1cf, 0x1d1, 0x1d1,
|
|
165
|
+
0x1d3, 0x1d3, 0x1d5, 0x1d5, 0x1d7, 0x1d7, 0x1d9, 0x1d9, 0x1db, 0x1db, 0x1de, 0x1de,
|
|
166
|
+
0x1e0, 0x1e0, 0x1e2, 0x1e2, 0x1e4, 0x1e4, 0x1e6, 0x1e6, 0x1e8, 0x1e8, 0x1ea, 0x1ea,
|
|
167
|
+
0x1ec, 0x1ec, 0x1ee, 0x1ee, 0x1f1, 0x1f2, 0x1f4, 0x1f4, 0x1f6, 0x1f8, 0x1fa, 0x1fa,
|
|
168
|
+
0x1fc, 0x1fc, 0x1fe, 0x1fe, 0x200, 0x200, 0x202, 0x202, 0x204, 0x204, 0x206, 0x206,
|
|
169
|
+
0x208, 0x208, 0x20a, 0x20a, 0x20c, 0x20c, 0x20e, 0x20e, 0x210, 0x210, 0x212, 0x212,
|
|
170
|
+
0x214, 0x214, 0x216, 0x216, 0x218, 0x218, 0x21a, 0x21a, 0x21c, 0x21c, 0x21e, 0x21e,
|
|
171
|
+
0x220, 0x220, 0x222, 0x222, 0x224, 0x224, 0x226, 0x226, 0x228, 0x228, 0x22a, 0x22a,
|
|
172
|
+
0x22c, 0x22c, 0x22e, 0x22e, 0x230, 0x230, 0x232, 0x232, 0x23a, 0x23b, 0x23d, 0x23e,
|
|
173
|
+
0x241, 0x241, 0x243, 0x246, 0x248, 0x248, 0x24a, 0x24a, 0x24c, 0x24c, 0x24e, 0x24e,
|
|
174
|
+
0x370, 0x370, 0x372, 0x372, 0x376, 0x376, 0x37f, 0x37f, 0x386, 0x386, 0x388, 0x38a,
|
|
175
|
+
0x38c, 0x38c, 0x38e, 0x38f, 0x391, 0x3a1, 0x3a3, 0x3ab, 0x3cf, 0x3cf, 0x3d2, 0x3d4,
|
|
176
|
+
0x3d8, 0x3d8, 0x3da, 0x3da, 0x3dc, 0x3dc, 0x3de, 0x3de, 0x3e0, 0x3e0, 0x3e2, 0x3e2,
|
|
177
|
+
0x3e4, 0x3e4, 0x3e6, 0x3e6, 0x3e8, 0x3e8, 0x3ea, 0x3ea, 0x3ec, 0x3ec, 0x3ee, 0x3ee,
|
|
178
|
+
0x3f4, 0x3f4, 0x3f7, 0x3f7, 0x3f9, 0x3fa, 0x3fd, 0x42f, 0x460, 0x460, 0x462, 0x462,
|
|
179
|
+
0x464, 0x464, 0x466, 0x466, 0x468, 0x468, 0x46a, 0x46a, 0x46c, 0x46c, 0x46e, 0x46e,
|
|
180
|
+
0x470, 0x470, 0x472, 0x472, 0x474, 0x474, 0x476, 0x476, 0x478, 0x478, 0x47a, 0x47a,
|
|
181
|
+
0x47c, 0x47c, 0x47e, 0x47e, 0x480, 0x480, 0x48a, 0x48a, 0x48c, 0x48c, 0x48e, 0x48e,
|
|
182
|
+
0x490, 0x490, 0x492, 0x492, 0x494, 0x494, 0x496, 0x496, 0x498, 0x498, 0x49a, 0x49a,
|
|
183
|
+
0x49c, 0x49c, 0x49e, 0x49e, 0x4a0, 0x4a0, 0x4a2, 0x4a2, 0x4a4, 0x4a4, 0x4a6, 0x4a6,
|
|
184
|
+
0x4a8, 0x4a8, 0x4aa, 0x4aa, 0x4ac, 0x4ac, 0x4ae, 0x4ae, 0x4b0, 0x4b0, 0x4b2, 0x4b2,
|
|
185
|
+
0x4b4, 0x4b4, 0x4b6, 0x4b6, 0x4b8, 0x4b8, 0x4ba, 0x4ba, 0x4bc, 0x4bc, 0x4be, 0x4be,
|
|
186
|
+
0x4c0, 0x4c1, 0x4c3, 0x4c3, 0x4c5, 0x4c5, 0x4c7, 0x4c7, 0x4c9, 0x4c9, 0x4cb, 0x4cb,
|
|
187
|
+
0x4cd, 0x4cd, 0x4d0, 0x4d0, 0x4d2, 0x4d2, 0x4d4, 0x4d4, 0x4d6, 0x4d6, 0x4d8, 0x4d8,
|
|
188
|
+
0x4da, 0x4da, 0x4dc, 0x4dc, 0x4de, 0x4de, 0x4e0, 0x4e0, 0x4e2, 0x4e2, 0x4e4, 0x4e4,
|
|
189
|
+
0x4e6, 0x4e6, 0x4e8, 0x4e8, 0x4ea, 0x4ea, 0x4ec, 0x4ec, 0x4ee, 0x4ee, 0x4f0, 0x4f0,
|
|
190
|
+
0x4f2, 0x4f2, 0x4f4, 0x4f4, 0x4f6, 0x4f6, 0x4f8, 0x4f8, 0x4fa, 0x4fa, 0x4fc, 0x4fc,
|
|
191
|
+
0x4fe, 0x4fe, 0x500, 0x500, 0x502, 0x502, 0x504, 0x504, 0x506, 0x506, 0x508, 0x508,
|
|
192
|
+
0x50a, 0x50a, 0x50c, 0x50c, 0x50e, 0x50e, 0x510, 0x510, 0x512, 0x512, 0x514, 0x514,
|
|
193
|
+
0x516, 0x516, 0x518, 0x518, 0x51a, 0x51a, 0x51c, 0x51c, 0x51e, 0x51e, 0x520, 0x520,
|
|
194
|
+
0x522, 0x522, 0x524, 0x524, 0x526, 0x526, 0x528, 0x528, 0x52a, 0x52a, 0x52c, 0x52c,
|
|
195
|
+
0x52e, 0x52e, 0x531, 0x556, 0x10a0, 0x10c5, 0x10c7, 0x10c7, 0x10cd, 0x10cd, 0x13a0, 0x13f5,
|
|
196
|
+
0x1c89, 0x1c89, 0x1c90, 0x1cba, 0x1cbd, 0x1cbf, 0x1e00, 0x1e00, 0x1e02, 0x1e02, 0x1e04, 0x1e04,
|
|
197
|
+
0x1e06, 0x1e06, 0x1e08, 0x1e08, 0x1e0a, 0x1e0a, 0x1e0c, 0x1e0c, 0x1e0e, 0x1e0e, 0x1e10, 0x1e10,
|
|
198
|
+
0x1e12, 0x1e12, 0x1e14, 0x1e14, 0x1e16, 0x1e16, 0x1e18, 0x1e18, 0x1e1a, 0x1e1a, 0x1e1c, 0x1e1c,
|
|
199
|
+
0x1e1e, 0x1e1e, 0x1e20, 0x1e20, 0x1e22, 0x1e22, 0x1e24, 0x1e24, 0x1e26, 0x1e26, 0x1e28, 0x1e28,
|
|
200
|
+
0x1e2a, 0x1e2a, 0x1e2c, 0x1e2c, 0x1e2e, 0x1e2e, 0x1e30, 0x1e30, 0x1e32, 0x1e32, 0x1e34, 0x1e34,
|
|
201
|
+
0x1e36, 0x1e36, 0x1e38, 0x1e38, 0x1e3a, 0x1e3a, 0x1e3c, 0x1e3c, 0x1e3e, 0x1e3e, 0x1e40, 0x1e40,
|
|
202
|
+
0x1e42, 0x1e42, 0x1e44, 0x1e44, 0x1e46, 0x1e46, 0x1e48, 0x1e48, 0x1e4a, 0x1e4a, 0x1e4c, 0x1e4c,
|
|
203
|
+
0x1e4e, 0x1e4e, 0x1e50, 0x1e50, 0x1e52, 0x1e52, 0x1e54, 0x1e54, 0x1e56, 0x1e56, 0x1e58, 0x1e58,
|
|
204
|
+
0x1e5a, 0x1e5a, 0x1e5c, 0x1e5c, 0x1e5e, 0x1e5e, 0x1e60, 0x1e60, 0x1e62, 0x1e62, 0x1e64, 0x1e64,
|
|
205
|
+
0x1e66, 0x1e66, 0x1e68, 0x1e68, 0x1e6a, 0x1e6a, 0x1e6c, 0x1e6c, 0x1e6e, 0x1e6e, 0x1e70, 0x1e70,
|
|
206
|
+
0x1e72, 0x1e72, 0x1e74, 0x1e74, 0x1e76, 0x1e76, 0x1e78, 0x1e78, 0x1e7a, 0x1e7a, 0x1e7c, 0x1e7c,
|
|
207
|
+
0x1e7e, 0x1e7e, 0x1e80, 0x1e80, 0x1e82, 0x1e82, 0x1e84, 0x1e84, 0x1e86, 0x1e86, 0x1e88, 0x1e88,
|
|
208
|
+
0x1e8a, 0x1e8a, 0x1e8c, 0x1e8c, 0x1e8e, 0x1e8e, 0x1e90, 0x1e90, 0x1e92, 0x1e92, 0x1e94, 0x1e94,
|
|
209
|
+
0x1e9e, 0x1e9e, 0x1ea0, 0x1ea0, 0x1ea2, 0x1ea2, 0x1ea4, 0x1ea4, 0x1ea6, 0x1ea6, 0x1ea8, 0x1ea8,
|
|
210
|
+
0x1eaa, 0x1eaa, 0x1eac, 0x1eac, 0x1eae, 0x1eae, 0x1eb0, 0x1eb0, 0x1eb2, 0x1eb2, 0x1eb4, 0x1eb4,
|
|
211
|
+
0x1eb6, 0x1eb6, 0x1eb8, 0x1eb8, 0x1eba, 0x1eba, 0x1ebc, 0x1ebc, 0x1ebe, 0x1ebe, 0x1ec0, 0x1ec0,
|
|
212
|
+
0x1ec2, 0x1ec2, 0x1ec4, 0x1ec4, 0x1ec6, 0x1ec6, 0x1ec8, 0x1ec8, 0x1eca, 0x1eca, 0x1ecc, 0x1ecc,
|
|
213
|
+
0x1ece, 0x1ece, 0x1ed0, 0x1ed0, 0x1ed2, 0x1ed2, 0x1ed4, 0x1ed4, 0x1ed6, 0x1ed6, 0x1ed8, 0x1ed8,
|
|
214
|
+
0x1eda, 0x1eda, 0x1edc, 0x1edc, 0x1ede, 0x1ede, 0x1ee0, 0x1ee0, 0x1ee2, 0x1ee2, 0x1ee4, 0x1ee4,
|
|
215
|
+
0x1ee6, 0x1ee6, 0x1ee8, 0x1ee8, 0x1eea, 0x1eea, 0x1eec, 0x1eec, 0x1eee, 0x1eee, 0x1ef0, 0x1ef0,
|
|
216
|
+
0x1ef2, 0x1ef2, 0x1ef4, 0x1ef4, 0x1ef6, 0x1ef6, 0x1ef8, 0x1ef8, 0x1efa, 0x1efa, 0x1efc, 0x1efc,
|
|
217
|
+
0x1efe, 0x1efe, 0x1f08, 0x1f0f, 0x1f18, 0x1f1d, 0x1f28, 0x1f2f, 0x1f38, 0x1f3f, 0x1f48, 0x1f4d,
|
|
218
|
+
0x1f59, 0x1f59, 0x1f5b, 0x1f5b, 0x1f5d, 0x1f5d, 0x1f5f, 0x1f5f, 0x1f68, 0x1f6f, 0x1f88, 0x1f8f,
|
|
219
|
+
0x1f98, 0x1f9f, 0x1fa8, 0x1faf, 0x1fb8, 0x1fbc, 0x1fc8, 0x1fcc, 0x1fd8, 0x1fdb, 0x1fe8, 0x1fec,
|
|
220
|
+
0x1ff8, 0x1ffc, 0x2102, 0x2102, 0x2107, 0x2107, 0x210b, 0x210d, 0x2110, 0x2112, 0x2115, 0x2115,
|
|
221
|
+
0x2119, 0x211d, 0x2124, 0x2124, 0x2126, 0x2126, 0x2128, 0x2128, 0x212a, 0x212d, 0x2130, 0x2133,
|
|
222
|
+
0x213e, 0x213f, 0x2145, 0x2145, 0x2183, 0x2183, 0x2c00, 0x2c2f, 0x2c60, 0x2c60, 0x2c62, 0x2c64,
|
|
223
|
+
0x2c67, 0x2c67, 0x2c69, 0x2c69, 0x2c6b, 0x2c6b, 0x2c6d, 0x2c70, 0x2c72, 0x2c72, 0x2c75, 0x2c75,
|
|
224
|
+
0x2c7e, 0x2c80, 0x2c82, 0x2c82, 0x2c84, 0x2c84, 0x2c86, 0x2c86, 0x2c88, 0x2c88, 0x2c8a, 0x2c8a,
|
|
225
|
+
0x2c8c, 0x2c8c, 0x2c8e, 0x2c8e, 0x2c90, 0x2c90, 0x2c92, 0x2c92, 0x2c94, 0x2c94, 0x2c96, 0x2c96,
|
|
226
|
+
0x2c98, 0x2c98, 0x2c9a, 0x2c9a, 0x2c9c, 0x2c9c, 0x2c9e, 0x2c9e, 0x2ca0, 0x2ca0, 0x2ca2, 0x2ca2,
|
|
227
|
+
0x2ca4, 0x2ca4, 0x2ca6, 0x2ca6, 0x2ca8, 0x2ca8, 0x2caa, 0x2caa, 0x2cac, 0x2cac, 0x2cae, 0x2cae,
|
|
228
|
+
0x2cb0, 0x2cb0, 0x2cb2, 0x2cb2, 0x2cb4, 0x2cb4, 0x2cb6, 0x2cb6, 0x2cb8, 0x2cb8, 0x2cba, 0x2cba,
|
|
229
|
+
0x2cbc, 0x2cbc, 0x2cbe, 0x2cbe, 0x2cc0, 0x2cc0, 0x2cc2, 0x2cc2, 0x2cc4, 0x2cc4, 0x2cc6, 0x2cc6,
|
|
230
|
+
0x2cc8, 0x2cc8, 0x2cca, 0x2cca, 0x2ccc, 0x2ccc, 0x2cce, 0x2cce, 0x2cd0, 0x2cd0, 0x2cd2, 0x2cd2,
|
|
231
|
+
0x2cd4, 0x2cd4, 0x2cd6, 0x2cd6, 0x2cd8, 0x2cd8, 0x2cda, 0x2cda, 0x2cdc, 0x2cdc, 0x2cde, 0x2cde,
|
|
232
|
+
0x2ce0, 0x2ce0, 0x2ce2, 0x2ce2, 0x2ceb, 0x2ceb, 0x2ced, 0x2ced, 0x2cf2, 0x2cf2, 0xa640, 0xa640,
|
|
233
|
+
0xa642, 0xa642, 0xa644, 0xa644, 0xa646, 0xa646, 0xa648, 0xa648, 0xa64a, 0xa64a, 0xa64c, 0xa64c,
|
|
234
|
+
0xa64e, 0xa64e, 0xa650, 0xa650, 0xa652, 0xa652, 0xa654, 0xa654, 0xa656, 0xa656, 0xa658, 0xa658,
|
|
235
|
+
0xa65a, 0xa65a, 0xa65c, 0xa65c, 0xa65e, 0xa65e, 0xa660, 0xa660, 0xa662, 0xa662, 0xa664, 0xa664,
|
|
236
|
+
0xa666, 0xa666, 0xa668, 0xa668, 0xa66a, 0xa66a, 0xa66c, 0xa66c, 0xa680, 0xa680, 0xa682, 0xa682,
|
|
237
|
+
0xa684, 0xa684, 0xa686, 0xa686, 0xa688, 0xa688, 0xa68a, 0xa68a, 0xa68c, 0xa68c, 0xa68e, 0xa68e,
|
|
238
|
+
0xa690, 0xa690, 0xa692, 0xa692, 0xa694, 0xa694, 0xa696, 0xa696, 0xa698, 0xa698, 0xa69a, 0xa69a,
|
|
239
|
+
0xa722, 0xa722, 0xa724, 0xa724, 0xa726, 0xa726, 0xa728, 0xa728, 0xa72a, 0xa72a, 0xa72c, 0xa72c,
|
|
240
|
+
0xa72e, 0xa72e, 0xa732, 0xa732, 0xa734, 0xa734, 0xa736, 0xa736, 0xa738, 0xa738, 0xa73a, 0xa73a,
|
|
241
|
+
0xa73c, 0xa73c, 0xa73e, 0xa73e, 0xa740, 0xa740, 0xa742, 0xa742, 0xa744, 0xa744, 0xa746, 0xa746,
|
|
242
|
+
0xa748, 0xa748, 0xa74a, 0xa74a, 0xa74c, 0xa74c, 0xa74e, 0xa74e, 0xa750, 0xa750, 0xa752, 0xa752,
|
|
243
|
+
0xa754, 0xa754, 0xa756, 0xa756, 0xa758, 0xa758, 0xa75a, 0xa75a, 0xa75c, 0xa75c, 0xa75e, 0xa75e,
|
|
244
|
+
0xa760, 0xa760, 0xa762, 0xa762, 0xa764, 0xa764, 0xa766, 0xa766, 0xa768, 0xa768, 0xa76a, 0xa76a,
|
|
245
|
+
0xa76c, 0xa76c, 0xa76e, 0xa76e, 0xa779, 0xa779, 0xa77b, 0xa77b, 0xa77d, 0xa77e, 0xa780, 0xa780,
|
|
246
|
+
0xa782, 0xa782, 0xa784, 0xa784, 0xa786, 0xa786, 0xa78b, 0xa78b, 0xa78d, 0xa78d, 0xa790, 0xa790,
|
|
247
|
+
0xa792, 0xa792, 0xa796, 0xa796, 0xa798, 0xa798, 0xa79a, 0xa79a, 0xa79c, 0xa79c, 0xa79e, 0xa79e,
|
|
248
|
+
0xa7a0, 0xa7a0, 0xa7a2, 0xa7a2, 0xa7a4, 0xa7a4, 0xa7a6, 0xa7a6, 0xa7a8, 0xa7a8, 0xa7aa, 0xa7ae,
|
|
249
|
+
0xa7b0, 0xa7b4, 0xa7b6, 0xa7b6, 0xa7b8, 0xa7b8, 0xa7ba, 0xa7ba, 0xa7bc, 0xa7bc, 0xa7be, 0xa7be,
|
|
250
|
+
0xa7c0, 0xa7c0, 0xa7c2, 0xa7c2, 0xa7c4, 0xa7c7, 0xa7c9, 0xa7c9, 0xa7cb, 0xa7cc, 0xa7ce, 0xa7ce,
|
|
251
|
+
0xa7d0, 0xa7d0, 0xa7d2, 0xa7d2, 0xa7d4, 0xa7d4, 0xa7d6, 0xa7d6, 0xa7d8, 0xa7d8, 0xa7da, 0xa7da,
|
|
252
|
+
0xa7dc, 0xa7dc, 0xa7f5, 0xa7f5, 0xff21, 0xff3a, 0x10400, 0x10427, 0x104b0, 0x104d3, 0x10570, 0x1057a,
|
|
253
|
+
0x1057c, 0x1058a, 0x1058c, 0x10592, 0x10594, 0x10595, 0x10c80, 0x10cb2, 0x10d50, 0x10d65, 0x118a0, 0x118bf,
|
|
254
|
+
0x16e40, 0x16e5f, 0x16ea0, 0x16eb8, 0x1d400, 0x1d419, 0x1d434, 0x1d44d, 0x1d468, 0x1d481, 0x1d49c, 0x1d49c,
|
|
255
|
+
0x1d49e, 0x1d49f, 0x1d4a2, 0x1d4a2, 0x1d4a5, 0x1d4a6, 0x1d4a9, 0x1d4ac, 0x1d4ae, 0x1d4b5, 0x1d4d0, 0x1d4e9,
|
|
256
|
+
0x1d504, 0x1d505, 0x1d507, 0x1d50a, 0x1d50d, 0x1d514, 0x1d516, 0x1d51c, 0x1d538, 0x1d539, 0x1d53b, 0x1d53e,
|
|
257
|
+
0x1d540, 0x1d544, 0x1d546, 0x1d546, 0x1d54a, 0x1d550, 0x1d56c, 0x1d585, 0x1d5a0, 0x1d5b9, 0x1d5d4, 0x1d5ed,
|
|
258
|
+
0x1d608, 0x1d621, 0x1d63c, 0x1d655, 0x1d670, 0x1d689, 0x1d6a8, 0x1d6c0, 0x1d6e2, 0x1d6fa, 0x1d71c, 0x1d734,
|
|
259
|
+
0x1d756, 0x1d76e, 0x1d790, 0x1d7a8, 0x1d7ca, 0x1d7ca, 0x1e900, 0x1e921,
|
|
260
|
+
].freeze
|
|
261
|
+
|
|
262
|
+
# Unicode simple uppercase mapping, as [start, end, step, delta] quadruples flattened into
|
|
263
|
+
# one array. Applied to the *pattern*, never to the input, and never via Ruby's own
|
|
264
|
+
# locale-aware casing (ARCHITECTURE.md section 4.4 forbids locale-dependent case
|
|
265
|
+
# conversion). Entries whose full uppercase mapping expands to more than one code point
|
|
266
|
+
# (U+00DF, U+FB00, ...) are absent by construction -- they map to themselves.
|
|
267
|
+
SIMPLE_UPPERCASE = [
|
|
268
|
+
0x61, 0x7a, 0x1, -32, 0xb5, 0xb5, 0x1, 0x2e7, 0xe0, 0xf6, 0x1, -32,
|
|
269
|
+
0xf8, 0xfe, 0x1, -32, 0xff, 0xff, 0x1, 0x79, 0x101, 0x12f, 0x2, -1,
|
|
270
|
+
0x131, 0x131, 0x1, -232, 0x133, 0x137, 0x2, -1, 0x13a, 0x148, 0x2, -1,
|
|
271
|
+
0x14b, 0x177, 0x2, -1, 0x17a, 0x17e, 0x2, -1, 0x17f, 0x17f, 0x1, -300,
|
|
272
|
+
0x180, 0x180, 0x1, 0xc3, 0x183, 0x185, 0x2, -1, 0x188, 0x188, 0x1, -1,
|
|
273
|
+
0x18c, 0x18c, 0x1, -1, 0x192, 0x192, 0x1, -1, 0x195, 0x195, 0x1, 0x61,
|
|
274
|
+
0x199, 0x199, 0x1, -1, 0x19a, 0x19a, 0x1, 0xa3, 0x19b, 0x19b, 0x1, 0xa641,
|
|
275
|
+
0x19e, 0x19e, 0x1, 0x82, 0x1a1, 0x1a5, 0x2, -1, 0x1a8, 0x1a8, 0x1, -1,
|
|
276
|
+
0x1ad, 0x1ad, 0x1, -1, 0x1b0, 0x1b0, 0x1, -1, 0x1b4, 0x1b6, 0x2, -1,
|
|
277
|
+
0x1b9, 0x1b9, 0x1, -1, 0x1bd, 0x1bd, 0x1, -1, 0x1bf, 0x1bf, 0x1, 0x38,
|
|
278
|
+
0x1c5, 0x1c5, 0x1, -1, 0x1c6, 0x1c6, 0x1, -2, 0x1c8, 0x1c8, 0x1, -1,
|
|
279
|
+
0x1c9, 0x1c9, 0x1, -2, 0x1cb, 0x1cb, 0x1, -1, 0x1cc, 0x1cc, 0x1, -2,
|
|
280
|
+
0x1ce, 0x1dc, 0x2, -1, 0x1dd, 0x1dd, 0x1, -79, 0x1df, 0x1ef, 0x2, -1,
|
|
281
|
+
0x1f2, 0x1f2, 0x1, -1, 0x1f3, 0x1f3, 0x1, -2, 0x1f5, 0x1f5, 0x1, -1,
|
|
282
|
+
0x1f9, 0x21f, 0x2, -1, 0x223, 0x233, 0x2, -1, 0x23c, 0x23c, 0x1, -1,
|
|
283
|
+
0x23f, 0x240, 0x1, 0x2a3f, 0x242, 0x242, 0x1, -1, 0x247, 0x24f, 0x2, -1,
|
|
284
|
+
0x250, 0x250, 0x1, 0x2a1f, 0x251, 0x251, 0x1, 0x2a1c, 0x252, 0x252, 0x1, 0x2a1e,
|
|
285
|
+
0x253, 0x253, 0x1, -210, 0x254, 0x254, 0x1, -206, 0x256, 0x257, 0x1, -205,
|
|
286
|
+
0x259, 0x259, 0x1, -202, 0x25b, 0x25b, 0x1, -203, 0x25c, 0x25c, 0x1, 0xa54f,
|
|
287
|
+
0x260, 0x260, 0x1, -205, 0x261, 0x261, 0x1, 0xa54b, 0x263, 0x263, 0x1, -207,
|
|
288
|
+
0x264, 0x264, 0x1, 0xa567, 0x265, 0x265, 0x1, 0xa528, 0x266, 0x266, 0x1, 0xa544,
|
|
289
|
+
0x268, 0x268, 0x1, -209, 0x269, 0x269, 0x1, -211, 0x26a, 0x26a, 0x1, 0xa544,
|
|
290
|
+
0x26b, 0x26b, 0x1, 0x29f7, 0x26c, 0x26c, 0x1, 0xa541, 0x26f, 0x26f, 0x1, -211,
|
|
291
|
+
0x271, 0x271, 0x1, 0x29fd, 0x272, 0x272, 0x1, -213, 0x275, 0x275, 0x1, -214,
|
|
292
|
+
0x27d, 0x27d, 0x1, 0x29e7, 0x280, 0x280, 0x1, -218, 0x282, 0x282, 0x1, 0xa543,
|
|
293
|
+
0x283, 0x283, 0x1, -218, 0x287, 0x287, 0x1, 0xa52a, 0x288, 0x288, 0x1, -218,
|
|
294
|
+
0x289, 0x289, 0x1, -69, 0x28a, 0x28b, 0x1, -217, 0x28c, 0x28c, 0x1, -71,
|
|
295
|
+
0x292, 0x292, 0x1, -219, 0x29d, 0x29d, 0x1, 0xa515, 0x29e, 0x29e, 0x1, 0xa512,
|
|
296
|
+
0x345, 0x345, 0x1, 0x54, 0x371, 0x373, 0x2, -1, 0x377, 0x377, 0x1, -1,
|
|
297
|
+
0x37b, 0x37d, 0x1, 0x82, 0x3ac, 0x3ac, 0x1, -38, 0x3ad, 0x3af, 0x1, -37,
|
|
298
|
+
0x3b1, 0x3c1, 0x1, -32, 0x3c2, 0x3c2, 0x1, -31, 0x3c3, 0x3cb, 0x1, -32,
|
|
299
|
+
0x3cc, 0x3cc, 0x1, -64, 0x3cd, 0x3ce, 0x1, -63, 0x3d0, 0x3d0, 0x1, -62,
|
|
300
|
+
0x3d1, 0x3d1, 0x1, -57, 0x3d5, 0x3d5, 0x1, -47, 0x3d6, 0x3d6, 0x1, -54,
|
|
301
|
+
0x3d7, 0x3d7, 0x1, -8, 0x3d9, 0x3ef, 0x2, -1, 0x3f0, 0x3f0, 0x1, -86,
|
|
302
|
+
0x3f1, 0x3f1, 0x1, -80, 0x3f2, 0x3f2, 0x1, 0x7, 0x3f3, 0x3f3, 0x1, -116,
|
|
303
|
+
0x3f5, 0x3f5, 0x1, -96, 0x3f8, 0x3f8, 0x1, -1, 0x3fb, 0x3fb, 0x1, -1,
|
|
304
|
+
0x430, 0x44f, 0x1, -32, 0x450, 0x45f, 0x1, -80, 0x461, 0x481, 0x2, -1,
|
|
305
|
+
0x48b, 0x4bf, 0x2, -1, 0x4c2, 0x4ce, 0x2, -1, 0x4cf, 0x4cf, 0x1, -15,
|
|
306
|
+
0x4d1, 0x52f, 0x2, -1, 0x561, 0x586, 0x1, -48, 0x10d0, 0x10fa, 0x1, 0xbc0,
|
|
307
|
+
0x10fd, 0x10ff, 0x1, 0xbc0, 0x13f8, 0x13fd, 0x1, -8, 0x1c80, 0x1c80, 0x1, -6254,
|
|
308
|
+
0x1c81, 0x1c81, 0x1, -6253, 0x1c82, 0x1c82, 0x1, -6244, 0x1c83, 0x1c84, 0x1, -6242,
|
|
309
|
+
0x1c85, 0x1c85, 0x1, -6243, 0x1c86, 0x1c86, 0x1, -6236, 0x1c87, 0x1c87, 0x1, -6181,
|
|
310
|
+
0x1c88, 0x1c88, 0x1, 0x89c2, 0x1c8a, 0x1c8a, 0x1, -1, 0x1d79, 0x1d79, 0x1, 0x8a04,
|
|
311
|
+
0x1d7d, 0x1d7d, 0x1, 0xee6, 0x1d8e, 0x1d8e, 0x1, 0x8a38, 0x1e01, 0x1e95, 0x2, -1,
|
|
312
|
+
0x1e9b, 0x1e9b, 0x1, -59, 0x1ea1, 0x1eff, 0x2, -1, 0x1f00, 0x1f07, 0x1, 0x8,
|
|
313
|
+
0x1f10, 0x1f15, 0x1, 0x8, 0x1f20, 0x1f27, 0x1, 0x8, 0x1f30, 0x1f37, 0x1, 0x8,
|
|
314
|
+
0x1f40, 0x1f45, 0x1, 0x8, 0x1f51, 0x1f57, 0x2, 0x8, 0x1f60, 0x1f67, 0x1, 0x8,
|
|
315
|
+
0x1f70, 0x1f71, 0x1, 0x4a, 0x1f72, 0x1f75, 0x1, 0x56, 0x1f76, 0x1f77, 0x1, 0x64,
|
|
316
|
+
0x1f78, 0x1f79, 0x1, 0x80, 0x1f7a, 0x1f7b, 0x1, 0x70, 0x1f7c, 0x1f7d, 0x1, 0x7e,
|
|
317
|
+
0x1fb0, 0x1fb1, 0x1, 0x8, 0x1fbe, 0x1fbe, 0x1, -7205, 0x1fd0, 0x1fd1, 0x1, 0x8,
|
|
318
|
+
0x1fe0, 0x1fe1, 0x1, 0x8, 0x1fe5, 0x1fe5, 0x1, 0x7, 0x214e, 0x214e, 0x1, -28,
|
|
319
|
+
0x2170, 0x217f, 0x1, -16, 0x2184, 0x2184, 0x1, -1, 0x24d0, 0x24e9, 0x1, -26,
|
|
320
|
+
0x2c30, 0x2c5f, 0x1, -48, 0x2c61, 0x2c61, 0x1, -1, 0x2c65, 0x2c65, 0x1, -10795,
|
|
321
|
+
0x2c66, 0x2c66, 0x1, -10792, 0x2c68, 0x2c6c, 0x2, -1, 0x2c73, 0x2c73, 0x1, -1,
|
|
322
|
+
0x2c76, 0x2c76, 0x1, -1, 0x2c81, 0x2ce3, 0x2, -1, 0x2cec, 0x2cee, 0x2, -1,
|
|
323
|
+
0x2cf3, 0x2cf3, 0x1, -1, 0x2d00, 0x2d25, 0x1, -7264, 0x2d27, 0x2d27, 0x1, -7264,
|
|
324
|
+
0x2d2d, 0x2d2d, 0x1, -7264, 0xa641, 0xa66d, 0x2, -1, 0xa681, 0xa69b, 0x2, -1,
|
|
325
|
+
0xa723, 0xa72f, 0x2, -1, 0xa733, 0xa76f, 0x2, -1, 0xa77a, 0xa77c, 0x2, -1,
|
|
326
|
+
0xa77f, 0xa787, 0x2, -1, 0xa78c, 0xa78c, 0x1, -1, 0xa791, 0xa793, 0x2, -1,
|
|
327
|
+
0xa794, 0xa794, 0x1, 0x30, 0xa797, 0xa7a9, 0x2, -1, 0xa7b5, 0xa7c3, 0x2, -1,
|
|
328
|
+
0xa7c8, 0xa7ca, 0x2, -1, 0xa7cd, 0xa7db, 0x2, -1, 0xa7f6, 0xa7f6, 0x1, -1,
|
|
329
|
+
0xab53, 0xab53, 0x1, -928, 0xab70, 0xabbf, 0x1, -38864, 0xff41, 0xff5a, 0x1, -32,
|
|
330
|
+
0x10428, 0x1044f, 0x1, -40, 0x104d8, 0x104fb, 0x1, -40, 0x10597, 0x105bb, 0x2, -39,
|
|
331
|
+
0x10598, 0x105a0, 0x2, -39, 0x105a4, 0x105b0, 0x2, -39, 0x105b4, 0x105b8, 0x2, -39,
|
|
332
|
+
0x105bc, 0x105bc, 0x1, -39, 0x10cc0, 0x10cf2, 0x1, -64, 0x10d70, 0x10d85, 0x1, -32,
|
|
333
|
+
0x118c0, 0x118df, 0x1, -32, 0x16e60, 0x16e7f, 0x1, -32, 0x16ebb, 0x16ed3, 0x1, -27,
|
|
334
|
+
0x1e922, 0x1e943, 0x1, -34,
|
|
335
|
+
].freeze
|
|
336
|
+
|
|
337
|
+
# True for a code point in Lu, Ll, Lt, Lm, Lo, Mn, Mc or Me. Binary search over
|
|
338
|
+
# LETTER_RANGES.
|
|
339
|
+
def self.letter?(cp)
|
|
340
|
+
return false if cp.negative?
|
|
341
|
+
return (cp >= 0x41 && cp <= 0x5a) || (cp >= 0x61 && cp <= 0x7a) if cp < 0x80
|
|
342
|
+
|
|
343
|
+
binary_range_search?(LETTER_RANGES, cp)
|
|
344
|
+
end
|
|
345
|
+
|
|
346
|
+
# True for a code point in Lu or Lt. Binary search over UPPER_RANGES.
|
|
347
|
+
def self.upper?(cp)
|
|
348
|
+
return false if cp.negative?
|
|
349
|
+
return cp >= 0x41 && cp <= 0x5a if cp < 0x80
|
|
350
|
+
|
|
351
|
+
binary_range_search?(UPPER_RANGES, cp)
|
|
352
|
+
end
|
|
353
|
+
|
|
354
|
+
# Simple uppercase mapping of one code point, or the code point itself when it has none.
|
|
355
|
+
def self.simple_uppercase(cp)
|
|
356
|
+
i = 0
|
|
357
|
+
while i < SIMPLE_UPPERCASE.length
|
|
358
|
+
start = SIMPLE_UPPERCASE[i]
|
|
359
|
+
return cp if cp < start
|
|
360
|
+
|
|
361
|
+
stop = SIMPLE_UPPERCASE[i + 1]
|
|
362
|
+
step = SIMPLE_UPPERCASE[i + 2]
|
|
363
|
+
return cp + SIMPLE_UPPERCASE[i + 3] if cp <= stop && ((cp - start) % step).zero?
|
|
364
|
+
|
|
365
|
+
i += 4
|
|
366
|
+
end
|
|
367
|
+
cp
|
|
368
|
+
end
|
|
369
|
+
|
|
370
|
+
def self.binary_range_search?(ranges, cp)
|
|
371
|
+
lo = 0
|
|
372
|
+
hi = (ranges.length / 2) - 1
|
|
373
|
+
while lo <= hi
|
|
374
|
+
mid = (lo + hi) / 2
|
|
375
|
+
start = ranges[mid * 2]
|
|
376
|
+
stop = ranges[(mid * 2) + 1]
|
|
377
|
+
if cp < start
|
|
378
|
+
hi = mid - 1
|
|
379
|
+
elsif cp > stop
|
|
380
|
+
lo = mid + 1
|
|
381
|
+
else
|
|
382
|
+
return true
|
|
383
|
+
end
|
|
384
|
+
end
|
|
385
|
+
false
|
|
386
|
+
end
|
|
387
|
+
private_class_method :binary_range_search?
|
|
388
|
+
end
|
|
389
|
+
end
|
|
390
|
+
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Polytypo
|
|
4
|
+
# The seven stable, cross-runtime error codes (docs/ARCHITECTURE.md section 4.6). Messages are
|
|
5
|
+
# English and are not part of the contract; codes are.
|
|
6
|
+
CODE_UNKNOWN_LOCALE = "POLYTYPO_UNKNOWN_LOCALE"
|
|
7
|
+
CODE_INVALID_MODE = "POLYTYPO_INVALID_MODE"
|
|
8
|
+
CODE_INVALID_DIALECT = "POLYTYPO_INVALID_DIALECT"
|
|
9
|
+
CODE_UNKNOWN_RULE = "POLYTYPO_UNKNOWN_RULE"
|
|
10
|
+
CODE_MALFORMED_LOCALE_DATA = "POLYTYPO_MALFORMED_LOCALE_DATA"
|
|
11
|
+
CODE_RULE_CONTRACT = "POLYTYPO_RULE_CONTRACT"
|
|
12
|
+
CODE_MALFORMED_INPUT = "POLYTYPO_MALFORMED_INPUT"
|
|
13
|
+
|
|
14
|
+
# The only error type Transform ever raises. One class is enough: the stable machine-readable
|
|
15
|
+
# +code+ is the contract, not the exception type or the message text.
|
|
16
|
+
class Error < StandardError
|
|
17
|
+
attr_reader :code
|
|
18
|
+
|
|
19
|
+
def initialize(code, message)
|
|
20
|
+
super(message)
|
|
21
|
+
@code = code
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "spans"
|
|
4
|
+
require_relative "parse_error"
|
|
5
|
+
require_relative "../engine/codepoints"
|
|
6
|
+
|
|
7
|
+
module Polytypo
|
|
8
|
+
module Modes
|
|
9
|
+
# HTML span extraction via a minimal, hand-rolled, position-tracking tokenizer -- not
|
|
10
|
+
# `nokogiri` (ARCHITECTURE.md section 2 names it for Ruby, but verified this session that
|
|
11
|
+
# neither its DOM (`Nokogiri::XML::Text#line`, line number only, no column) nor its SAX
|
|
12
|
+
# callbacks (no position argument at all) give the exact character-level offsets the span
|
|
13
|
+
# model needs). Mirrors what Python's `html.parser` and Go's `x/net/html` low-level
|
|
14
|
+
# `Tokenizer` already do for the same reason: neither is a full HTML5 tree-construction
|
|
15
|
+
# implementation either, and this does not need to be one.
|
|
16
|
+
#
|
|
17
|
+
# Operates directly on Ruby's code-point-indexed String -- no separate byte/char offset
|
|
18
|
+
# conversion is needed here the way Go's port needed one for its byte-oriented tokenizer.
|
|
19
|
+
module Html
|
|
20
|
+
# modes.md 3.6, exhaustive and CLOSED: extending it is a spec change, not an implementation
|
|
21
|
+
# decision. svg/math are here because in MathML a quotation mark, a hyphen and a prime are
|
|
22
|
+
# operators and identifiers -- substituting a curly glyph changes what the expression means.
|
|
23
|
+
SKIPPED_ELEMENTS = %w[code pre kbd samp var script style textarea svg math].freeze
|
|
24
|
+
|
|
25
|
+
# HTML5 void elements: never pushed onto the element stack, since an author is not required
|
|
26
|
+
# to close them and this tokenizer never synthesizes a matching end tag for one.
|
|
27
|
+
VOID_ELEMENTS = %w[area base br col embed hr img input link meta param source track wbr].freeze
|
|
28
|
+
|
|
29
|
+
TAG_NAME_STOP = [">", "/", " ", "\t", "\n", "\r", "\f"].freeze
|
|
30
|
+
|
|
31
|
+
def self.ascii_alpha?(ch)
|
|
32
|
+
(ch >= "a" && ch <= "z") || (ch >= "A" && ch <= "Z")
|
|
33
|
+
end
|
|
34
|
+
private_class_method :ascii_alpha?
|
|
35
|
+
|
|
36
|
+
def self.ascii_alnum?(ch)
|
|
37
|
+
ascii_alpha?(ch) || (ch >= "0" && ch <= "9")
|
|
38
|
+
end
|
|
39
|
+
private_class_method :ascii_alnum?
|
|
40
|
+
|
|
41
|
+
def self.hex_digit?(ch)
|
|
42
|
+
(ch >= "0" && ch <= "9") || (ch >= "a" && ch <= "f") || (ch >= "A" && ch <= "F")
|
|
43
|
+
end
|
|
44
|
+
private_class_method :hex_digit?
|
|
45
|
+
|
|
46
|
+
# Byte ranges [start, end) of every well-formed character reference in chars (a code-point
|
|
47
|
+
# array slice) -- &name;, Ӓ, — -- as spec/rules/modes.md 3.6 requires them
|
|
48
|
+
# treated: opaque units, split out of the surrounding text span so their exact source
|
|
49
|
+
# spelling survives untouched. "Well-formed" is syntactic (shape only, matching the
|
|
50
|
+
# html5lib/Python html.parser precedent this project's other ports follow), not validated
|
|
51
|
+
# against the registered named-character-reference table: a bare & that begins no
|
|
52
|
+
# well-formed reference is deliberately left inside its span (modes.md 3.6: "Tom & Jerry's
|
|
53
|
+
# \"book\"" must not lose the pairing of its quotation marks to a spurious boundary).
|
|
54
|
+
def self.find_entity_refs(chars)
|
|
55
|
+
refs = []
|
|
56
|
+
n = chars.length
|
|
57
|
+
i = 0
|
|
58
|
+
while i < n
|
|
59
|
+
unless chars[i] == "&"
|
|
60
|
+
i += 1
|
|
61
|
+
next
|
|
62
|
+
end
|
|
63
|
+
start = i
|
|
64
|
+
j = i + 1
|
|
65
|
+
if j < n && chars[j] == "#" && j + 1 < n && (chars[j + 1] == "x" || chars[j + 1] == "X")
|
|
66
|
+
k = j + 2
|
|
67
|
+
k += 1 while k < n && hex_digit?(chars[k])
|
|
68
|
+
if k > j + 2 && k < n && chars[k] == ";"
|
|
69
|
+
refs << [start, k + 1]
|
|
70
|
+
i = k + 1
|
|
71
|
+
next
|
|
72
|
+
end
|
|
73
|
+
elsif j < n && chars[j] == "#"
|
|
74
|
+
k = j + 1
|
|
75
|
+
k += 1 while k < n && chars[k] >= "0" && chars[k] <= "9"
|
|
76
|
+
if k > j + 1 && k < n && chars[k] == ";"
|
|
77
|
+
refs << [start, k + 1]
|
|
78
|
+
i = k + 1
|
|
79
|
+
next
|
|
80
|
+
end
|
|
81
|
+
elsif j < n && ascii_alpha?(chars[j])
|
|
82
|
+
k = j + 1
|
|
83
|
+
k += 1 while k < n && ascii_alnum?(chars[k])
|
|
84
|
+
if k < n && chars[k] == ";"
|
|
85
|
+
refs << [start, k + 1]
|
|
86
|
+
i = k + 1
|
|
87
|
+
next
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
i += 1
|
|
91
|
+
end
|
|
92
|
+
refs
|
|
93
|
+
end
|
|
94
|
+
private_class_method :find_entity_refs
|
|
95
|
+
|
|
96
|
+
# Parses "<name ...>", "</name>" or "<name .../>" (as an array of chars) into
|
|
97
|
+
# [name, closing, self_closing]. Comments, declarations and processing instructions
|
|
98
|
+
# ("<!--", "<!", "<?") have no element name and return name == nil.
|
|
99
|
+
def self.read_tag(chars)
|
|
100
|
+
return nil if chars.empty? || chars[0] != "<"
|
|
101
|
+
|
|
102
|
+
i = 1
|
|
103
|
+
closing = i < chars.length && chars[i] == "/"
|
|
104
|
+
i += 1 if closing
|
|
105
|
+
return nil if i >= chars.length || chars[i] == "!" || chars[i] == "?"
|
|
106
|
+
return nil unless ascii_alpha?(chars[i])
|
|
107
|
+
|
|
108
|
+
name_start = i
|
|
109
|
+
i += 1 while i < chars.length && !TAG_NAME_STOP.include?(chars[i])
|
|
110
|
+
name = chars[name_start...i].join.downcase
|
|
111
|
+
self_closing = chars.join.rstrip.end_with?("/>")
|
|
112
|
+
[name, closing, self_closing]
|
|
113
|
+
end
|
|
114
|
+
# Public: also used by Markdown for its inline-raw-HTML tag-stack handling.
|
|
115
|
+
|
|
116
|
+
def self.last_stack_index(stack, tag)
|
|
117
|
+
stack.rindex(tag)
|
|
118
|
+
end
|
|
119
|
+
private_class_method :last_stack_index
|
|
120
|
+
|
|
121
|
+
# Locates the processable spans of an HTML document. The tokenizer is used only to locate
|
|
122
|
+
# spans and is then discarded (modes.md 4: "the document is never serialised") --
|
|
123
|
+
# attributes, tag syntax and entity spelling are never touched.
|
|
124
|
+
def self.html_spans(source)
|
|
125
|
+
cp = Polytypo::Engine::Codepoints.to_codepoints(source)
|
|
126
|
+
chars = cp.map { |c| [c].pack("U") }
|
|
127
|
+
n = chars.length
|
|
128
|
+
|
|
129
|
+
spans = []
|
|
130
|
+
stack = []
|
|
131
|
+
skip_depth = 0
|
|
132
|
+
i = 0
|
|
133
|
+
|
|
134
|
+
ParseError.wrap do
|
|
135
|
+
while i < n
|
|
136
|
+
if chars[i] == "<"
|
|
137
|
+
tag_end = find_tag_end(chars, i)
|
|
138
|
+
if tag_end.nil?
|
|
139
|
+
# A bare "<" with no closing ">" before EOF: the rest of the document is text.
|
|
140
|
+
emit_text(spans, chars, i, n, skip_depth)
|
|
141
|
+
i = n
|
|
142
|
+
next
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
tag_chars = chars[i...tag_end]
|
|
146
|
+
parsed = read_tag(tag_chars)
|
|
147
|
+
if parsed
|
|
148
|
+
name, closing, self_closing = parsed
|
|
149
|
+
if closing
|
|
150
|
+
idx = last_stack_index(stack, name)
|
|
151
|
+
if idx
|
|
152
|
+
popped = stack[idx..]
|
|
153
|
+
stack = stack[0...idx]
|
|
154
|
+
popped.each { |n2| skip_depth -= 1 if SKIPPED_ELEMENTS.include?(n2) }
|
|
155
|
+
end
|
|
156
|
+
elsif self_closing || VOID_ELEMENTS.include?(name)
|
|
157
|
+
# Opens and closes atomically; no persistent stack effect regardless of the
|
|
158
|
+
# tag's identity, since there is no subsequent content inside it to skip.
|
|
159
|
+
else
|
|
160
|
+
stack << name
|
|
161
|
+
skip_depth += 1 if SKIPPED_ELEMENTS.include?(name)
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
# Comments/declarations/processing instructions (parsed.nil?) have no stack effect.
|
|
165
|
+
i = tag_end
|
|
166
|
+
else
|
|
167
|
+
text_end = i
|
|
168
|
+
text_end += 1 while text_end < n && chars[text_end] != "<"
|
|
169
|
+
emit_text(spans, chars, i, text_end, skip_depth)
|
|
170
|
+
i = text_end
|
|
171
|
+
end
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
spans
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
# Finds the index just past the ">" that closes the tag starting at chars[start] == "<",
|
|
179
|
+
# honoring comments ("<!--" ... "-->") and quoted attribute values so a ">" inside either
|
|
180
|
+
# does not end the tag early. Returns nil if unterminated.
|
|
181
|
+
def self.find_tag_end(chars, start)
|
|
182
|
+
n = chars.length
|
|
183
|
+
if chars[start, 4] == ["<", "!", "-", "-"]
|
|
184
|
+
close = index_of_sequence(chars, ["-", "-", ">"], start + 4)
|
|
185
|
+
return close.nil? ? nil : close + 3
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
i = start + 1
|
|
189
|
+
quote = nil
|
|
190
|
+
while i < n
|
|
191
|
+
ch = chars[i]
|
|
192
|
+
if quote
|
|
193
|
+
quote = nil if ch == quote
|
|
194
|
+
elsif ch == "\"" || ch == "'"
|
|
195
|
+
quote = ch
|
|
196
|
+
elsif ch == ">"
|
|
197
|
+
return i + 1
|
|
198
|
+
end
|
|
199
|
+
i += 1
|
|
200
|
+
end
|
|
201
|
+
nil
|
|
202
|
+
end
|
|
203
|
+
private_class_method :find_tag_end
|
|
204
|
+
|
|
205
|
+
def self.index_of_sequence(chars, seq, from)
|
|
206
|
+
n = chars.length
|
|
207
|
+
m = seq.length
|
|
208
|
+
i = from
|
|
209
|
+
while i + m <= n
|
|
210
|
+
return i if chars[i, m] == seq
|
|
211
|
+
|
|
212
|
+
i += 1
|
|
213
|
+
end
|
|
214
|
+
nil
|
|
215
|
+
end
|
|
216
|
+
private_class_method :index_of_sequence
|
|
217
|
+
|
|
218
|
+
def self.emit_text(spans, chars, from, to, skip_depth)
|
|
219
|
+
return if skip_depth != 0 || to <= from
|
|
220
|
+
|
|
221
|
+
slice = chars[from...to]
|
|
222
|
+
refs = find_entity_refs(slice)
|
|
223
|
+
cursor = 0
|
|
224
|
+
refs.each do |ref_start, ref_end|
|
|
225
|
+
spans << Spans::Span.new(from + cursor, from + ref_start) if ref_start > cursor
|
|
226
|
+
cursor = ref_end
|
|
227
|
+
end
|
|
228
|
+
spans << Spans::Span.new(from + cursor, to) if cursor < slice.length
|
|
229
|
+
end
|
|
230
|
+
private_class_method :emit_text
|
|
231
|
+
end
|
|
232
|
+
end
|
|
233
|
+
end
|