polytypo 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/README.md +109 -0
  4. data/lib/polytypo/data/README.md +20 -0
  5. data/lib/polytypo/data/UNICODE +1 -0
  6. data/lib/polytypo/data/VERSION +1 -0
  7. data/lib/polytypo/data/fixtures/de-CH.json +501 -0
  8. data/lib/polytypo/data/fixtures/de-DE.json +547 -0
  9. data/lib/polytypo/data/fixtures/el.json +239 -0
  10. data/lib/polytypo/data/fixtures/en-GB.json +1274 -0
  11. data/lib/polytypo/data/fixtures/en-US.json +1807 -0
  12. data/lib/polytypo/data/fixtures/fi.json +1306 -0
  13. data/lib/polytypo/data/fixtures/fr-CA.json +268 -0
  14. data/lib/polytypo/data/fixtures/fr.json +603 -0
  15. data/lib/polytypo/data/fixtures/locale-resolution.json +209 -0
  16. data/lib/polytypo/data/fixtures/ru.json +688 -0
  17. data/lib/polytypo/data/fixtures/sv.json +1290 -0
  18. data/lib/polytypo/data/locales/de-CH.json +77 -0
  19. data/lib/polytypo/data/locales/de-DE.json +76 -0
  20. data/lib/polytypo/data/locales/el.json +90 -0
  21. data/lib/polytypo/data/locales/en-GB.json +115 -0
  22. data/lib/polytypo/data/locales/en-US.json +133 -0
  23. data/lib/polytypo/data/locales/fi.json +136 -0
  24. data/lib/polytypo/data/locales/fr-CA.json +78 -0
  25. data/lib/polytypo/data/locales/fr.json +84 -0
  26. data/lib/polytypo/data/locales/registry.json +9 -0
  27. data/lib/polytypo/data/locales/ru.json +112 -0
  28. data/lib/polytypo/data/locales/sv.json +124 -0
  29. data/lib/polytypo/data/rules/dashes.md +1238 -0
  30. data/lib/polytypo/data/rules/order.json +78 -0
  31. data/lib/polytypo/data/schema/fixtures.schema.json +79 -0
  32. data/lib/polytypo/data/schema/locale.schema.json +235 -0
  33. data/lib/polytypo/data/schema/registry.schema.json +29 -0
  34. data/lib/polytypo/data/schema/resolution.schema.json +50 -0
  35. data/lib/polytypo/engine/codepoints.rb +24 -0
  36. data/lib/polytypo/engine/edits.rb +64 -0
  37. data/lib/polytypo/engine/locale.rb +138 -0
  38. data/lib/polytypo/engine/pipeline.rb +61 -0
  39. data/lib/polytypo/engine/registry.rb +47 -0
  40. data/lib/polytypo/engine/rules/apostrophe.rb +127 -0
  41. data/lib/polytypo/engine/rules/dash_shared.rb +342 -0
  42. data/lib/polytypo/engine/rules/dashes.rb +125 -0
  43. data/lib/polytypo/engine/rules/ellipsis.rb +100 -0
  44. data/lib/polytypo/engine/rules/hyphen.rb +207 -0
  45. data/lib/polytypo/engine/rules/nbsp.rb +616 -0
  46. data/lib/polytypo/engine/rules/quote_ambiguity.rb +241 -0
  47. data/lib/polytypo/engine/rules/quotes.rb +420 -0
  48. data/lib/polytypo/engine/rules/ranges.rb +124 -0
  49. data/lib/polytypo/engine/rules/spaces.rb +232 -0
  50. data/lib/polytypo/engine/rules/symbols.rb +291 -0
  51. data/lib/polytypo/engine/rules.rb +19 -0
  52. data/lib/polytypo/engine/sentinels.rb +23 -0
  53. data/lib/polytypo/engine/unicode_util.rb +390 -0
  54. data/lib/polytypo/errors.rb +24 -0
  55. data/lib/polytypo/modes/html.rb +233 -0
  56. data/lib/polytypo/modes/markdown.rb +187 -0
  57. data/lib/polytypo/modes/parse_error.rb +19 -0
  58. data/lib/polytypo/modes/runner.rb +57 -0
  59. data/lib/polytypo/modes/spans.rb +132 -0
  60. data/lib/polytypo/version.rb +5 -0
  61. data/lib/polytypo.rb +91 -0
  62. data/polytypo.gemspec +37 -0
  63. metadata +122 -0
@@ -0,0 +1,390 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Polytypo
4
+ module Engine
5
+ # Shared Unicode general-category data for the rule engine, ported verbatim from the JS
6
+ # reference implementation's src/engine/unicode.ts -- a DERIVED artifact generated from the
7
+ # Node/V8 UCD (Unicode 17.0, matching spec/UNICODE), not from Ruby's own Onigmo Unicode
8
+ # tables. Using the same frozen table as JS (rather than a Ruby-native \\p{} Regexp property
9
+ # escape or another gem's own bundled UCD) keeps this port on the exact same Unicode version
10
+ # as the canonical reference implementation, deterministically, regardless of which Onigmo
11
+ # version a given Ruby install ships -- and keeps regex out of core rule logic entirely,
12
+ # consistent with every other port (ARCHITECTURE.md section 4.1 exists for Go's RE2, but the
13
+ # portability argument -- one deterministic table, not five interpreters' own Unicode data --
14
+ # applies here too).
15
+ module UnicodeUtil
16
+ # Lu, Ll, Lt, Lm, Lo, Mn, Mc, Me as inclusive code-point ranges, flattened to
17
+ # [start, end, start, end, ...] and sorted ascending.
18
+ LETTER_RANGES = [
19
+ 0x41, 0x5a, 0x61, 0x7a, 0xaa, 0xaa, 0xb5, 0xb5, 0xba, 0xba, 0xc0, 0xd6,
20
+ 0xd8, 0xf6, 0xf8, 0x2c1, 0x2c6, 0x2d1, 0x2e0, 0x2e4, 0x2ec, 0x2ec, 0x2ee, 0x2ee,
21
+ 0x300, 0x374, 0x376, 0x377, 0x37a, 0x37d, 0x37f, 0x37f, 0x386, 0x386, 0x388, 0x38a,
22
+ 0x38c, 0x38c, 0x38e, 0x3a1, 0x3a3, 0x3f5, 0x3f7, 0x481, 0x483, 0x52f, 0x531, 0x556,
23
+ 0x559, 0x559, 0x560, 0x588, 0x591, 0x5bd, 0x5bf, 0x5bf, 0x5c1, 0x5c2, 0x5c4, 0x5c5,
24
+ 0x5c7, 0x5c7, 0x5d0, 0x5ea, 0x5ef, 0x5f2, 0x610, 0x61a, 0x620, 0x65f, 0x66e, 0x6d3,
25
+ 0x6d5, 0x6dc, 0x6df, 0x6e8, 0x6ea, 0x6ef, 0x6fa, 0x6fc, 0x6ff, 0x6ff, 0x710, 0x74a,
26
+ 0x74d, 0x7b1, 0x7ca, 0x7f5, 0x7fa, 0x7fa, 0x7fd, 0x7fd, 0x800, 0x82d, 0x840, 0x85b,
27
+ 0x860, 0x86a, 0x870, 0x887, 0x889, 0x88f, 0x897, 0x8e1, 0x8e3, 0x963, 0x971, 0x983,
28
+ 0x985, 0x98c, 0x98f, 0x990, 0x993, 0x9a8, 0x9aa, 0x9b0, 0x9b2, 0x9b2, 0x9b6, 0x9b9,
29
+ 0x9bc, 0x9c4, 0x9c7, 0x9c8, 0x9cb, 0x9ce, 0x9d7, 0x9d7, 0x9dc, 0x9dd, 0x9df, 0x9e3,
30
+ 0x9f0, 0x9f1, 0x9fc, 0x9fc, 0x9fe, 0x9fe, 0xa01, 0xa03, 0xa05, 0xa0a, 0xa0f, 0xa10,
31
+ 0xa13, 0xa28, 0xa2a, 0xa30, 0xa32, 0xa33, 0xa35, 0xa36, 0xa38, 0xa39, 0xa3c, 0xa3c,
32
+ 0xa3e, 0xa42, 0xa47, 0xa48, 0xa4b, 0xa4d, 0xa51, 0xa51, 0xa59, 0xa5c, 0xa5e, 0xa5e,
33
+ 0xa70, 0xa75, 0xa81, 0xa83, 0xa85, 0xa8d, 0xa8f, 0xa91, 0xa93, 0xaa8, 0xaaa, 0xab0,
34
+ 0xab2, 0xab3, 0xab5, 0xab9, 0xabc, 0xac5, 0xac7, 0xac9, 0xacb, 0xacd, 0xad0, 0xad0,
35
+ 0xae0, 0xae3, 0xaf9, 0xaff, 0xb01, 0xb03, 0xb05, 0xb0c, 0xb0f, 0xb10, 0xb13, 0xb28,
36
+ 0xb2a, 0xb30, 0xb32, 0xb33, 0xb35, 0xb39, 0xb3c, 0xb44, 0xb47, 0xb48, 0xb4b, 0xb4d,
37
+ 0xb55, 0xb57, 0xb5c, 0xb5d, 0xb5f, 0xb63, 0xb71, 0xb71, 0xb82, 0xb83, 0xb85, 0xb8a,
38
+ 0xb8e, 0xb90, 0xb92, 0xb95, 0xb99, 0xb9a, 0xb9c, 0xb9c, 0xb9e, 0xb9f, 0xba3, 0xba4,
39
+ 0xba8, 0xbaa, 0xbae, 0xbb9, 0xbbe, 0xbc2, 0xbc6, 0xbc8, 0xbca, 0xbcd, 0xbd0, 0xbd0,
40
+ 0xbd7, 0xbd7, 0xc00, 0xc0c, 0xc0e, 0xc10, 0xc12, 0xc28, 0xc2a, 0xc39, 0xc3c, 0xc44,
41
+ 0xc46, 0xc48, 0xc4a, 0xc4d, 0xc55, 0xc56, 0xc58, 0xc5a, 0xc5c, 0xc5d, 0xc60, 0xc63,
42
+ 0xc80, 0xc83, 0xc85, 0xc8c, 0xc8e, 0xc90, 0xc92, 0xca8, 0xcaa, 0xcb3, 0xcb5, 0xcb9,
43
+ 0xcbc, 0xcc4, 0xcc6, 0xcc8, 0xcca, 0xccd, 0xcd5, 0xcd6, 0xcdc, 0xcde, 0xce0, 0xce3,
44
+ 0xcf1, 0xcf3, 0xd00, 0xd0c, 0xd0e, 0xd10, 0xd12, 0xd44, 0xd46, 0xd48, 0xd4a, 0xd4e,
45
+ 0xd54, 0xd57, 0xd5f, 0xd63, 0xd7a, 0xd7f, 0xd81, 0xd83, 0xd85, 0xd96, 0xd9a, 0xdb1,
46
+ 0xdb3, 0xdbb, 0xdbd, 0xdbd, 0xdc0, 0xdc6, 0xdca, 0xdca, 0xdcf, 0xdd4, 0xdd6, 0xdd6,
47
+ 0xdd8, 0xddf, 0xdf2, 0xdf3, 0xe01, 0xe3a, 0xe40, 0xe4e, 0xe81, 0xe82, 0xe84, 0xe84,
48
+ 0xe86, 0xe8a, 0xe8c, 0xea3, 0xea5, 0xea5, 0xea7, 0xebd, 0xec0, 0xec4, 0xec6, 0xec6,
49
+ 0xec8, 0xece, 0xedc, 0xedf, 0xf00, 0xf00, 0xf18, 0xf19, 0xf35, 0xf35, 0xf37, 0xf37,
50
+ 0xf39, 0xf39, 0xf3e, 0xf47, 0xf49, 0xf6c, 0xf71, 0xf84, 0xf86, 0xf97, 0xf99, 0xfbc,
51
+ 0xfc6, 0xfc6, 0x1000, 0x103f, 0x1050, 0x108f, 0x109a, 0x109d, 0x10a0, 0x10c5, 0x10c7, 0x10c7,
52
+ 0x10cd, 0x10cd, 0x10d0, 0x10fa, 0x10fc, 0x1248, 0x124a, 0x124d, 0x1250, 0x1256, 0x1258, 0x1258,
53
+ 0x125a, 0x125d, 0x1260, 0x1288, 0x128a, 0x128d, 0x1290, 0x12b0, 0x12b2, 0x12b5, 0x12b8, 0x12be,
54
+ 0x12c0, 0x12c0, 0x12c2, 0x12c5, 0x12c8, 0x12d6, 0x12d8, 0x1310, 0x1312, 0x1315, 0x1318, 0x135a,
55
+ 0x135d, 0x135f, 0x1380, 0x138f, 0x13a0, 0x13f5, 0x13f8, 0x13fd, 0x1401, 0x166c, 0x166f, 0x167f,
56
+ 0x1681, 0x169a, 0x16a0, 0x16ea, 0x16f1, 0x16f8, 0x1700, 0x1715, 0x171f, 0x1734, 0x1740, 0x1753,
57
+ 0x1760, 0x176c, 0x176e, 0x1770, 0x1772, 0x1773, 0x1780, 0x17d3, 0x17d7, 0x17d7, 0x17dc, 0x17dd,
58
+ 0x180b, 0x180d, 0x180f, 0x180f, 0x1820, 0x1878, 0x1880, 0x18aa, 0x18b0, 0x18f5, 0x1900, 0x191e,
59
+ 0x1920, 0x192b, 0x1930, 0x193b, 0x1950, 0x196d, 0x1970, 0x1974, 0x1980, 0x19ab, 0x19b0, 0x19c9,
60
+ 0x1a00, 0x1a1b, 0x1a20, 0x1a5e, 0x1a60, 0x1a7c, 0x1a7f, 0x1a7f, 0x1aa7, 0x1aa7, 0x1ab0, 0x1add,
61
+ 0x1ae0, 0x1aeb, 0x1b00, 0x1b4c, 0x1b6b, 0x1b73, 0x1b80, 0x1baf, 0x1bba, 0x1bf3, 0x1c00, 0x1c37,
62
+ 0x1c4d, 0x1c4f, 0x1c5a, 0x1c7d, 0x1c80, 0x1c8a, 0x1c90, 0x1cba, 0x1cbd, 0x1cbf, 0x1cd0, 0x1cd2,
63
+ 0x1cd4, 0x1cfa, 0x1d00, 0x1f15, 0x1f18, 0x1f1d, 0x1f20, 0x1f45, 0x1f48, 0x1f4d, 0x1f50, 0x1f57,
64
+ 0x1f59, 0x1f59, 0x1f5b, 0x1f5b, 0x1f5d, 0x1f5d, 0x1f5f, 0x1f7d, 0x1f80, 0x1fb4, 0x1fb6, 0x1fbc,
65
+ 0x1fbe, 0x1fbe, 0x1fc2, 0x1fc4, 0x1fc6, 0x1fcc, 0x1fd0, 0x1fd3, 0x1fd6, 0x1fdb, 0x1fe0, 0x1fec,
66
+ 0x1ff2, 0x1ff4, 0x1ff6, 0x1ffc, 0x2071, 0x2071, 0x207f, 0x207f, 0x2090, 0x209c, 0x20d0, 0x20f0,
67
+ 0x2102, 0x2102, 0x2107, 0x2107, 0x210a, 0x2113, 0x2115, 0x2115, 0x2119, 0x211d, 0x2124, 0x2124,
68
+ 0x2126, 0x2126, 0x2128, 0x2128, 0x212a, 0x212d, 0x212f, 0x2139, 0x213c, 0x213f, 0x2145, 0x2149,
69
+ 0x214e, 0x214e, 0x2183, 0x2184, 0x2c00, 0x2ce4, 0x2ceb, 0x2cf3, 0x2d00, 0x2d25, 0x2d27, 0x2d27,
70
+ 0x2d2d, 0x2d2d, 0x2d30, 0x2d67, 0x2d6f, 0x2d6f, 0x2d7f, 0x2d96, 0x2da0, 0x2da6, 0x2da8, 0x2dae,
71
+ 0x2db0, 0x2db6, 0x2db8, 0x2dbe, 0x2dc0, 0x2dc6, 0x2dc8, 0x2dce, 0x2dd0, 0x2dd6, 0x2dd8, 0x2dde,
72
+ 0x2de0, 0x2dff, 0x2e2f, 0x2e2f, 0x3005, 0x3006, 0x302a, 0x302f, 0x3031, 0x3035, 0x303b, 0x303c,
73
+ 0x3041, 0x3096, 0x3099, 0x309a, 0x309d, 0x309f, 0x30a1, 0x30fa, 0x30fc, 0x30ff, 0x3105, 0x312f,
74
+ 0x3131, 0x318e, 0x31a0, 0x31bf, 0x31f0, 0x31ff, 0x3400, 0x4dbf, 0x4e00, 0xa48c, 0xa4d0, 0xa4fd,
75
+ 0xa500, 0xa60c, 0xa610, 0xa61f, 0xa62a, 0xa62b, 0xa640, 0xa672, 0xa674, 0xa67d, 0xa67f, 0xa6e5,
76
+ 0xa6f0, 0xa6f1, 0xa717, 0xa71f, 0xa722, 0xa788, 0xa78b, 0xa7dc, 0xa7f1, 0xa827, 0xa82c, 0xa82c,
77
+ 0xa840, 0xa873, 0xa880, 0xa8c5, 0xa8e0, 0xa8f7, 0xa8fb, 0xa8fb, 0xa8fd, 0xa8ff, 0xa90a, 0xa92d,
78
+ 0xa930, 0xa953, 0xa960, 0xa97c, 0xa980, 0xa9c0, 0xa9cf, 0xa9cf, 0xa9e0, 0xa9ef, 0xa9fa, 0xa9fe,
79
+ 0xaa00, 0xaa36, 0xaa40, 0xaa4d, 0xaa60, 0xaa76, 0xaa7a, 0xaac2, 0xaadb, 0xaadd, 0xaae0, 0xaaef,
80
+ 0xaaf2, 0xaaf6, 0xab01, 0xab06, 0xab09, 0xab0e, 0xab11, 0xab16, 0xab20, 0xab26, 0xab28, 0xab2e,
81
+ 0xab30, 0xab5a, 0xab5c, 0xab69, 0xab70, 0xabea, 0xabec, 0xabed, 0xac00, 0xd7a3, 0xd7b0, 0xd7c6,
82
+ 0xd7cb, 0xd7fb, 0xf900, 0xfa6d, 0xfa70, 0xfad9, 0xfb00, 0xfb06, 0xfb13, 0xfb17, 0xfb1d, 0xfb28,
83
+ 0xfb2a, 0xfb36, 0xfb38, 0xfb3c, 0xfb3e, 0xfb3e, 0xfb40, 0xfb41, 0xfb43, 0xfb44, 0xfb46, 0xfbb1,
84
+ 0xfbd3, 0xfd3d, 0xfd50, 0xfd8f, 0xfd92, 0xfdc7, 0xfdf0, 0xfdfb, 0xfe00, 0xfe0f, 0xfe20, 0xfe2f,
85
+ 0xfe70, 0xfe74, 0xfe76, 0xfefc, 0xff21, 0xff3a, 0xff41, 0xff5a, 0xff66, 0xffbe, 0xffc2, 0xffc7,
86
+ 0xffca, 0xffcf, 0xffd2, 0xffd7, 0xffda, 0xffdc, 0x10000, 0x1000b, 0x1000d, 0x10026, 0x10028, 0x1003a,
87
+ 0x1003c, 0x1003d, 0x1003f, 0x1004d, 0x10050, 0x1005d, 0x10080, 0x100fa, 0x101fd, 0x101fd, 0x10280, 0x1029c,
88
+ 0x102a0, 0x102d0, 0x102e0, 0x102e0, 0x10300, 0x1031f, 0x1032d, 0x10340, 0x10342, 0x10349, 0x10350, 0x1037a,
89
+ 0x10380, 0x1039d, 0x103a0, 0x103c3, 0x103c8, 0x103cf, 0x10400, 0x1049d, 0x104b0, 0x104d3, 0x104d8, 0x104fb,
90
+ 0x10500, 0x10527, 0x10530, 0x10563, 0x10570, 0x1057a, 0x1057c, 0x1058a, 0x1058c, 0x10592, 0x10594, 0x10595,
91
+ 0x10597, 0x105a1, 0x105a3, 0x105b1, 0x105b3, 0x105b9, 0x105bb, 0x105bc, 0x105c0, 0x105f3, 0x10600, 0x10736,
92
+ 0x10740, 0x10755, 0x10760, 0x10767, 0x10780, 0x10785, 0x10787, 0x107b0, 0x107b2, 0x107ba, 0x10800, 0x10805,
93
+ 0x10808, 0x10808, 0x1080a, 0x10835, 0x10837, 0x10838, 0x1083c, 0x1083c, 0x1083f, 0x10855, 0x10860, 0x10876,
94
+ 0x10880, 0x1089e, 0x108e0, 0x108f2, 0x108f4, 0x108f5, 0x10900, 0x10915, 0x10920, 0x10939, 0x10940, 0x10959,
95
+ 0x10980, 0x109b7, 0x109be, 0x109bf, 0x10a00, 0x10a03, 0x10a05, 0x10a06, 0x10a0c, 0x10a13, 0x10a15, 0x10a17,
96
+ 0x10a19, 0x10a35, 0x10a38, 0x10a3a, 0x10a3f, 0x10a3f, 0x10a60, 0x10a7c, 0x10a80, 0x10a9c, 0x10ac0, 0x10ac7,
97
+ 0x10ac9, 0x10ae6, 0x10b00, 0x10b35, 0x10b40, 0x10b55, 0x10b60, 0x10b72, 0x10b80, 0x10b91, 0x10c00, 0x10c48,
98
+ 0x10c80, 0x10cb2, 0x10cc0, 0x10cf2, 0x10d00, 0x10d27, 0x10d4a, 0x10d65, 0x10d69, 0x10d6d, 0x10d6f, 0x10d85,
99
+ 0x10e80, 0x10ea9, 0x10eab, 0x10eac, 0x10eb0, 0x10eb1, 0x10ec2, 0x10ec7, 0x10efa, 0x10f1c, 0x10f27, 0x10f27,
100
+ 0x10f30, 0x10f50, 0x10f70, 0x10f85, 0x10fb0, 0x10fc4, 0x10fe0, 0x10ff6, 0x11000, 0x11046, 0x11070, 0x11075,
101
+ 0x1107f, 0x110ba, 0x110c2, 0x110c2, 0x110d0, 0x110e8, 0x11100, 0x11134, 0x11144, 0x11147, 0x11150, 0x11173,
102
+ 0x11176, 0x11176, 0x11180, 0x111c4, 0x111c9, 0x111cc, 0x111ce, 0x111cf, 0x111da, 0x111da, 0x111dc, 0x111dc,
103
+ 0x11200, 0x11211, 0x11213, 0x11237, 0x1123e, 0x11241, 0x11280, 0x11286, 0x11288, 0x11288, 0x1128a, 0x1128d,
104
+ 0x1128f, 0x1129d, 0x1129f, 0x112a8, 0x112b0, 0x112ea, 0x11300, 0x11303, 0x11305, 0x1130c, 0x1130f, 0x11310,
105
+ 0x11313, 0x11328, 0x1132a, 0x11330, 0x11332, 0x11333, 0x11335, 0x11339, 0x1133b, 0x11344, 0x11347, 0x11348,
106
+ 0x1134b, 0x1134d, 0x11350, 0x11350, 0x11357, 0x11357, 0x1135d, 0x11363, 0x11366, 0x1136c, 0x11370, 0x11374,
107
+ 0x11380, 0x11389, 0x1138b, 0x1138b, 0x1138e, 0x1138e, 0x11390, 0x113b5, 0x113b7, 0x113c0, 0x113c2, 0x113c2,
108
+ 0x113c5, 0x113c5, 0x113c7, 0x113ca, 0x113cc, 0x113d3, 0x113e1, 0x113e2, 0x11400, 0x1144a, 0x1145e, 0x11461,
109
+ 0x11480, 0x114c5, 0x114c7, 0x114c7, 0x11580, 0x115b5, 0x115b8, 0x115c0, 0x115d8, 0x115dd, 0x11600, 0x11640,
110
+ 0x11644, 0x11644, 0x11680, 0x116b8, 0x11700, 0x1171a, 0x1171d, 0x1172b, 0x11740, 0x11746, 0x11800, 0x1183a,
111
+ 0x118a0, 0x118df, 0x118ff, 0x11906, 0x11909, 0x11909, 0x1190c, 0x11913, 0x11915, 0x11916, 0x11918, 0x11935,
112
+ 0x11937, 0x11938, 0x1193b, 0x11943, 0x119a0, 0x119a7, 0x119aa, 0x119d7, 0x119da, 0x119e1, 0x119e3, 0x119e4,
113
+ 0x11a00, 0x11a3e, 0x11a47, 0x11a47, 0x11a50, 0x11a99, 0x11a9d, 0x11a9d, 0x11ab0, 0x11af8, 0x11b60, 0x11b67,
114
+ 0x11bc0, 0x11be0, 0x11c00, 0x11c08, 0x11c0a, 0x11c36, 0x11c38, 0x11c40, 0x11c72, 0x11c8f, 0x11c92, 0x11ca7,
115
+ 0x11ca9, 0x11cb6, 0x11d00, 0x11d06, 0x11d08, 0x11d09, 0x11d0b, 0x11d36, 0x11d3a, 0x11d3a, 0x11d3c, 0x11d3d,
116
+ 0x11d3f, 0x11d47, 0x11d60, 0x11d65, 0x11d67, 0x11d68, 0x11d6a, 0x11d8e, 0x11d90, 0x11d91, 0x11d93, 0x11d98,
117
+ 0x11db0, 0x11ddb, 0x11ee0, 0x11ef6, 0x11f00, 0x11f10, 0x11f12, 0x11f3a, 0x11f3e, 0x11f42, 0x11f5a, 0x11f5a,
118
+ 0x11fb0, 0x11fb0, 0x12000, 0x12399, 0x12480, 0x12543, 0x12f90, 0x12ff0, 0x13000, 0x1342f, 0x13440, 0x13455,
119
+ 0x13460, 0x143fa, 0x14400, 0x14646, 0x16100, 0x1612f, 0x16800, 0x16a38, 0x16a40, 0x16a5e, 0x16a70, 0x16abe,
120
+ 0x16ad0, 0x16aed, 0x16af0, 0x16af4, 0x16b00, 0x16b36, 0x16b40, 0x16b43, 0x16b63, 0x16b77, 0x16b7d, 0x16b8f,
121
+ 0x16d40, 0x16d6c, 0x16e40, 0x16e7f, 0x16ea0, 0x16eb8, 0x16ebb, 0x16ed3, 0x16f00, 0x16f4a, 0x16f4f, 0x16f87,
122
+ 0x16f8f, 0x16f9f, 0x16fe0, 0x16fe1, 0x16fe3, 0x16fe4, 0x16ff0, 0x16ff3, 0x17000, 0x18cd5, 0x18cff, 0x18d1e,
123
+ 0x18d80, 0x18df2, 0x1aff0, 0x1aff3, 0x1aff5, 0x1affb, 0x1affd, 0x1affe, 0x1b000, 0x1b122, 0x1b132, 0x1b132,
124
+ 0x1b150, 0x1b152, 0x1b155, 0x1b155, 0x1b164, 0x1b167, 0x1b170, 0x1b2fb, 0x1bc00, 0x1bc6a, 0x1bc70, 0x1bc7c,
125
+ 0x1bc80, 0x1bc88, 0x1bc90, 0x1bc99, 0x1bc9d, 0x1bc9e, 0x1cf00, 0x1cf2d, 0x1cf30, 0x1cf46, 0x1d165, 0x1d169,
126
+ 0x1d16d, 0x1d172, 0x1d17b, 0x1d182, 0x1d185, 0x1d18b, 0x1d1aa, 0x1d1ad, 0x1d242, 0x1d244, 0x1d400, 0x1d454,
127
+ 0x1d456, 0x1d49c, 0x1d49e, 0x1d49f, 0x1d4a2, 0x1d4a2, 0x1d4a5, 0x1d4a6, 0x1d4a9, 0x1d4ac, 0x1d4ae, 0x1d4b9,
128
+ 0x1d4bb, 0x1d4bb, 0x1d4bd, 0x1d4c3, 0x1d4c5, 0x1d505, 0x1d507, 0x1d50a, 0x1d50d, 0x1d514, 0x1d516, 0x1d51c,
129
+ 0x1d51e, 0x1d539, 0x1d53b, 0x1d53e, 0x1d540, 0x1d544, 0x1d546, 0x1d546, 0x1d54a, 0x1d550, 0x1d552, 0x1d6a5,
130
+ 0x1d6a8, 0x1d6c0, 0x1d6c2, 0x1d6da, 0x1d6dc, 0x1d6fa, 0x1d6fc, 0x1d714, 0x1d716, 0x1d734, 0x1d736, 0x1d74e,
131
+ 0x1d750, 0x1d76e, 0x1d770, 0x1d788, 0x1d78a, 0x1d7a8, 0x1d7aa, 0x1d7c2, 0x1d7c4, 0x1d7cb, 0x1da00, 0x1da36,
132
+ 0x1da3b, 0x1da6c, 0x1da75, 0x1da75, 0x1da84, 0x1da84, 0x1da9b, 0x1da9f, 0x1daa1, 0x1daaf, 0x1df00, 0x1df1e,
133
+ 0x1df25, 0x1df2a, 0x1e000, 0x1e006, 0x1e008, 0x1e018, 0x1e01b, 0x1e021, 0x1e023, 0x1e024, 0x1e026, 0x1e02a,
134
+ 0x1e030, 0x1e06d, 0x1e08f, 0x1e08f, 0x1e100, 0x1e12c, 0x1e130, 0x1e13d, 0x1e14e, 0x1e14e, 0x1e290, 0x1e2ae,
135
+ 0x1e2c0, 0x1e2ef, 0x1e4d0, 0x1e4ef, 0x1e5d0, 0x1e5f0, 0x1e6c0, 0x1e6de, 0x1e6e0, 0x1e6f5, 0x1e6fe, 0x1e6ff,
136
+ 0x1e7e0, 0x1e7e6, 0x1e7e8, 0x1e7eb, 0x1e7ed, 0x1e7ee, 0x1e7f0, 0x1e7fe, 0x1e800, 0x1e8c4, 0x1e8d0, 0x1e8d6,
137
+ 0x1e900, 0x1e94b, 0x1ee00, 0x1ee03, 0x1ee05, 0x1ee1f, 0x1ee21, 0x1ee22, 0x1ee24, 0x1ee24, 0x1ee27, 0x1ee27,
138
+ 0x1ee29, 0x1ee32, 0x1ee34, 0x1ee37, 0x1ee39, 0x1ee39, 0x1ee3b, 0x1ee3b, 0x1ee42, 0x1ee42, 0x1ee47, 0x1ee47,
139
+ 0x1ee49, 0x1ee49, 0x1ee4b, 0x1ee4b, 0x1ee4d, 0x1ee4f, 0x1ee51, 0x1ee52, 0x1ee54, 0x1ee54, 0x1ee57, 0x1ee57,
140
+ 0x1ee59, 0x1ee59, 0x1ee5b, 0x1ee5b, 0x1ee5d, 0x1ee5d, 0x1ee5f, 0x1ee5f, 0x1ee61, 0x1ee62, 0x1ee64, 0x1ee64,
141
+ 0x1ee67, 0x1ee6a, 0x1ee6c, 0x1ee72, 0x1ee74, 0x1ee77, 0x1ee79, 0x1ee7c, 0x1ee7e, 0x1ee7e, 0x1ee80, 0x1ee89,
142
+ 0x1ee8b, 0x1ee9b, 0x1eea1, 0x1eea3, 0x1eea5, 0x1eea9, 0x1eeab, 0x1eebb, 0x20000, 0x2a6df, 0x2a700, 0x2b81d,
143
+ 0x2b820, 0x2cead, 0x2ceb0, 0x2ebe0, 0x2ebf0, 0x2ee5d, 0x2f800, 0x2fa1d, 0x30000, 0x3134a, 0x31350, 0x33479,
144
+ 0xe0100, 0xe01ef,
145
+ ].freeze
146
+
147
+ # Lu and Lt as inclusive code-point ranges, same flattened shape -- spec 3.1 UPPER, read by
148
+ # nbsp's N7.
149
+ UPPER_RANGES = [
150
+ 0x41, 0x5a, 0xc0, 0xd6, 0xd8, 0xde, 0x100, 0x100, 0x102, 0x102, 0x104, 0x104,
151
+ 0x106, 0x106, 0x108, 0x108, 0x10a, 0x10a, 0x10c, 0x10c, 0x10e, 0x10e, 0x110, 0x110,
152
+ 0x112, 0x112, 0x114, 0x114, 0x116, 0x116, 0x118, 0x118, 0x11a, 0x11a, 0x11c, 0x11c,
153
+ 0x11e, 0x11e, 0x120, 0x120, 0x122, 0x122, 0x124, 0x124, 0x126, 0x126, 0x128, 0x128,
154
+ 0x12a, 0x12a, 0x12c, 0x12c, 0x12e, 0x12e, 0x130, 0x130, 0x132, 0x132, 0x134, 0x134,
155
+ 0x136, 0x136, 0x139, 0x139, 0x13b, 0x13b, 0x13d, 0x13d, 0x13f, 0x13f, 0x141, 0x141,
156
+ 0x143, 0x143, 0x145, 0x145, 0x147, 0x147, 0x14a, 0x14a, 0x14c, 0x14c, 0x14e, 0x14e,
157
+ 0x150, 0x150, 0x152, 0x152, 0x154, 0x154, 0x156, 0x156, 0x158, 0x158, 0x15a, 0x15a,
158
+ 0x15c, 0x15c, 0x15e, 0x15e, 0x160, 0x160, 0x162, 0x162, 0x164, 0x164, 0x166, 0x166,
159
+ 0x168, 0x168, 0x16a, 0x16a, 0x16c, 0x16c, 0x16e, 0x16e, 0x170, 0x170, 0x172, 0x172,
160
+ 0x174, 0x174, 0x176, 0x176, 0x178, 0x179, 0x17b, 0x17b, 0x17d, 0x17d, 0x181, 0x182,
161
+ 0x184, 0x184, 0x186, 0x187, 0x189, 0x18b, 0x18e, 0x191, 0x193, 0x194, 0x196, 0x198,
162
+ 0x19c, 0x19d, 0x19f, 0x1a0, 0x1a2, 0x1a2, 0x1a4, 0x1a4, 0x1a6, 0x1a7, 0x1a9, 0x1a9,
163
+ 0x1ac, 0x1ac, 0x1ae, 0x1af, 0x1b1, 0x1b3, 0x1b5, 0x1b5, 0x1b7, 0x1b8, 0x1bc, 0x1bc,
164
+ 0x1c4, 0x1c5, 0x1c7, 0x1c8, 0x1ca, 0x1cb, 0x1cd, 0x1cd, 0x1cf, 0x1cf, 0x1d1, 0x1d1,
165
+ 0x1d3, 0x1d3, 0x1d5, 0x1d5, 0x1d7, 0x1d7, 0x1d9, 0x1d9, 0x1db, 0x1db, 0x1de, 0x1de,
166
+ 0x1e0, 0x1e0, 0x1e2, 0x1e2, 0x1e4, 0x1e4, 0x1e6, 0x1e6, 0x1e8, 0x1e8, 0x1ea, 0x1ea,
167
+ 0x1ec, 0x1ec, 0x1ee, 0x1ee, 0x1f1, 0x1f2, 0x1f4, 0x1f4, 0x1f6, 0x1f8, 0x1fa, 0x1fa,
168
+ 0x1fc, 0x1fc, 0x1fe, 0x1fe, 0x200, 0x200, 0x202, 0x202, 0x204, 0x204, 0x206, 0x206,
169
+ 0x208, 0x208, 0x20a, 0x20a, 0x20c, 0x20c, 0x20e, 0x20e, 0x210, 0x210, 0x212, 0x212,
170
+ 0x214, 0x214, 0x216, 0x216, 0x218, 0x218, 0x21a, 0x21a, 0x21c, 0x21c, 0x21e, 0x21e,
171
+ 0x220, 0x220, 0x222, 0x222, 0x224, 0x224, 0x226, 0x226, 0x228, 0x228, 0x22a, 0x22a,
172
+ 0x22c, 0x22c, 0x22e, 0x22e, 0x230, 0x230, 0x232, 0x232, 0x23a, 0x23b, 0x23d, 0x23e,
173
+ 0x241, 0x241, 0x243, 0x246, 0x248, 0x248, 0x24a, 0x24a, 0x24c, 0x24c, 0x24e, 0x24e,
174
+ 0x370, 0x370, 0x372, 0x372, 0x376, 0x376, 0x37f, 0x37f, 0x386, 0x386, 0x388, 0x38a,
175
+ 0x38c, 0x38c, 0x38e, 0x38f, 0x391, 0x3a1, 0x3a3, 0x3ab, 0x3cf, 0x3cf, 0x3d2, 0x3d4,
176
+ 0x3d8, 0x3d8, 0x3da, 0x3da, 0x3dc, 0x3dc, 0x3de, 0x3de, 0x3e0, 0x3e0, 0x3e2, 0x3e2,
177
+ 0x3e4, 0x3e4, 0x3e6, 0x3e6, 0x3e8, 0x3e8, 0x3ea, 0x3ea, 0x3ec, 0x3ec, 0x3ee, 0x3ee,
178
+ 0x3f4, 0x3f4, 0x3f7, 0x3f7, 0x3f9, 0x3fa, 0x3fd, 0x42f, 0x460, 0x460, 0x462, 0x462,
179
+ 0x464, 0x464, 0x466, 0x466, 0x468, 0x468, 0x46a, 0x46a, 0x46c, 0x46c, 0x46e, 0x46e,
180
+ 0x470, 0x470, 0x472, 0x472, 0x474, 0x474, 0x476, 0x476, 0x478, 0x478, 0x47a, 0x47a,
181
+ 0x47c, 0x47c, 0x47e, 0x47e, 0x480, 0x480, 0x48a, 0x48a, 0x48c, 0x48c, 0x48e, 0x48e,
182
+ 0x490, 0x490, 0x492, 0x492, 0x494, 0x494, 0x496, 0x496, 0x498, 0x498, 0x49a, 0x49a,
183
+ 0x49c, 0x49c, 0x49e, 0x49e, 0x4a0, 0x4a0, 0x4a2, 0x4a2, 0x4a4, 0x4a4, 0x4a6, 0x4a6,
184
+ 0x4a8, 0x4a8, 0x4aa, 0x4aa, 0x4ac, 0x4ac, 0x4ae, 0x4ae, 0x4b0, 0x4b0, 0x4b2, 0x4b2,
185
+ 0x4b4, 0x4b4, 0x4b6, 0x4b6, 0x4b8, 0x4b8, 0x4ba, 0x4ba, 0x4bc, 0x4bc, 0x4be, 0x4be,
186
+ 0x4c0, 0x4c1, 0x4c3, 0x4c3, 0x4c5, 0x4c5, 0x4c7, 0x4c7, 0x4c9, 0x4c9, 0x4cb, 0x4cb,
187
+ 0x4cd, 0x4cd, 0x4d0, 0x4d0, 0x4d2, 0x4d2, 0x4d4, 0x4d4, 0x4d6, 0x4d6, 0x4d8, 0x4d8,
188
+ 0x4da, 0x4da, 0x4dc, 0x4dc, 0x4de, 0x4de, 0x4e0, 0x4e0, 0x4e2, 0x4e2, 0x4e4, 0x4e4,
189
+ 0x4e6, 0x4e6, 0x4e8, 0x4e8, 0x4ea, 0x4ea, 0x4ec, 0x4ec, 0x4ee, 0x4ee, 0x4f0, 0x4f0,
190
+ 0x4f2, 0x4f2, 0x4f4, 0x4f4, 0x4f6, 0x4f6, 0x4f8, 0x4f8, 0x4fa, 0x4fa, 0x4fc, 0x4fc,
191
+ 0x4fe, 0x4fe, 0x500, 0x500, 0x502, 0x502, 0x504, 0x504, 0x506, 0x506, 0x508, 0x508,
192
+ 0x50a, 0x50a, 0x50c, 0x50c, 0x50e, 0x50e, 0x510, 0x510, 0x512, 0x512, 0x514, 0x514,
193
+ 0x516, 0x516, 0x518, 0x518, 0x51a, 0x51a, 0x51c, 0x51c, 0x51e, 0x51e, 0x520, 0x520,
194
+ 0x522, 0x522, 0x524, 0x524, 0x526, 0x526, 0x528, 0x528, 0x52a, 0x52a, 0x52c, 0x52c,
195
+ 0x52e, 0x52e, 0x531, 0x556, 0x10a0, 0x10c5, 0x10c7, 0x10c7, 0x10cd, 0x10cd, 0x13a0, 0x13f5,
196
+ 0x1c89, 0x1c89, 0x1c90, 0x1cba, 0x1cbd, 0x1cbf, 0x1e00, 0x1e00, 0x1e02, 0x1e02, 0x1e04, 0x1e04,
197
+ 0x1e06, 0x1e06, 0x1e08, 0x1e08, 0x1e0a, 0x1e0a, 0x1e0c, 0x1e0c, 0x1e0e, 0x1e0e, 0x1e10, 0x1e10,
198
+ 0x1e12, 0x1e12, 0x1e14, 0x1e14, 0x1e16, 0x1e16, 0x1e18, 0x1e18, 0x1e1a, 0x1e1a, 0x1e1c, 0x1e1c,
199
+ 0x1e1e, 0x1e1e, 0x1e20, 0x1e20, 0x1e22, 0x1e22, 0x1e24, 0x1e24, 0x1e26, 0x1e26, 0x1e28, 0x1e28,
200
+ 0x1e2a, 0x1e2a, 0x1e2c, 0x1e2c, 0x1e2e, 0x1e2e, 0x1e30, 0x1e30, 0x1e32, 0x1e32, 0x1e34, 0x1e34,
201
+ 0x1e36, 0x1e36, 0x1e38, 0x1e38, 0x1e3a, 0x1e3a, 0x1e3c, 0x1e3c, 0x1e3e, 0x1e3e, 0x1e40, 0x1e40,
202
+ 0x1e42, 0x1e42, 0x1e44, 0x1e44, 0x1e46, 0x1e46, 0x1e48, 0x1e48, 0x1e4a, 0x1e4a, 0x1e4c, 0x1e4c,
203
+ 0x1e4e, 0x1e4e, 0x1e50, 0x1e50, 0x1e52, 0x1e52, 0x1e54, 0x1e54, 0x1e56, 0x1e56, 0x1e58, 0x1e58,
204
+ 0x1e5a, 0x1e5a, 0x1e5c, 0x1e5c, 0x1e5e, 0x1e5e, 0x1e60, 0x1e60, 0x1e62, 0x1e62, 0x1e64, 0x1e64,
205
+ 0x1e66, 0x1e66, 0x1e68, 0x1e68, 0x1e6a, 0x1e6a, 0x1e6c, 0x1e6c, 0x1e6e, 0x1e6e, 0x1e70, 0x1e70,
206
+ 0x1e72, 0x1e72, 0x1e74, 0x1e74, 0x1e76, 0x1e76, 0x1e78, 0x1e78, 0x1e7a, 0x1e7a, 0x1e7c, 0x1e7c,
207
+ 0x1e7e, 0x1e7e, 0x1e80, 0x1e80, 0x1e82, 0x1e82, 0x1e84, 0x1e84, 0x1e86, 0x1e86, 0x1e88, 0x1e88,
208
+ 0x1e8a, 0x1e8a, 0x1e8c, 0x1e8c, 0x1e8e, 0x1e8e, 0x1e90, 0x1e90, 0x1e92, 0x1e92, 0x1e94, 0x1e94,
209
+ 0x1e9e, 0x1e9e, 0x1ea0, 0x1ea0, 0x1ea2, 0x1ea2, 0x1ea4, 0x1ea4, 0x1ea6, 0x1ea6, 0x1ea8, 0x1ea8,
210
+ 0x1eaa, 0x1eaa, 0x1eac, 0x1eac, 0x1eae, 0x1eae, 0x1eb0, 0x1eb0, 0x1eb2, 0x1eb2, 0x1eb4, 0x1eb4,
211
+ 0x1eb6, 0x1eb6, 0x1eb8, 0x1eb8, 0x1eba, 0x1eba, 0x1ebc, 0x1ebc, 0x1ebe, 0x1ebe, 0x1ec0, 0x1ec0,
212
+ 0x1ec2, 0x1ec2, 0x1ec4, 0x1ec4, 0x1ec6, 0x1ec6, 0x1ec8, 0x1ec8, 0x1eca, 0x1eca, 0x1ecc, 0x1ecc,
213
+ 0x1ece, 0x1ece, 0x1ed0, 0x1ed0, 0x1ed2, 0x1ed2, 0x1ed4, 0x1ed4, 0x1ed6, 0x1ed6, 0x1ed8, 0x1ed8,
214
+ 0x1eda, 0x1eda, 0x1edc, 0x1edc, 0x1ede, 0x1ede, 0x1ee0, 0x1ee0, 0x1ee2, 0x1ee2, 0x1ee4, 0x1ee4,
215
+ 0x1ee6, 0x1ee6, 0x1ee8, 0x1ee8, 0x1eea, 0x1eea, 0x1eec, 0x1eec, 0x1eee, 0x1eee, 0x1ef0, 0x1ef0,
216
+ 0x1ef2, 0x1ef2, 0x1ef4, 0x1ef4, 0x1ef6, 0x1ef6, 0x1ef8, 0x1ef8, 0x1efa, 0x1efa, 0x1efc, 0x1efc,
217
+ 0x1efe, 0x1efe, 0x1f08, 0x1f0f, 0x1f18, 0x1f1d, 0x1f28, 0x1f2f, 0x1f38, 0x1f3f, 0x1f48, 0x1f4d,
218
+ 0x1f59, 0x1f59, 0x1f5b, 0x1f5b, 0x1f5d, 0x1f5d, 0x1f5f, 0x1f5f, 0x1f68, 0x1f6f, 0x1f88, 0x1f8f,
219
+ 0x1f98, 0x1f9f, 0x1fa8, 0x1faf, 0x1fb8, 0x1fbc, 0x1fc8, 0x1fcc, 0x1fd8, 0x1fdb, 0x1fe8, 0x1fec,
220
+ 0x1ff8, 0x1ffc, 0x2102, 0x2102, 0x2107, 0x2107, 0x210b, 0x210d, 0x2110, 0x2112, 0x2115, 0x2115,
221
+ 0x2119, 0x211d, 0x2124, 0x2124, 0x2126, 0x2126, 0x2128, 0x2128, 0x212a, 0x212d, 0x2130, 0x2133,
222
+ 0x213e, 0x213f, 0x2145, 0x2145, 0x2183, 0x2183, 0x2c00, 0x2c2f, 0x2c60, 0x2c60, 0x2c62, 0x2c64,
223
+ 0x2c67, 0x2c67, 0x2c69, 0x2c69, 0x2c6b, 0x2c6b, 0x2c6d, 0x2c70, 0x2c72, 0x2c72, 0x2c75, 0x2c75,
224
+ 0x2c7e, 0x2c80, 0x2c82, 0x2c82, 0x2c84, 0x2c84, 0x2c86, 0x2c86, 0x2c88, 0x2c88, 0x2c8a, 0x2c8a,
225
+ 0x2c8c, 0x2c8c, 0x2c8e, 0x2c8e, 0x2c90, 0x2c90, 0x2c92, 0x2c92, 0x2c94, 0x2c94, 0x2c96, 0x2c96,
226
+ 0x2c98, 0x2c98, 0x2c9a, 0x2c9a, 0x2c9c, 0x2c9c, 0x2c9e, 0x2c9e, 0x2ca0, 0x2ca0, 0x2ca2, 0x2ca2,
227
+ 0x2ca4, 0x2ca4, 0x2ca6, 0x2ca6, 0x2ca8, 0x2ca8, 0x2caa, 0x2caa, 0x2cac, 0x2cac, 0x2cae, 0x2cae,
228
+ 0x2cb0, 0x2cb0, 0x2cb2, 0x2cb2, 0x2cb4, 0x2cb4, 0x2cb6, 0x2cb6, 0x2cb8, 0x2cb8, 0x2cba, 0x2cba,
229
+ 0x2cbc, 0x2cbc, 0x2cbe, 0x2cbe, 0x2cc0, 0x2cc0, 0x2cc2, 0x2cc2, 0x2cc4, 0x2cc4, 0x2cc6, 0x2cc6,
230
+ 0x2cc8, 0x2cc8, 0x2cca, 0x2cca, 0x2ccc, 0x2ccc, 0x2cce, 0x2cce, 0x2cd0, 0x2cd0, 0x2cd2, 0x2cd2,
231
+ 0x2cd4, 0x2cd4, 0x2cd6, 0x2cd6, 0x2cd8, 0x2cd8, 0x2cda, 0x2cda, 0x2cdc, 0x2cdc, 0x2cde, 0x2cde,
232
+ 0x2ce0, 0x2ce0, 0x2ce2, 0x2ce2, 0x2ceb, 0x2ceb, 0x2ced, 0x2ced, 0x2cf2, 0x2cf2, 0xa640, 0xa640,
233
+ 0xa642, 0xa642, 0xa644, 0xa644, 0xa646, 0xa646, 0xa648, 0xa648, 0xa64a, 0xa64a, 0xa64c, 0xa64c,
234
+ 0xa64e, 0xa64e, 0xa650, 0xa650, 0xa652, 0xa652, 0xa654, 0xa654, 0xa656, 0xa656, 0xa658, 0xa658,
235
+ 0xa65a, 0xa65a, 0xa65c, 0xa65c, 0xa65e, 0xa65e, 0xa660, 0xa660, 0xa662, 0xa662, 0xa664, 0xa664,
236
+ 0xa666, 0xa666, 0xa668, 0xa668, 0xa66a, 0xa66a, 0xa66c, 0xa66c, 0xa680, 0xa680, 0xa682, 0xa682,
237
+ 0xa684, 0xa684, 0xa686, 0xa686, 0xa688, 0xa688, 0xa68a, 0xa68a, 0xa68c, 0xa68c, 0xa68e, 0xa68e,
238
+ 0xa690, 0xa690, 0xa692, 0xa692, 0xa694, 0xa694, 0xa696, 0xa696, 0xa698, 0xa698, 0xa69a, 0xa69a,
239
+ 0xa722, 0xa722, 0xa724, 0xa724, 0xa726, 0xa726, 0xa728, 0xa728, 0xa72a, 0xa72a, 0xa72c, 0xa72c,
240
+ 0xa72e, 0xa72e, 0xa732, 0xa732, 0xa734, 0xa734, 0xa736, 0xa736, 0xa738, 0xa738, 0xa73a, 0xa73a,
241
+ 0xa73c, 0xa73c, 0xa73e, 0xa73e, 0xa740, 0xa740, 0xa742, 0xa742, 0xa744, 0xa744, 0xa746, 0xa746,
242
+ 0xa748, 0xa748, 0xa74a, 0xa74a, 0xa74c, 0xa74c, 0xa74e, 0xa74e, 0xa750, 0xa750, 0xa752, 0xa752,
243
+ 0xa754, 0xa754, 0xa756, 0xa756, 0xa758, 0xa758, 0xa75a, 0xa75a, 0xa75c, 0xa75c, 0xa75e, 0xa75e,
244
+ 0xa760, 0xa760, 0xa762, 0xa762, 0xa764, 0xa764, 0xa766, 0xa766, 0xa768, 0xa768, 0xa76a, 0xa76a,
245
+ 0xa76c, 0xa76c, 0xa76e, 0xa76e, 0xa779, 0xa779, 0xa77b, 0xa77b, 0xa77d, 0xa77e, 0xa780, 0xa780,
246
+ 0xa782, 0xa782, 0xa784, 0xa784, 0xa786, 0xa786, 0xa78b, 0xa78b, 0xa78d, 0xa78d, 0xa790, 0xa790,
247
+ 0xa792, 0xa792, 0xa796, 0xa796, 0xa798, 0xa798, 0xa79a, 0xa79a, 0xa79c, 0xa79c, 0xa79e, 0xa79e,
248
+ 0xa7a0, 0xa7a0, 0xa7a2, 0xa7a2, 0xa7a4, 0xa7a4, 0xa7a6, 0xa7a6, 0xa7a8, 0xa7a8, 0xa7aa, 0xa7ae,
249
+ 0xa7b0, 0xa7b4, 0xa7b6, 0xa7b6, 0xa7b8, 0xa7b8, 0xa7ba, 0xa7ba, 0xa7bc, 0xa7bc, 0xa7be, 0xa7be,
250
+ 0xa7c0, 0xa7c0, 0xa7c2, 0xa7c2, 0xa7c4, 0xa7c7, 0xa7c9, 0xa7c9, 0xa7cb, 0xa7cc, 0xa7ce, 0xa7ce,
251
+ 0xa7d0, 0xa7d0, 0xa7d2, 0xa7d2, 0xa7d4, 0xa7d4, 0xa7d6, 0xa7d6, 0xa7d8, 0xa7d8, 0xa7da, 0xa7da,
252
+ 0xa7dc, 0xa7dc, 0xa7f5, 0xa7f5, 0xff21, 0xff3a, 0x10400, 0x10427, 0x104b0, 0x104d3, 0x10570, 0x1057a,
253
+ 0x1057c, 0x1058a, 0x1058c, 0x10592, 0x10594, 0x10595, 0x10c80, 0x10cb2, 0x10d50, 0x10d65, 0x118a0, 0x118bf,
254
+ 0x16e40, 0x16e5f, 0x16ea0, 0x16eb8, 0x1d400, 0x1d419, 0x1d434, 0x1d44d, 0x1d468, 0x1d481, 0x1d49c, 0x1d49c,
255
+ 0x1d49e, 0x1d49f, 0x1d4a2, 0x1d4a2, 0x1d4a5, 0x1d4a6, 0x1d4a9, 0x1d4ac, 0x1d4ae, 0x1d4b5, 0x1d4d0, 0x1d4e9,
256
+ 0x1d504, 0x1d505, 0x1d507, 0x1d50a, 0x1d50d, 0x1d514, 0x1d516, 0x1d51c, 0x1d538, 0x1d539, 0x1d53b, 0x1d53e,
257
+ 0x1d540, 0x1d544, 0x1d546, 0x1d546, 0x1d54a, 0x1d550, 0x1d56c, 0x1d585, 0x1d5a0, 0x1d5b9, 0x1d5d4, 0x1d5ed,
258
+ 0x1d608, 0x1d621, 0x1d63c, 0x1d655, 0x1d670, 0x1d689, 0x1d6a8, 0x1d6c0, 0x1d6e2, 0x1d6fa, 0x1d71c, 0x1d734,
259
+ 0x1d756, 0x1d76e, 0x1d790, 0x1d7a8, 0x1d7ca, 0x1d7ca, 0x1e900, 0x1e921,
260
+ ].freeze
261
+
262
+ # Unicode simple uppercase mapping, as [start, end, step, delta] quadruples flattened into
263
+ # one array. Applied to the *pattern*, never to the input, and never via Ruby's own
264
+ # locale-aware casing (ARCHITECTURE.md section 4.4 forbids locale-dependent case
265
+ # conversion). Entries whose full uppercase mapping expands to more than one code point
266
+ # (U+00DF, U+FB00, ...) are absent by construction -- they map to themselves.
267
+ SIMPLE_UPPERCASE = [
268
+ 0x61, 0x7a, 0x1, -32, 0xb5, 0xb5, 0x1, 0x2e7, 0xe0, 0xf6, 0x1, -32,
269
+ 0xf8, 0xfe, 0x1, -32, 0xff, 0xff, 0x1, 0x79, 0x101, 0x12f, 0x2, -1,
270
+ 0x131, 0x131, 0x1, -232, 0x133, 0x137, 0x2, -1, 0x13a, 0x148, 0x2, -1,
271
+ 0x14b, 0x177, 0x2, -1, 0x17a, 0x17e, 0x2, -1, 0x17f, 0x17f, 0x1, -300,
272
+ 0x180, 0x180, 0x1, 0xc3, 0x183, 0x185, 0x2, -1, 0x188, 0x188, 0x1, -1,
273
+ 0x18c, 0x18c, 0x1, -1, 0x192, 0x192, 0x1, -1, 0x195, 0x195, 0x1, 0x61,
274
+ 0x199, 0x199, 0x1, -1, 0x19a, 0x19a, 0x1, 0xa3, 0x19b, 0x19b, 0x1, 0xa641,
275
+ 0x19e, 0x19e, 0x1, 0x82, 0x1a1, 0x1a5, 0x2, -1, 0x1a8, 0x1a8, 0x1, -1,
276
+ 0x1ad, 0x1ad, 0x1, -1, 0x1b0, 0x1b0, 0x1, -1, 0x1b4, 0x1b6, 0x2, -1,
277
+ 0x1b9, 0x1b9, 0x1, -1, 0x1bd, 0x1bd, 0x1, -1, 0x1bf, 0x1bf, 0x1, 0x38,
278
+ 0x1c5, 0x1c5, 0x1, -1, 0x1c6, 0x1c6, 0x1, -2, 0x1c8, 0x1c8, 0x1, -1,
279
+ 0x1c9, 0x1c9, 0x1, -2, 0x1cb, 0x1cb, 0x1, -1, 0x1cc, 0x1cc, 0x1, -2,
280
+ 0x1ce, 0x1dc, 0x2, -1, 0x1dd, 0x1dd, 0x1, -79, 0x1df, 0x1ef, 0x2, -1,
281
+ 0x1f2, 0x1f2, 0x1, -1, 0x1f3, 0x1f3, 0x1, -2, 0x1f5, 0x1f5, 0x1, -1,
282
+ 0x1f9, 0x21f, 0x2, -1, 0x223, 0x233, 0x2, -1, 0x23c, 0x23c, 0x1, -1,
283
+ 0x23f, 0x240, 0x1, 0x2a3f, 0x242, 0x242, 0x1, -1, 0x247, 0x24f, 0x2, -1,
284
+ 0x250, 0x250, 0x1, 0x2a1f, 0x251, 0x251, 0x1, 0x2a1c, 0x252, 0x252, 0x1, 0x2a1e,
285
+ 0x253, 0x253, 0x1, -210, 0x254, 0x254, 0x1, -206, 0x256, 0x257, 0x1, -205,
286
+ 0x259, 0x259, 0x1, -202, 0x25b, 0x25b, 0x1, -203, 0x25c, 0x25c, 0x1, 0xa54f,
287
+ 0x260, 0x260, 0x1, -205, 0x261, 0x261, 0x1, 0xa54b, 0x263, 0x263, 0x1, -207,
288
+ 0x264, 0x264, 0x1, 0xa567, 0x265, 0x265, 0x1, 0xa528, 0x266, 0x266, 0x1, 0xa544,
289
+ 0x268, 0x268, 0x1, -209, 0x269, 0x269, 0x1, -211, 0x26a, 0x26a, 0x1, 0xa544,
290
+ 0x26b, 0x26b, 0x1, 0x29f7, 0x26c, 0x26c, 0x1, 0xa541, 0x26f, 0x26f, 0x1, -211,
291
+ 0x271, 0x271, 0x1, 0x29fd, 0x272, 0x272, 0x1, -213, 0x275, 0x275, 0x1, -214,
292
+ 0x27d, 0x27d, 0x1, 0x29e7, 0x280, 0x280, 0x1, -218, 0x282, 0x282, 0x1, 0xa543,
293
+ 0x283, 0x283, 0x1, -218, 0x287, 0x287, 0x1, 0xa52a, 0x288, 0x288, 0x1, -218,
294
+ 0x289, 0x289, 0x1, -69, 0x28a, 0x28b, 0x1, -217, 0x28c, 0x28c, 0x1, -71,
295
+ 0x292, 0x292, 0x1, -219, 0x29d, 0x29d, 0x1, 0xa515, 0x29e, 0x29e, 0x1, 0xa512,
296
+ 0x345, 0x345, 0x1, 0x54, 0x371, 0x373, 0x2, -1, 0x377, 0x377, 0x1, -1,
297
+ 0x37b, 0x37d, 0x1, 0x82, 0x3ac, 0x3ac, 0x1, -38, 0x3ad, 0x3af, 0x1, -37,
298
+ 0x3b1, 0x3c1, 0x1, -32, 0x3c2, 0x3c2, 0x1, -31, 0x3c3, 0x3cb, 0x1, -32,
299
+ 0x3cc, 0x3cc, 0x1, -64, 0x3cd, 0x3ce, 0x1, -63, 0x3d0, 0x3d0, 0x1, -62,
300
+ 0x3d1, 0x3d1, 0x1, -57, 0x3d5, 0x3d5, 0x1, -47, 0x3d6, 0x3d6, 0x1, -54,
301
+ 0x3d7, 0x3d7, 0x1, -8, 0x3d9, 0x3ef, 0x2, -1, 0x3f0, 0x3f0, 0x1, -86,
302
+ 0x3f1, 0x3f1, 0x1, -80, 0x3f2, 0x3f2, 0x1, 0x7, 0x3f3, 0x3f3, 0x1, -116,
303
+ 0x3f5, 0x3f5, 0x1, -96, 0x3f8, 0x3f8, 0x1, -1, 0x3fb, 0x3fb, 0x1, -1,
304
+ 0x430, 0x44f, 0x1, -32, 0x450, 0x45f, 0x1, -80, 0x461, 0x481, 0x2, -1,
305
+ 0x48b, 0x4bf, 0x2, -1, 0x4c2, 0x4ce, 0x2, -1, 0x4cf, 0x4cf, 0x1, -15,
306
+ 0x4d1, 0x52f, 0x2, -1, 0x561, 0x586, 0x1, -48, 0x10d0, 0x10fa, 0x1, 0xbc0,
307
+ 0x10fd, 0x10ff, 0x1, 0xbc0, 0x13f8, 0x13fd, 0x1, -8, 0x1c80, 0x1c80, 0x1, -6254,
308
+ 0x1c81, 0x1c81, 0x1, -6253, 0x1c82, 0x1c82, 0x1, -6244, 0x1c83, 0x1c84, 0x1, -6242,
309
+ 0x1c85, 0x1c85, 0x1, -6243, 0x1c86, 0x1c86, 0x1, -6236, 0x1c87, 0x1c87, 0x1, -6181,
310
+ 0x1c88, 0x1c88, 0x1, 0x89c2, 0x1c8a, 0x1c8a, 0x1, -1, 0x1d79, 0x1d79, 0x1, 0x8a04,
311
+ 0x1d7d, 0x1d7d, 0x1, 0xee6, 0x1d8e, 0x1d8e, 0x1, 0x8a38, 0x1e01, 0x1e95, 0x2, -1,
312
+ 0x1e9b, 0x1e9b, 0x1, -59, 0x1ea1, 0x1eff, 0x2, -1, 0x1f00, 0x1f07, 0x1, 0x8,
313
+ 0x1f10, 0x1f15, 0x1, 0x8, 0x1f20, 0x1f27, 0x1, 0x8, 0x1f30, 0x1f37, 0x1, 0x8,
314
+ 0x1f40, 0x1f45, 0x1, 0x8, 0x1f51, 0x1f57, 0x2, 0x8, 0x1f60, 0x1f67, 0x1, 0x8,
315
+ 0x1f70, 0x1f71, 0x1, 0x4a, 0x1f72, 0x1f75, 0x1, 0x56, 0x1f76, 0x1f77, 0x1, 0x64,
316
+ 0x1f78, 0x1f79, 0x1, 0x80, 0x1f7a, 0x1f7b, 0x1, 0x70, 0x1f7c, 0x1f7d, 0x1, 0x7e,
317
+ 0x1fb0, 0x1fb1, 0x1, 0x8, 0x1fbe, 0x1fbe, 0x1, -7205, 0x1fd0, 0x1fd1, 0x1, 0x8,
318
+ 0x1fe0, 0x1fe1, 0x1, 0x8, 0x1fe5, 0x1fe5, 0x1, 0x7, 0x214e, 0x214e, 0x1, -28,
319
+ 0x2170, 0x217f, 0x1, -16, 0x2184, 0x2184, 0x1, -1, 0x24d0, 0x24e9, 0x1, -26,
320
+ 0x2c30, 0x2c5f, 0x1, -48, 0x2c61, 0x2c61, 0x1, -1, 0x2c65, 0x2c65, 0x1, -10795,
321
+ 0x2c66, 0x2c66, 0x1, -10792, 0x2c68, 0x2c6c, 0x2, -1, 0x2c73, 0x2c73, 0x1, -1,
322
+ 0x2c76, 0x2c76, 0x1, -1, 0x2c81, 0x2ce3, 0x2, -1, 0x2cec, 0x2cee, 0x2, -1,
323
+ 0x2cf3, 0x2cf3, 0x1, -1, 0x2d00, 0x2d25, 0x1, -7264, 0x2d27, 0x2d27, 0x1, -7264,
324
+ 0x2d2d, 0x2d2d, 0x1, -7264, 0xa641, 0xa66d, 0x2, -1, 0xa681, 0xa69b, 0x2, -1,
325
+ 0xa723, 0xa72f, 0x2, -1, 0xa733, 0xa76f, 0x2, -1, 0xa77a, 0xa77c, 0x2, -1,
326
+ 0xa77f, 0xa787, 0x2, -1, 0xa78c, 0xa78c, 0x1, -1, 0xa791, 0xa793, 0x2, -1,
327
+ 0xa794, 0xa794, 0x1, 0x30, 0xa797, 0xa7a9, 0x2, -1, 0xa7b5, 0xa7c3, 0x2, -1,
328
+ 0xa7c8, 0xa7ca, 0x2, -1, 0xa7cd, 0xa7db, 0x2, -1, 0xa7f6, 0xa7f6, 0x1, -1,
329
+ 0xab53, 0xab53, 0x1, -928, 0xab70, 0xabbf, 0x1, -38864, 0xff41, 0xff5a, 0x1, -32,
330
+ 0x10428, 0x1044f, 0x1, -40, 0x104d8, 0x104fb, 0x1, -40, 0x10597, 0x105bb, 0x2, -39,
331
+ 0x10598, 0x105a0, 0x2, -39, 0x105a4, 0x105b0, 0x2, -39, 0x105b4, 0x105b8, 0x2, -39,
332
+ 0x105bc, 0x105bc, 0x1, -39, 0x10cc0, 0x10cf2, 0x1, -64, 0x10d70, 0x10d85, 0x1, -32,
333
+ 0x118c0, 0x118df, 0x1, -32, 0x16e60, 0x16e7f, 0x1, -32, 0x16ebb, 0x16ed3, 0x1, -27,
334
+ 0x1e922, 0x1e943, 0x1, -34,
335
+ ].freeze
336
+
337
+ # True for a code point in Lu, Ll, Lt, Lm, Lo, Mn, Mc or Me. Binary search over
338
+ # LETTER_RANGES.
339
+ def self.letter?(cp)
340
+ return false if cp.negative?
341
+ return (cp >= 0x41 && cp <= 0x5a) || (cp >= 0x61 && cp <= 0x7a) if cp < 0x80
342
+
343
+ binary_range_search?(LETTER_RANGES, cp)
344
+ end
345
+
346
+ # True for a code point in Lu or Lt. Binary search over UPPER_RANGES.
347
+ def self.upper?(cp)
348
+ return false if cp.negative?
349
+ return cp >= 0x41 && cp <= 0x5a if cp < 0x80
350
+
351
+ binary_range_search?(UPPER_RANGES, cp)
352
+ end
353
+
354
+ # Simple uppercase mapping of one code point, or the code point itself when it has none.
355
+ def self.simple_uppercase(cp)
356
+ i = 0
357
+ while i < SIMPLE_UPPERCASE.length
358
+ start = SIMPLE_UPPERCASE[i]
359
+ return cp if cp < start
360
+
361
+ stop = SIMPLE_UPPERCASE[i + 1]
362
+ step = SIMPLE_UPPERCASE[i + 2]
363
+ return cp + SIMPLE_UPPERCASE[i + 3] if cp <= stop && ((cp - start) % step).zero?
364
+
365
+ i += 4
366
+ end
367
+ cp
368
+ end
369
+
370
+ def self.binary_range_search?(ranges, cp)
371
+ lo = 0
372
+ hi = (ranges.length / 2) - 1
373
+ while lo <= hi
374
+ mid = (lo + hi) / 2
375
+ start = ranges[mid * 2]
376
+ stop = ranges[(mid * 2) + 1]
377
+ if cp < start
378
+ hi = mid - 1
379
+ elsif cp > stop
380
+ lo = mid + 1
381
+ else
382
+ return true
383
+ end
384
+ end
385
+ false
386
+ end
387
+ private_class_method :binary_range_search?
388
+ end
389
+ end
390
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Polytypo
4
+ # The seven stable, cross-runtime error codes (docs/ARCHITECTURE.md section 4.6). Messages are
5
+ # English and are not part of the contract; codes are.
6
+ CODE_UNKNOWN_LOCALE = "POLYTYPO_UNKNOWN_LOCALE"
7
+ CODE_INVALID_MODE = "POLYTYPO_INVALID_MODE"
8
+ CODE_INVALID_DIALECT = "POLYTYPO_INVALID_DIALECT"
9
+ CODE_UNKNOWN_RULE = "POLYTYPO_UNKNOWN_RULE"
10
+ CODE_MALFORMED_LOCALE_DATA = "POLYTYPO_MALFORMED_LOCALE_DATA"
11
+ CODE_RULE_CONTRACT = "POLYTYPO_RULE_CONTRACT"
12
+ CODE_MALFORMED_INPUT = "POLYTYPO_MALFORMED_INPUT"
13
+
14
+ # The only error type Transform ever raises. One class is enough: the stable machine-readable
15
+ # +code+ is the contract, not the exception type or the message text.
16
+ class Error < StandardError
17
+ attr_reader :code
18
+
19
+ def initialize(code, message)
20
+ super(message)
21
+ @code = code
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,233 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "spans"
4
+ require_relative "parse_error"
5
+ require_relative "../engine/codepoints"
6
+
7
+ module Polytypo
8
+ module Modes
9
+ # HTML span extraction via a minimal, hand-rolled, position-tracking tokenizer -- not
10
+ # `nokogiri` (ARCHITECTURE.md section 2 names it for Ruby, but verified this session that
11
+ # neither its DOM (`Nokogiri::XML::Text#line`, line number only, no column) nor its SAX
12
+ # callbacks (no position argument at all) give the exact character-level offsets the span
13
+ # model needs). Mirrors what Python's `html.parser` and Go's `x/net/html` low-level
14
+ # `Tokenizer` already do for the same reason: neither is a full HTML5 tree-construction
15
+ # implementation either, and this does not need to be one.
16
+ #
17
+ # Operates directly on Ruby's code-point-indexed String -- no separate byte/char offset
18
+ # conversion is needed here the way Go's port needed one for its byte-oriented tokenizer.
19
+ module Html
20
+ # modes.md 3.6, exhaustive and CLOSED: extending it is a spec change, not an implementation
21
+ # decision. svg/math are here because in MathML a quotation mark, a hyphen and a prime are
22
+ # operators and identifiers -- substituting a curly glyph changes what the expression means.
23
+ SKIPPED_ELEMENTS = %w[code pre kbd samp var script style textarea svg math].freeze
24
+
25
+ # HTML5 void elements: never pushed onto the element stack, since an author is not required
26
+ # to close them and this tokenizer never synthesizes a matching end tag for one.
27
+ VOID_ELEMENTS = %w[area base br col embed hr img input link meta param source track wbr].freeze
28
+
29
+ TAG_NAME_STOP = [">", "/", " ", "\t", "\n", "\r", "\f"].freeze
30
+
31
+ def self.ascii_alpha?(ch)
32
+ (ch >= "a" && ch <= "z") || (ch >= "A" && ch <= "Z")
33
+ end
34
+ private_class_method :ascii_alpha?
35
+
36
+ def self.ascii_alnum?(ch)
37
+ ascii_alpha?(ch) || (ch >= "0" && ch <= "9")
38
+ end
39
+ private_class_method :ascii_alnum?
40
+
41
+ def self.hex_digit?(ch)
42
+ (ch >= "0" && ch <= "9") || (ch >= "a" && ch <= "f") || (ch >= "A" && ch <= "F")
43
+ end
44
+ private_class_method :hex_digit?
45
+
46
+ # Byte ranges [start, end) of every well-formed character reference in chars (a code-point
47
+ # array slice) -- &name;, &#1234;, &#x2014; -- as spec/rules/modes.md 3.6 requires them
48
+ # treated: opaque units, split out of the surrounding text span so their exact source
49
+ # spelling survives untouched. "Well-formed" is syntactic (shape only, matching the
50
+ # html5lib/Python html.parser precedent this project's other ports follow), not validated
51
+ # against the registered named-character-reference table: a bare & that begins no
52
+ # well-formed reference is deliberately left inside its span (modes.md 3.6: "Tom & Jerry's
53
+ # \"book\"" must not lose the pairing of its quotation marks to a spurious boundary).
54
+ def self.find_entity_refs(chars)
55
+ refs = []
56
+ n = chars.length
57
+ i = 0
58
+ while i < n
59
+ unless chars[i] == "&"
60
+ i += 1
61
+ next
62
+ end
63
+ start = i
64
+ j = i + 1
65
+ if j < n && chars[j] == "#" && j + 1 < n && (chars[j + 1] == "x" || chars[j + 1] == "X")
66
+ k = j + 2
67
+ k += 1 while k < n && hex_digit?(chars[k])
68
+ if k > j + 2 && k < n && chars[k] == ";"
69
+ refs << [start, k + 1]
70
+ i = k + 1
71
+ next
72
+ end
73
+ elsif j < n && chars[j] == "#"
74
+ k = j + 1
75
+ k += 1 while k < n && chars[k] >= "0" && chars[k] <= "9"
76
+ if k > j + 1 && k < n && chars[k] == ";"
77
+ refs << [start, k + 1]
78
+ i = k + 1
79
+ next
80
+ end
81
+ elsif j < n && ascii_alpha?(chars[j])
82
+ k = j + 1
83
+ k += 1 while k < n && ascii_alnum?(chars[k])
84
+ if k < n && chars[k] == ";"
85
+ refs << [start, k + 1]
86
+ i = k + 1
87
+ next
88
+ end
89
+ end
90
+ i += 1
91
+ end
92
+ refs
93
+ end
94
+ private_class_method :find_entity_refs
95
+
96
+ # Parses "<name ...>", "</name>" or "<name .../>" (as an array of chars) into
97
+ # [name, closing, self_closing]. Comments, declarations and processing instructions
98
+ # ("<!--", "<!", "<?") have no element name and return name == nil.
99
+ def self.read_tag(chars)
100
+ return nil if chars.empty? || chars[0] != "<"
101
+
102
+ i = 1
103
+ closing = i < chars.length && chars[i] == "/"
104
+ i += 1 if closing
105
+ return nil if i >= chars.length || chars[i] == "!" || chars[i] == "?"
106
+ return nil unless ascii_alpha?(chars[i])
107
+
108
+ name_start = i
109
+ i += 1 while i < chars.length && !TAG_NAME_STOP.include?(chars[i])
110
+ name = chars[name_start...i].join.downcase
111
+ self_closing = chars.join.rstrip.end_with?("/>")
112
+ [name, closing, self_closing]
113
+ end
114
+ # Public: also used by Markdown for its inline-raw-HTML tag-stack handling.
115
+
116
+ def self.last_stack_index(stack, tag)
117
+ stack.rindex(tag)
118
+ end
119
+ private_class_method :last_stack_index
120
+
121
+ # Locates the processable spans of an HTML document. The tokenizer is used only to locate
122
+ # spans and is then discarded (modes.md 4: "the document is never serialised") --
123
+ # attributes, tag syntax and entity spelling are never touched.
124
+ def self.html_spans(source)
125
+ cp = Polytypo::Engine::Codepoints.to_codepoints(source)
126
+ chars = cp.map { |c| [c].pack("U") }
127
+ n = chars.length
128
+
129
+ spans = []
130
+ stack = []
131
+ skip_depth = 0
132
+ i = 0
133
+
134
+ ParseError.wrap do
135
+ while i < n
136
+ if chars[i] == "<"
137
+ tag_end = find_tag_end(chars, i)
138
+ if tag_end.nil?
139
+ # A bare "<" with no closing ">" before EOF: the rest of the document is text.
140
+ emit_text(spans, chars, i, n, skip_depth)
141
+ i = n
142
+ next
143
+ end
144
+
145
+ tag_chars = chars[i...tag_end]
146
+ parsed = read_tag(tag_chars)
147
+ if parsed
148
+ name, closing, self_closing = parsed
149
+ if closing
150
+ idx = last_stack_index(stack, name)
151
+ if idx
152
+ popped = stack[idx..]
153
+ stack = stack[0...idx]
154
+ popped.each { |n2| skip_depth -= 1 if SKIPPED_ELEMENTS.include?(n2) }
155
+ end
156
+ elsif self_closing || VOID_ELEMENTS.include?(name)
157
+ # Opens and closes atomically; no persistent stack effect regardless of the
158
+ # tag's identity, since there is no subsequent content inside it to skip.
159
+ else
160
+ stack << name
161
+ skip_depth += 1 if SKIPPED_ELEMENTS.include?(name)
162
+ end
163
+ end
164
+ # Comments/declarations/processing instructions (parsed.nil?) have no stack effect.
165
+ i = tag_end
166
+ else
167
+ text_end = i
168
+ text_end += 1 while text_end < n && chars[text_end] != "<"
169
+ emit_text(spans, chars, i, text_end, skip_depth)
170
+ i = text_end
171
+ end
172
+ end
173
+ end
174
+
175
+ spans
176
+ end
177
+
178
+ # Finds the index just past the ">" that closes the tag starting at chars[start] == "<",
179
+ # honoring comments ("<!--" ... "-->") and quoted attribute values so a ">" inside either
180
+ # does not end the tag early. Returns nil if unterminated.
181
+ def self.find_tag_end(chars, start)
182
+ n = chars.length
183
+ if chars[start, 4] == ["<", "!", "-", "-"]
184
+ close = index_of_sequence(chars, ["-", "-", ">"], start + 4)
185
+ return close.nil? ? nil : close + 3
186
+ end
187
+
188
+ i = start + 1
189
+ quote = nil
190
+ while i < n
191
+ ch = chars[i]
192
+ if quote
193
+ quote = nil if ch == quote
194
+ elsif ch == "\"" || ch == "'"
195
+ quote = ch
196
+ elsif ch == ">"
197
+ return i + 1
198
+ end
199
+ i += 1
200
+ end
201
+ nil
202
+ end
203
+ private_class_method :find_tag_end
204
+
205
+ def self.index_of_sequence(chars, seq, from)
206
+ n = chars.length
207
+ m = seq.length
208
+ i = from
209
+ while i + m <= n
210
+ return i if chars[i, m] == seq
211
+
212
+ i += 1
213
+ end
214
+ nil
215
+ end
216
+ private_class_method :index_of_sequence
217
+
218
+ def self.emit_text(spans, chars, from, to, skip_depth)
219
+ return if skip_depth != 0 || to <= from
220
+
221
+ slice = chars[from...to]
222
+ refs = find_entity_refs(slice)
223
+ cursor = 0
224
+ refs.each do |ref_start, ref_end|
225
+ spans << Spans::Span.new(from + cursor, from + ref_start) if ref_start > cursor
226
+ cursor = ref_end
227
+ end
228
+ spans << Spans::Span.new(from + cursor, to) if cursor < slice.length
229
+ end
230
+ private_class_method :emit_text
231
+ end
232
+ end
233
+ end