polytypo 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +109 -0
- data/lib/polytypo/data/README.md +20 -0
- data/lib/polytypo/data/UNICODE +1 -0
- data/lib/polytypo/data/VERSION +1 -0
- data/lib/polytypo/data/fixtures/de-CH.json +501 -0
- data/lib/polytypo/data/fixtures/de-DE.json +547 -0
- data/lib/polytypo/data/fixtures/el.json +239 -0
- data/lib/polytypo/data/fixtures/en-GB.json +1274 -0
- data/lib/polytypo/data/fixtures/en-US.json +1807 -0
- data/lib/polytypo/data/fixtures/fi.json +1306 -0
- data/lib/polytypo/data/fixtures/fr-CA.json +268 -0
- data/lib/polytypo/data/fixtures/fr.json +603 -0
- data/lib/polytypo/data/fixtures/locale-resolution.json +209 -0
- data/lib/polytypo/data/fixtures/ru.json +688 -0
- data/lib/polytypo/data/fixtures/sv.json +1290 -0
- data/lib/polytypo/data/locales/de-CH.json +77 -0
- data/lib/polytypo/data/locales/de-DE.json +76 -0
- data/lib/polytypo/data/locales/el.json +90 -0
- data/lib/polytypo/data/locales/en-GB.json +115 -0
- data/lib/polytypo/data/locales/en-US.json +133 -0
- data/lib/polytypo/data/locales/fi.json +136 -0
- data/lib/polytypo/data/locales/fr-CA.json +78 -0
- data/lib/polytypo/data/locales/fr.json +84 -0
- data/lib/polytypo/data/locales/registry.json +9 -0
- data/lib/polytypo/data/locales/ru.json +112 -0
- data/lib/polytypo/data/locales/sv.json +124 -0
- data/lib/polytypo/data/rules/dashes.md +1238 -0
- data/lib/polytypo/data/rules/order.json +78 -0
- data/lib/polytypo/data/schema/fixtures.schema.json +79 -0
- data/lib/polytypo/data/schema/locale.schema.json +235 -0
- data/lib/polytypo/data/schema/registry.schema.json +29 -0
- data/lib/polytypo/data/schema/resolution.schema.json +50 -0
- data/lib/polytypo/engine/codepoints.rb +24 -0
- data/lib/polytypo/engine/edits.rb +64 -0
- data/lib/polytypo/engine/locale.rb +138 -0
- data/lib/polytypo/engine/pipeline.rb +61 -0
- data/lib/polytypo/engine/registry.rb +47 -0
- data/lib/polytypo/engine/rules/apostrophe.rb +127 -0
- data/lib/polytypo/engine/rules/dash_shared.rb +342 -0
- data/lib/polytypo/engine/rules/dashes.rb +125 -0
- data/lib/polytypo/engine/rules/ellipsis.rb +100 -0
- data/lib/polytypo/engine/rules/hyphen.rb +207 -0
- data/lib/polytypo/engine/rules/nbsp.rb +616 -0
- data/lib/polytypo/engine/rules/quote_ambiguity.rb +241 -0
- data/lib/polytypo/engine/rules/quotes.rb +420 -0
- data/lib/polytypo/engine/rules/ranges.rb +124 -0
- data/lib/polytypo/engine/rules/spaces.rb +232 -0
- data/lib/polytypo/engine/rules/symbols.rb +291 -0
- data/lib/polytypo/engine/rules.rb +19 -0
- data/lib/polytypo/engine/sentinels.rb +23 -0
- data/lib/polytypo/engine/unicode_util.rb +390 -0
- data/lib/polytypo/errors.rb +24 -0
- data/lib/polytypo/modes/html.rb +233 -0
- data/lib/polytypo/modes/markdown.rb +187 -0
- data/lib/polytypo/modes/parse_error.rb +19 -0
- data/lib/polytypo/modes/runner.rb +57 -0
- data/lib/polytypo/modes/spans.rb +132 -0
- data/lib/polytypo/version.rb +5 -0
- data/lib/polytypo.rb +91 -0
- data/polytypo.gemspec +37 -0
- metadata +122 -0
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "dash_shared"
|
|
4
|
+
require_relative "../edits"
|
|
5
|
+
require_relative "../registry"
|
|
6
|
+
require_relative "../unicode_util"
|
|
7
|
+
|
|
8
|
+
module Polytypo
|
|
9
|
+
module Engine
|
|
10
|
+
module Rules
|
|
11
|
+
# `ranges` -- spec/rules/ranges.md, order 25. Explicit opt-in: off by default
|
|
12
|
+
# (order.json's "default": "off" for this rule id; the pipeline's rule-plan resolution, not
|
|
13
|
+
# this file, is what enforces that). Split out of `dashes` (spec 0.5.0) -- see ranges.md 1
|
|
14
|
+
# and dashes.md 7.11 for why this is opt-in rather than a bounded structural fix: separating
|
|
15
|
+
# a genuine numeric range (`5-10`) from a compound label sharing the identical shape
|
|
16
|
+
# (`Figure 5-10`) needs the preceding word, which is exactly the open-ended, per-locale
|
|
17
|
+
# context this project's rules are built never to consult.
|
|
18
|
+
#
|
|
19
|
+
# This module's own behaviour does not depend on whether the rule is enabled -- enable/disable
|
|
20
|
+
# is the pipeline's concern (a rule's scan function only runs when it is active).
|
|
21
|
+
#
|
|
22
|
+
# Explicit index-based scanning only: no regex anywhere, and every index addresses the
|
|
23
|
+
# code-point array, never a native string (ARCHITECTURE.md 4.1, 4.2).
|
|
24
|
+
module Ranges
|
|
25
|
+
SOLIDUS = 0x2f
|
|
26
|
+
|
|
27
|
+
module_function
|
|
28
|
+
|
|
29
|
+
# ranges.md 3.3 G5: equal-length ASCII digit runs compare lexicographically, so no integer
|
|
30
|
+
# arithmetic (and no locale-dependent parsing) is needed.
|
|
31
|
+
def non_decreasing?(cp, left_start, right_start, length)
|
|
32
|
+
(0...length).each do |i|
|
|
33
|
+
l = cp[left_start + i]
|
|
34
|
+
r = cp[right_start + i]
|
|
35
|
+
return true if l < r
|
|
36
|
+
return false if l > r
|
|
37
|
+
end
|
|
38
|
+
true
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# ranges.md 3.3, G1-G5. left/right are the token's post-joiner-walk flank indices (both
|
|
42
|
+
# already known to be DIGIT by the caller).
|
|
43
|
+
def guards_pass?(cp, left, right)
|
|
44
|
+
n = cp.length
|
|
45
|
+
a = left
|
|
46
|
+
a -= 1 while a.positive? && DashShared.digit?(cp[a - 1])
|
|
47
|
+
b = right
|
|
48
|
+
b += 1 while b + 1 < n && DashShared.digit?(cp[b + 1])
|
|
49
|
+
|
|
50
|
+
before = DashShared.effective_neighbour(cp, a - 1, -1)
|
|
51
|
+
after = DashShared.effective_neighbour(cp, b + 1, 1)
|
|
52
|
+
|
|
53
|
+
# G1 -- no letter adjacency.
|
|
54
|
+
return false if Polytypo::Engine::UnicodeUtil.letter?(before)
|
|
55
|
+
|
|
56
|
+
# G2 -- no chain: an ISO date, an ISBN or a phone number always trips this. This guard
|
|
57
|
+
# reads the input as it stood before this rule (or `dashes`) made any edit in this
|
|
58
|
+
# pipeline pass -- ranges.md 4's own reasoning for why `ranges` must run before `dashes`.
|
|
59
|
+
return false if DashShared.dash_union?(before)
|
|
60
|
+
return false if DashShared.dash_union?(after)
|
|
61
|
+
|
|
62
|
+
# G3 -- not part of a decimal or a path.
|
|
63
|
+
return false if before == DashShared::FULL_STOP || before == DashShared::COMMA || before == SOLIDUS
|
|
64
|
+
return false if after == SOLIDUS
|
|
65
|
+
|
|
66
|
+
# G4 -- run lengths: equal, or the directional (1,2) branch with no leading zero on Rrun.
|
|
67
|
+
left_length = left - a + 1
|
|
68
|
+
right_length = b - right + 1
|
|
69
|
+
return true if left_length == 1 && right_length == 2 && cp[right] != DashShared::DIGIT_ZERO
|
|
70
|
+
return false if left_length != right_length
|
|
71
|
+
|
|
72
|
+
# G5 -- non-decreasing (sound only because G4 guarantees equal length in this branch).
|
|
73
|
+
non_decreasing?(cp, a, right, left_length)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def scan(cp, locale_data, _ctx)
|
|
77
|
+
edits = []
|
|
78
|
+
style = locale_data["dash"]["range"]
|
|
79
|
+
|
|
80
|
+
DashShared.find_tokens(cp).each do |token|
|
|
81
|
+
# ranges.md 3.2 -- a range candidate iff both flanks are DIGIT. `ranges` never
|
|
82
|
+
# processes any other token shape; that is `dashes`' territory, and `dashes` declines
|
|
83
|
+
# a digit-flanked token unconditionally too (operator decision, spec 0.5.0) -- neither
|
|
84
|
+
# rule reinterprets the other's shape, whether or not `ranges` is enabled.
|
|
85
|
+
next unless DashShared.digit?(token.left_cp) && DashShared.digit?(token.right_cp)
|
|
86
|
+
|
|
87
|
+
next unless guards_pass?(cp, token.left, token.right)
|
|
88
|
+
|
|
89
|
+
# "none": the locale has no verified range convention, so nothing is substituted --
|
|
90
|
+
# not a fallback to dash.parenthetical, nothing (ranges.md 2).
|
|
91
|
+
next if style == "none"
|
|
92
|
+
|
|
93
|
+
if DashShared.spaced_style?(style)
|
|
94
|
+
# T1: a tight token may not become spaced across a digit run that has a far dash.
|
|
95
|
+
next if token.lsp.zero? && token.rsp.zero? &&
|
|
96
|
+
DashShared.spacing_transition_blocked?(cp, token.left, token.right)
|
|
97
|
+
|
|
98
|
+
# T2: the emitted U+0020 must not land where `spaces` (order 10) would delete it.
|
|
99
|
+
next if DashShared.strip_before_or_close_bracket?(token.right_cp)
|
|
100
|
+
next if DashShared.open_bracket?(token.left_cp)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# ranges.md 3.3.1: never make an edit whose entire content is invisible. Try the
|
|
104
|
+
# unbound replacement first; only add the joiner pair if the dash itself is genuinely
|
|
105
|
+
# changing.
|
|
106
|
+
unbound = DashShared.build_replacement(style, false)
|
|
107
|
+
only_binding_would_change = !DashShared.spaced_style?(style) &&
|
|
108
|
+
DashShared.same_content?(cp, token.span_start, token.span_end, unbound)
|
|
109
|
+
bind = !DashShared.spaced_style?(style) && !only_binding_would_change
|
|
110
|
+
replacement = bind ? DashShared.build_replacement(style, true) : unbound
|
|
111
|
+
|
|
112
|
+
next if DashShared.same_content?(cp, token.span_start, token.span_end, replacement)
|
|
113
|
+
|
|
114
|
+
edits << Polytypo::Engine::Edit.new(token.span_start, token.span_end, replacement, "ranges")
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
edits
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
Polytypo::Engine::Registry.register("ranges", ->(cp, locale_data, ctx) { Ranges.scan(cp, locale_data, ctx) })
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
end
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../edits"
|
|
4
|
+
require_relative "../sentinels"
|
|
5
|
+
require_relative "../unicode_util"
|
|
6
|
+
require_relative "../registry"
|
|
7
|
+
|
|
8
|
+
module Polytypo
|
|
9
|
+
module Engine
|
|
10
|
+
module Rules
|
|
11
|
+
# spec/rules/spaces.md. No regex anywhere (ARCHITECTURE.md section 4.1): a single
|
|
12
|
+
# left-to-right scan over the code-point array with explicit lookaround by index. Reads no
|
|
13
|
+
# locale data (spec/rules/spaces.md section 2): behaviour is identical in every locale.
|
|
14
|
+
module Spaces
|
|
15
|
+
SPACE = 0x20
|
|
16
|
+
|
|
17
|
+
LF = 0x0A
|
|
18
|
+
CR = 0x0D
|
|
19
|
+
VT = 0x0B
|
|
20
|
+
FF = 0x0C
|
|
21
|
+
NEL = 0x85
|
|
22
|
+
LS = 0x2028
|
|
23
|
+
PS = 0x2029
|
|
24
|
+
|
|
25
|
+
COMMA = 0x2C
|
|
26
|
+
FULL_STOP = 0x2E
|
|
27
|
+
SEMICOLON = 0x3B
|
|
28
|
+
COLON = 0x3A
|
|
29
|
+
EXCLAMATION = 0x21
|
|
30
|
+
QUESTION = 0x3F
|
|
31
|
+
ELLIPSIS = 0x2026
|
|
32
|
+
|
|
33
|
+
PAREN_OPEN = 0x28
|
|
34
|
+
PAREN_CLOSE = 0x29
|
|
35
|
+
SQUARE_OPEN = 0x5B
|
|
36
|
+
SQUARE_CLOSE = 0x5D
|
|
37
|
+
CURLY_OPEN = 0x7B
|
|
38
|
+
CURLY_CLOSE = 0x7D
|
|
39
|
+
|
|
40
|
+
HYPHEN_MINUS = 0x2D
|
|
41
|
+
CARET = 0x5E
|
|
42
|
+
SOLIDUS = 0x2F
|
|
43
|
+
REVERSE_SOLIDUS = 0x5C
|
|
44
|
+
VERTICAL_LINE = 0x7C
|
|
45
|
+
ASTERISK = 0x2A
|
|
46
|
+
DIGIT_ZERO = 0x30
|
|
47
|
+
DIGIT_NINE = 0x39
|
|
48
|
+
LETTER_D_UPPER = 0x44
|
|
49
|
+
LETTER_D_LOWER = 0x64
|
|
50
|
+
LETTER_P_UPPER = 0x50
|
|
51
|
+
LETTER_P_LOWER = 0x70
|
|
52
|
+
LETTER_O_UPPER = 0x4F
|
|
53
|
+
LETTER_O_LOWER = 0x6F
|
|
54
|
+
|
|
55
|
+
# cp[i], or NONE if i is out of bounds -- the spec's own boundary value.
|
|
56
|
+
def self.at(cp, i)
|
|
57
|
+
return NONE if i.negative? || i >= cp.length
|
|
58
|
+
|
|
59
|
+
cp[i]
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# BREAK (spaces.md 3.1), including LINE_MARKER: a member of BREAK for every rule,
|
|
63
|
+
# everywhere (modes.md 3.2).
|
|
64
|
+
def self.break?(value)
|
|
65
|
+
value == LF || value == CR || value == VT || value == FF ||
|
|
66
|
+
value == NEL || value == LS || value == PS || value == LINE_MARKER
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# STRIP-BEFORE: exactly six code points. U+2026 is deliberately absent -- with it,
|
|
70
|
+
# "Wait ..." would keep its space here, `ellipsis` would yield "Wait …", and a second
|
|
71
|
+
# pipeline pass would then strip that space, a composition divergence (spaces.md 3.4).
|
|
72
|
+
def self.strip_before?(value)
|
|
73
|
+
value == COMMA || value == FULL_STOP || value == SEMICOLON ||
|
|
74
|
+
value == COLON || value == EXCLAMATION || value == QUESTION
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def self.dotlike?(value)
|
|
78
|
+
value == FULL_STOP || value == ELLIPSIS
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def self.open_bracket?(value)
|
|
82
|
+
value == PAREN_OPEN || value == SQUARE_OPEN || value == CURLY_OPEN
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def self.close_bracket?(value)
|
|
86
|
+
value == PAREN_CLOSE || value == SQUARE_CLOSE || value == CURLY_CLOSE
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def self.matching_closer(open)
|
|
90
|
+
return PAREN_CLOSE if open == PAREN_OPEN
|
|
91
|
+
return SQUARE_CLOSE if open == SQUARE_OPEN
|
|
92
|
+
|
|
93
|
+
CURLY_CLOSE
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# spaces.md 3.3: "- [ ] item" must not become "- [] item". The run may still collapse to
|
|
97
|
+
# length 1.
|
|
98
|
+
def self.empty_bracket_guarded?(left, right)
|
|
99
|
+
open_bracket?(left) && right == matching_closer(left)
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# spaces.md 3.4. A run of dots is a different token from a terminal full stop -- a
|
|
103
|
+
# relative path, a truncation, a typed ellipsis -- and deleting the space before it
|
|
104
|
+
# merges the run with a preceding abbreviation dot ("See ../docs" -> "See../docs").
|
|
105
|
+
#
|
|
106
|
+
# `e` indexes `right` in the input array, and the run is measured there. Measuring it
|
|
107
|
+
# after any edit had been applied would break the Chicago spaced ellipsis
|
|
108
|
+
# "Hello . . .", where every dot is a lone dot at decision time and all three spaces
|
|
109
|
+
# must still strip.
|
|
110
|
+
def self.lone_dot?(cp, e)
|
|
111
|
+
return true if at(cp, e) != FULL_STOP
|
|
112
|
+
|
|
113
|
+
!dotlike?(at(cp, e + 1))
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def self.digit_ascii?(value)
|
|
117
|
+
value >= DIGIT_ZERO && value <= DIGIT_NINE
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# spaces.md 3.6: the recognised "mouth" glyphs of a Western text emoticon.
|
|
121
|
+
def self.emoticon_mouth?(value)
|
|
122
|
+
value == PAREN_OPEN || value == PAREN_CLOSE || value == SQUARE_OPEN ||
|
|
123
|
+
value == SQUARE_CLOSE || value == LETTER_D_UPPER || value == LETTER_D_LOWER ||
|
|
124
|
+
value == LETTER_P_UPPER || value == LETTER_P_LOWER ||
|
|
125
|
+
value == LETTER_O_UPPER || value == LETTER_O_LOWER ||
|
|
126
|
+
value == SOLIDUS || value == REVERSE_SOLIDUS ||
|
|
127
|
+
value == VERTICAL_LINE || value == ASTERISK
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# spaces.md 3.6, the emoticon guard's eye side. A colon or semicolon immediately
|
|
131
|
+
# followed by an optional "nose" and a recognised "mouth" is the eye of a Western text
|
|
132
|
+
# emoticon (":-)", ":)", ";-)"), not sentence punctuation, and the space in front of it
|
|
133
|
+
# must survive.
|
|
134
|
+
#
|
|
135
|
+
# The mouth must not itself run into a letter or an ASCII digit -- ":Deal" is a colon
|
|
136
|
+
# before a capitalised word, not a face -- which is the one check needed to keep this
|
|
137
|
+
# from firing on ordinary prose. `e` indexes `right`, exactly as `lone_dot?` does.
|
|
138
|
+
def self.emoticon_eye_fires?(cp, e)
|
|
139
|
+
eye = at(cp, e)
|
|
140
|
+
return false unless eye == COLON || eye == SEMICOLON
|
|
141
|
+
|
|
142
|
+
i = e + 1
|
|
143
|
+
nose = at(cp, i)
|
|
144
|
+
i += 1 if nose == HYPHEN_MINUS || nose == CARET
|
|
145
|
+
|
|
146
|
+
return false unless emoticon_mouth?(at(cp, i))
|
|
147
|
+
|
|
148
|
+
i += 1
|
|
149
|
+
after = at(cp, i)
|
|
150
|
+
!UnicodeUtil.letter?(after) && !digit_ascii?(after)
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# spaces.md 3.6, the mouth side. "(" and "[" are EMOTICON-MOUTH members and
|
|
154
|
+
# OPEN-BRACKET members at once, so without this the opening-bracket clause deleted the
|
|
155
|
+
# space after an emoticon the eye side had just recognised: "a :( b" became "a :(b",
|
|
156
|
+
# and then "a:(b" on a second pass, because a letter after the mouth stops the eye side
|
|
157
|
+
# firing.
|
|
158
|
+
#
|
|
159
|
+
# The walk mirrors the eye side's, backwards from `s`, and needs no trailing check: the
|
|
160
|
+
# code point after the mouth is the space run itself, which is neither a letter nor a
|
|
161
|
+
# digit. A nose with no eye behind it is not a face -- EMOTICON-NOSE and EMOTICON-EYE
|
|
162
|
+
# are disjoint, so the walk cannot mistake one for the other.
|
|
163
|
+
def self.emoticon_mouth_fires?(cp, s)
|
|
164
|
+
return false unless emoticon_mouth?(at(cp, s - 1))
|
|
165
|
+
|
|
166
|
+
j = s - 2
|
|
167
|
+
nose = at(cp, j)
|
|
168
|
+
j -= 1 if nose == HYPHEN_MINUS || nose == CARET
|
|
169
|
+
|
|
170
|
+
eye = at(cp, j)
|
|
171
|
+
eye == COLON || eye == SEMICOLON
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
# spaces.md 3.2 step 5: the replacement length is a pure function of the two bounding
|
|
175
|
+
# code points, computed once. A two-pass "collapse then strip" formulation would need a
|
|
176
|
+
# fixed-point loop, which two runtimes would iterate differently.
|
|
177
|
+
def self.replacement_length(cp, s, e, left, right)
|
|
178
|
+
# The guard is a clause of this decision, not a "skip the run" branch: "( )"
|
|
179
|
+
# collapses to "( )" (spaces.md 3.3, normative reading).
|
|
180
|
+
return 1 if empty_bracket_guarded?(left, right)
|
|
181
|
+
return 0 if open_bracket?(left) && !emoticon_mouth_fires?(cp, s)
|
|
182
|
+
return 0 if close_bracket?(right)
|
|
183
|
+
return 0 if strip_before?(right) && lone_dot?(cp, e) && !emoticon_eye_fires?(cp, e)
|
|
184
|
+
|
|
185
|
+
1
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
# locale_data is unused: order.json declares "localeData": [] for this rule.
|
|
189
|
+
def self.scan(cp, _locale_data, _ctx)
|
|
190
|
+
n = cp.length
|
|
191
|
+
edits = []
|
|
192
|
+
i = 0
|
|
193
|
+
|
|
194
|
+
while i < n
|
|
195
|
+
if at(cp, i) != SPACE
|
|
196
|
+
i += 1
|
|
197
|
+
next
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
s = i
|
|
201
|
+
e = s
|
|
202
|
+
e += 1 while e < n && at(cp, e) == SPACE
|
|
203
|
+
k = e - s
|
|
204
|
+
|
|
205
|
+
left = at(cp, s - 1)
|
|
206
|
+
right = at(cp, e)
|
|
207
|
+
|
|
208
|
+
# Boundary guard (3.2 step 4): indentation, Markdown hard breaks and text-unit
|
|
209
|
+
# edges are structural. A span boundary marker counts as NONE here -- the one place
|
|
210
|
+
# in the whole spec where a marker is not opaque content, per modes.md 3.3.
|
|
211
|
+
if left == NONE || Polytypo::Engine.marker?(left) || break?(left) ||
|
|
212
|
+
right == NONE || Polytypo::Engine.marker?(right) || break?(right)
|
|
213
|
+
i = e
|
|
214
|
+
next
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
length = replacement_length(cp, s, e, left, right)
|
|
218
|
+
if length != k
|
|
219
|
+
replacement = length.zero? ? [] : [SPACE]
|
|
220
|
+
edits << Edit.new(s, e, replacement, "spaces")
|
|
221
|
+
end
|
|
222
|
+
i = e
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
edits
|
|
226
|
+
end
|
|
227
|
+
end
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
Registry.register("spaces", ->(cp, locale_data, ctx) { Rules::Spaces.scan(cp, locale_data, ctx) })
|
|
231
|
+
end
|
|
232
|
+
end
|
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../edits"
|
|
4
|
+
require_relative "../sentinels"
|
|
5
|
+
require_relative "../unicode_util"
|
|
6
|
+
require_relative "../registry"
|
|
7
|
+
|
|
8
|
+
module Polytypo
|
|
9
|
+
module Engine
|
|
10
|
+
module Rules
|
|
11
|
+
# `symbols` -- spec/rules/symbols.md (spec 0.1.0), order 60.
|
|
12
|
+
#
|
|
13
|
+
# Three unrelated substitutions, one left-to-right scan: `(c)`/`(r)`/`(tm)` literals
|
|
14
|
+
# become (c)/(r)/(tm) signs (3.2); a `DIGIT+ (MUL-LETTER DIGIT+)+` chain becomes
|
|
15
|
+
# multiplication signs, converted whole or not at all (3.3); the literal `+/-` becomes
|
|
16
|
+
# plus-minus (3.4). No locale data -- order.json declares "localeData": [] for this rule.
|
|
17
|
+
# No regex, no case folding anywhere (ARCHITECTURE.md section 4.1, 4.4): the accepted
|
|
18
|
+
# spellings are enumerated explicitly rather than derived from `downcase`/`upcase`, which
|
|
19
|
+
# are locale-dependent in the host process (Turkish dotless i).
|
|
20
|
+
module Symbols
|
|
21
|
+
PAREN_OPEN = 0x28
|
|
22
|
+
PAREN_CLOSE = 0x29
|
|
23
|
+
SQUARE_OPEN = 0x5B
|
|
24
|
+
SQUARE_CLOSE = 0x5D
|
|
25
|
+
|
|
26
|
+
COPYRIGHT = 0xA9
|
|
27
|
+
REGISTERED = 0xAE
|
|
28
|
+
TRADEMARK = 0x2122
|
|
29
|
+
MULTIPLICATION = 0xD7
|
|
30
|
+
|
|
31
|
+
SPACE = 0x20
|
|
32
|
+
NBSP = 0xA0
|
|
33
|
+
NNBSP = 0x202F
|
|
34
|
+
|
|
35
|
+
LOWER_X = 0x78
|
|
36
|
+
UPPER_X = 0x58
|
|
37
|
+
CYRILLIC_LOWER_HA = 0x445
|
|
38
|
+
CYRILLIC_UPPER_HA = 0x425
|
|
39
|
+
DIGIT_ZERO = 0x30
|
|
40
|
+
DIGIT_NINE = 0x39
|
|
41
|
+
|
|
42
|
+
PLUS = 0x2B
|
|
43
|
+
SOLIDUS = 0x2F
|
|
44
|
+
HYPHEN_MINUS = 0x2D
|
|
45
|
+
PLUS_MINUS = 0xB1
|
|
46
|
+
|
|
47
|
+
# cp[i], or NONE if i is out of bounds -- the spec's own boundary value.
|
|
48
|
+
def self.at(cp, i)
|
|
49
|
+
return NONE if i.negative? || i >= cp.length
|
|
50
|
+
|
|
51
|
+
cp[i]
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def self.digit?(cp)
|
|
55
|
+
cp >= DIGIT_ZERO && cp <= DIGIT_NINE
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def self.alnum?(cp)
|
|
59
|
+
digit?(cp) || UnicodeUtil.letter?(cp)
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# SPACE u NOBREAK-SPACE (symbols.md 3.1): U+0020, U+00A0, U+202F only -- no tabs, no
|
|
63
|
+
# other Unicode spaces, no line breaks. Narrower than nbsp's SPACELIKE on purpose: this
|
|
64
|
+
# rule only needs the spacing a multiplication chain can legally carry.
|
|
65
|
+
def self.space_like?(cp)
|
|
66
|
+
cp == SPACE || cp == NBSP || cp == NNBSP
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# MUL-LETTER (symbols.md 3.1): exactly four code points, enumerated, never case-folded
|
|
70
|
+
# and never derived from locale data. The Cyrillic pair is unconditional -- U+0445
|
|
71
|
+
# between two ASCII digits is a Russian dimension typed on a Cyrillic layout or
|
|
72
|
+
# keyboard-layout debris, and the glyphs are visually identical to the Latin ones in
|
|
73
|
+
# every font, so no human review can catch a missed conversion.
|
|
74
|
+
def self.mul_letter?(cp)
|
|
75
|
+
cp == LOWER_X || cp == UPPER_X || cp == CYRILLIC_LOWER_HA || cp == CYRILLIC_UPPER_HA
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# One row of the trademark table (symbols.md 3.1). guarded_by_s1 is true only for the
|
|
79
|
+
# (c)/(r) rows -- the (tm) rows are exempt from S1 (3.2 step 3).
|
|
80
|
+
TrademarkRow = Struct.new(:literal, :to, :guarded_by_s1)
|
|
81
|
+
|
|
82
|
+
# Exhaustive and case-explicit; nothing else matches. Longest literals first (the
|
|
83
|
+
# 4-code-point (tm) rows before the 3-code-point (c)/(r) rows), per 3.2 step 1.
|
|
84
|
+
TRADEMARK_TABLE = [
|
|
85
|
+
TrademarkRow.new([PAREN_OPEN, 0x74, 0x6D, PAREN_CLOSE], TRADEMARK, false), # (tm)
|
|
86
|
+
TrademarkRow.new([PAREN_OPEN, 0x54, 0x4D, PAREN_CLOSE], TRADEMARK, false), # (TM)
|
|
87
|
+
TrademarkRow.new([PAREN_OPEN, 0x54, 0x6D, PAREN_CLOSE], TRADEMARK, false), # (Tm)
|
|
88
|
+
TrademarkRow.new([PAREN_OPEN, 0x74, 0x4D, PAREN_CLOSE], TRADEMARK, false), # (tM)
|
|
89
|
+
TrademarkRow.new([PAREN_OPEN, 0x63, PAREN_CLOSE], COPYRIGHT, true), # (c)
|
|
90
|
+
TrademarkRow.new([PAREN_OPEN, 0x43, PAREN_CLOSE], COPYRIGHT, true), # (C)
|
|
91
|
+
TrademarkRow.new([PAREN_OPEN, 0x72, PAREN_CLOSE], REGISTERED, true), # (r)
|
|
92
|
+
TrademarkRow.new([PAREN_OPEN, 0x52, PAREN_CLOSE], REGISTERED, true), # (R)
|
|
93
|
+
].freeze
|
|
94
|
+
|
|
95
|
+
def self.matches_at?(cp, i, literal)
|
|
96
|
+
return false if i + literal.length > cp.length
|
|
97
|
+
|
|
98
|
+
literal.each_with_index { |want, j| return false if at(cp, i + j) != want }
|
|
99
|
+
true
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# symbols.md 3.2. Returns the Edit, or nil when no row matches or a guard rejects the
|
|
103
|
+
# candidate.
|
|
104
|
+
def self.trademark_at(cp, i)
|
|
105
|
+
TRADEMARK_TABLE.each do |row|
|
|
106
|
+
next unless matches_at?(cp, i, row.literal)
|
|
107
|
+
|
|
108
|
+
end_index = i + row.literal.length
|
|
109
|
+
before = at(cp, i - 1)
|
|
110
|
+
after = at(cp, end_index)
|
|
111
|
+
|
|
112
|
+
# S1 -- left adjacency, (c)/(r) rows only: a one-letter argument list ("f(c)") is
|
|
113
|
+
# common, "(tm)" tucked against a product name is not a call. The three replacement
|
|
114
|
+
# signs are listed so that "(c)(r)" converges to "(c)(r)" in one run (symbols.md 5).
|
|
115
|
+
if row.guarded_by_s1 && before != NONE &&
|
|
116
|
+
(alnum?(before) || before == PAREN_CLOSE || before == SQUARE_CLOSE ||
|
|
117
|
+
before == COPYRIGHT || before == REGISTERED || before == TRADEMARK)
|
|
118
|
+
return nil
|
|
119
|
+
end
|
|
120
|
+
# S2 -- right adjacency: "(r)evolution", "(c)ompiler".
|
|
121
|
+
return nil if after != NONE && alnum?(after)
|
|
122
|
+
# S3 -- no nesting: "((c))" is ASCII art or code.
|
|
123
|
+
return nil if before == PAREN_OPEN
|
|
124
|
+
|
|
125
|
+
return Edit.new(i, end_index, [row.to], "symbols")
|
|
126
|
+
end
|
|
127
|
+
nil
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# One MUL-LETTER position within a multiplication chain, plus whether it carried a
|
|
131
|
+
# space on each side (0 or 1 code point -- symbols.md 3.3 never allows more than one).
|
|
132
|
+
ChainLink = Struct.new(:letter_index, :left_space, :right_space)
|
|
133
|
+
|
|
134
|
+
# The result of reading a whole DIGIT+ (MUL-LETTER DIGIT+)+ shape starting at a maximal
|
|
135
|
+
# digit run (symbols.md 3.3 step 1). end_index is the last code point read, whether or
|
|
136
|
+
# not any link was completed -- the caller resumes scanning at end_index + 1 either way.
|
|
137
|
+
# first_run_end is the last digit of the very first digit run, used only by guard M4.
|
|
138
|
+
Chain = Struct.new(:end_index, :first_run_end, :links)
|
|
139
|
+
|
|
140
|
+
# Reads the chain greedily and unconditionally; guard decisions happen afterward in
|
|
141
|
+
# chain_edits, never here, so the scan shape itself cannot express "how many links
|
|
142
|
+
# converted" -- only "whole chain or nothing" (symbols.md 3.3 step 2, 5).
|
|
143
|
+
def self.read_chain(cp, a)
|
|
144
|
+
p = a
|
|
145
|
+
p += 1 while digit?(at(cp, p))
|
|
146
|
+
first_run_end = p - 1
|
|
147
|
+
end_index = p - 1
|
|
148
|
+
links = []
|
|
149
|
+
|
|
150
|
+
loop do
|
|
151
|
+
q = p
|
|
152
|
+
left_space = 0
|
|
153
|
+
if space_like?(at(cp, q))
|
|
154
|
+
left_space = 1
|
|
155
|
+
q += 1
|
|
156
|
+
end
|
|
157
|
+
break unless mul_letter?(at(cp, q))
|
|
158
|
+
|
|
159
|
+
letter_index = q
|
|
160
|
+
q += 1
|
|
161
|
+
right_space = 0
|
|
162
|
+
if space_like?(at(cp, q))
|
|
163
|
+
right_space = 1
|
|
164
|
+
q += 1
|
|
165
|
+
end
|
|
166
|
+
break unless digit?(at(cp, q))
|
|
167
|
+
|
|
168
|
+
q += 1 while digit?(at(cp, q))
|
|
169
|
+
links << ChainLink.new(letter_index, left_space, right_space)
|
|
170
|
+
end_index = q - 1
|
|
171
|
+
p = q
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
Chain.new(end_index, first_run_end, links)
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
# Applies guards M1-M4 to a chain read by read_chain. Returns the edits, or nil when the
|
|
178
|
+
# chain is declined whole -- symbols.md 3.3 never half-converts a chain.
|
|
179
|
+
def self.chain_edits(cp, a, chain)
|
|
180
|
+
links = chain.links
|
|
181
|
+
first = links.first
|
|
182
|
+
return nil if first.nil?
|
|
183
|
+
|
|
184
|
+
# M1 -- every link symmetric, and every link agreeing with the first. "5x4 x 3" is
|
|
185
|
+
# ambiguous input and is declined whole rather than half-converted.
|
|
186
|
+
sp = first.left_space
|
|
187
|
+
links.each do |link|
|
|
188
|
+
return nil if link.left_space != link.right_space
|
|
189
|
+
return nil if link.left_space != sp
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
# M2/M3 -- the chain's OUTER boundaries, not each link. Applying them per link is
|
|
193
|
+
# exactly what made the pairwise form reject chains: the letter past the middle digit
|
|
194
|
+
# run is itself a MUL-LETTER, hence a LETTER.
|
|
195
|
+
before = at(cp, a - 1)
|
|
196
|
+
after = at(cp, chain.end_index + 1)
|
|
197
|
+
return nil if before != NONE && UnicodeUtil.letter?(before)
|
|
198
|
+
return nil if after != NONE && UnicodeUtil.letter?(after)
|
|
199
|
+
|
|
200
|
+
# M4 -- hexadecimal literal veto. Latin lowercase only: a hex literal is never
|
|
201
|
+
# written with Cyrillic. Inspects the first link alone.
|
|
202
|
+
if sp.zero? && at(cp, first.letter_index) == LOWER_X &&
|
|
203
|
+
chain.first_run_end == a && at(cp, a) == DIGIT_ZERO
|
|
204
|
+
return nil
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
# There is no M5. It was removed rather than extended (symbols.md 3.3 step 7, 7.3).
|
|
208
|
+
edits = []
|
|
209
|
+
links.each do |link|
|
|
210
|
+
j = link.letter_index
|
|
211
|
+
replacement =
|
|
212
|
+
sp.zero? ? [MULTIPLICATION] : [at(cp, j - 1), MULTIPLICATION, at(cp, j + 1)]
|
|
213
|
+
start_index = j - sp
|
|
214
|
+
end_span = j + sp + 1
|
|
215
|
+
next if replacement == cp[start_index...end_span]
|
|
216
|
+
|
|
217
|
+
edits << Edit.new(start_index, end_span, replacement, "symbols")
|
|
218
|
+
end
|
|
219
|
+
edits
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
# symbols.md 3.4. Only the literal "+/-"; the bare "+-" is never converted, in any
|
|
223
|
+
# context (3.4, 7.11).
|
|
224
|
+
def self.plus_minus_at(cp, i)
|
|
225
|
+
return nil if at(cp, i + 1) != SOLIDUS || at(cp, i + 2) != HYPHEN_MINUS
|
|
226
|
+
|
|
227
|
+
# F1 -- not a character class: "[+/-]" is a regular expression.
|
|
228
|
+
return nil if at(cp, i - 1) == SQUARE_OPEN
|
|
229
|
+
|
|
230
|
+
# F2 -- numeric context, with one optional intervening space so "+/-5" and "+/- 5"
|
|
231
|
+
# both work. Without it, prose that names the characters ("lines marked +/- were
|
|
232
|
+
# edited") is corrupted.
|
|
233
|
+
j = i + 3
|
|
234
|
+
j += 1 if at(cp, j) == SPACE
|
|
235
|
+
return nil unless digit?(at(cp, j))
|
|
236
|
+
|
|
237
|
+
Edit.new(i, i + 3, [PLUS_MINUS], "symbols")
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
# symbols.md 3.5: one left-to-right scan. The three branches key on different code
|
|
241
|
+
# points -- U+0028, a DIGIT, U+002B -- so no two can match at the same index; on a
|
|
242
|
+
# successful edit the scan continues from the index after the matched span.
|
|
243
|
+
#
|
|
244
|
+
# locale_data is unused: order.json declares "localeData": [] for this rule.
|
|
245
|
+
def self.scan(cp, _locale_data, _ctx)
|
|
246
|
+
n = cp.length
|
|
247
|
+
edits = []
|
|
248
|
+
i = 0
|
|
249
|
+
|
|
250
|
+
while i < n
|
|
251
|
+
current = at(cp, i)
|
|
252
|
+
|
|
253
|
+
if current == PAREN_OPEN
|
|
254
|
+
edit = trademark_at(cp, i)
|
|
255
|
+
if edit
|
|
256
|
+
edits << edit
|
|
257
|
+
i = edit.end
|
|
258
|
+
next
|
|
259
|
+
end
|
|
260
|
+
elsif digit?(current) && !digit?(at(cp, i - 1))
|
|
261
|
+
# Keyed on the start of a maximal digit run, not on the letter.
|
|
262
|
+
chain = read_chain(cp, i)
|
|
263
|
+
chain_result = chain_edits(cp, i, chain)
|
|
264
|
+
edits.concat(chain_result) if chain_result
|
|
265
|
+
# Continue past the chain whether or not it converted. A declined chain has no
|
|
266
|
+
# convertible sub-chain: any sub-chain starts right after a MUL-LETTER, which is
|
|
267
|
+
# a LETTER, so M2 would reject it too.
|
|
268
|
+
i = chain.end_index + 1
|
|
269
|
+
next
|
|
270
|
+
elsif current == PLUS
|
|
271
|
+
edit = plus_minus_at(cp, i)
|
|
272
|
+
if edit
|
|
273
|
+
edits << edit
|
|
274
|
+
i = edit.end
|
|
275
|
+
next
|
|
276
|
+
end
|
|
277
|
+
end
|
|
278
|
+
i += 1
|
|
279
|
+
end
|
|
280
|
+
|
|
281
|
+
edits
|
|
282
|
+
end
|
|
283
|
+
end
|
|
284
|
+
end
|
|
285
|
+
end
|
|
286
|
+
end
|
|
287
|
+
|
|
288
|
+
Polytypo::Engine::Registry.register(
|
|
289
|
+
"symbols",
|
|
290
|
+
->(cp, locale_data, ctx) { Polytypo::Engine::Rules::Symbols.scan(cp, locale_data, ctx) },
|
|
291
|
+
)
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Requiring this file is enough to register every rule id with Registry -- mirrors the other
|
|
4
|
+
# three ports' own "importing the rules package/module is enough, no separate caller-side step"
|
|
5
|
+
# pattern. Order here does not matter (registration order is never pipeline order -- that comes
|
|
6
|
+
# from order.json, ARCHITECTURE.md section 4.5), but dash_shared and quote_ambiguity are required
|
|
7
|
+
# first since dashes/ranges and quotes/apostrophe depend on them.
|
|
8
|
+
require_relative "rules/dash_shared"
|
|
9
|
+
require_relative "rules/quote_ambiguity"
|
|
10
|
+
|
|
11
|
+
require_relative "rules/spaces"
|
|
12
|
+
require_relative "rules/ellipsis"
|
|
13
|
+
require_relative "rules/ranges"
|
|
14
|
+
require_relative "rules/dashes"
|
|
15
|
+
require_relative "rules/hyphen"
|
|
16
|
+
require_relative "rules/quotes"
|
|
17
|
+
require_relative "rules/apostrophe"
|
|
18
|
+
require_relative "rules/symbols"
|
|
19
|
+
require_relative "rules/nbsp"
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Polytypo
|
|
4
|
+
module Engine
|
|
5
|
+
# The three non-code-point values a rule can meet in the array it scans. They live together
|
|
6
|
+
# and must stay pairwise disjoint (mirrors src/engine/sentinels.ts in the JS reference
|
|
7
|
+
# implementation and its Python/Go equivalents):
|
|
8
|
+
#
|
|
9
|
+
# - NONE — there is nothing at that index; the array ends here.
|
|
10
|
+
# - MARKER — a span boundary whose skipped region has no line terminator. Per modes.md 3.3
|
|
11
|
+
# it is opaque content everywhere except OPENISH/CLOSEISH, where it is a member of both.
|
|
12
|
+
# - LINE_MARKER — a span boundary whose skipped region contains a line terminator. A member
|
|
13
|
+
# of BREAK for every rule, everywhere.
|
|
14
|
+
MARKER = -1
|
|
15
|
+
LINE_MARKER = -2
|
|
16
|
+
NONE = -3
|
|
17
|
+
|
|
18
|
+
# True for either span boundary. NONE is deliberately not a marker: it is not in the array.
|
|
19
|
+
def self.marker?(value)
|
|
20
|
+
value == MARKER || value == LINE_MARKER
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|