polytypo 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +109 -0
- data/lib/polytypo/data/README.md +20 -0
- data/lib/polytypo/data/UNICODE +1 -0
- data/lib/polytypo/data/VERSION +1 -0
- data/lib/polytypo/data/fixtures/de-CH.json +501 -0
- data/lib/polytypo/data/fixtures/de-DE.json +547 -0
- data/lib/polytypo/data/fixtures/el.json +239 -0
- data/lib/polytypo/data/fixtures/en-GB.json +1274 -0
- data/lib/polytypo/data/fixtures/en-US.json +1807 -0
- data/lib/polytypo/data/fixtures/fi.json +1306 -0
- data/lib/polytypo/data/fixtures/fr-CA.json +268 -0
- data/lib/polytypo/data/fixtures/fr.json +603 -0
- data/lib/polytypo/data/fixtures/locale-resolution.json +209 -0
- data/lib/polytypo/data/fixtures/ru.json +688 -0
- data/lib/polytypo/data/fixtures/sv.json +1290 -0
- data/lib/polytypo/data/locales/de-CH.json +77 -0
- data/lib/polytypo/data/locales/de-DE.json +76 -0
- data/lib/polytypo/data/locales/el.json +90 -0
- data/lib/polytypo/data/locales/en-GB.json +115 -0
- data/lib/polytypo/data/locales/en-US.json +133 -0
- data/lib/polytypo/data/locales/fi.json +136 -0
- data/lib/polytypo/data/locales/fr-CA.json +78 -0
- data/lib/polytypo/data/locales/fr.json +84 -0
- data/lib/polytypo/data/locales/registry.json +9 -0
- data/lib/polytypo/data/locales/ru.json +112 -0
- data/lib/polytypo/data/locales/sv.json +124 -0
- data/lib/polytypo/data/rules/dashes.md +1238 -0
- data/lib/polytypo/data/rules/order.json +78 -0
- data/lib/polytypo/data/schema/fixtures.schema.json +79 -0
- data/lib/polytypo/data/schema/locale.schema.json +235 -0
- data/lib/polytypo/data/schema/registry.schema.json +29 -0
- data/lib/polytypo/data/schema/resolution.schema.json +50 -0
- data/lib/polytypo/engine/codepoints.rb +24 -0
- data/lib/polytypo/engine/edits.rb +64 -0
- data/lib/polytypo/engine/locale.rb +138 -0
- data/lib/polytypo/engine/pipeline.rb +61 -0
- data/lib/polytypo/engine/registry.rb +47 -0
- data/lib/polytypo/engine/rules/apostrophe.rb +127 -0
- data/lib/polytypo/engine/rules/dash_shared.rb +342 -0
- data/lib/polytypo/engine/rules/dashes.rb +125 -0
- data/lib/polytypo/engine/rules/ellipsis.rb +100 -0
- data/lib/polytypo/engine/rules/hyphen.rb +207 -0
- data/lib/polytypo/engine/rules/nbsp.rb +616 -0
- data/lib/polytypo/engine/rules/quote_ambiguity.rb +241 -0
- data/lib/polytypo/engine/rules/quotes.rb +420 -0
- data/lib/polytypo/engine/rules/ranges.rb +124 -0
- data/lib/polytypo/engine/rules/spaces.rb +232 -0
- data/lib/polytypo/engine/rules/symbols.rb +291 -0
- data/lib/polytypo/engine/rules.rb +19 -0
- data/lib/polytypo/engine/sentinels.rb +23 -0
- data/lib/polytypo/engine/unicode_util.rb +390 -0
- data/lib/polytypo/errors.rb +24 -0
- data/lib/polytypo/modes/html.rb +233 -0
- data/lib/polytypo/modes/markdown.rb +187 -0
- data/lib/polytypo/modes/parse_error.rb +19 -0
- data/lib/polytypo/modes/runner.rb +57 -0
- data/lib/polytypo/modes/spans.rb +132 -0
- data/lib/polytypo/version.rb +5 -0
- data/lib/polytypo.rb +91 -0
- data/polytypo.gemspec +37 -0
- metadata +122 -0
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../sentinels"
|
|
4
|
+
require_relative "../unicode_util"
|
|
5
|
+
require_relative "../codepoints"
|
|
6
|
+
|
|
7
|
+
module Polytypo
|
|
8
|
+
module Engine
|
|
9
|
+
module Rules
|
|
10
|
+
# Shared ambiguous-medial-span predicate -- spec/rules/quotes.md 3.2 ("Listed elision veto"
|
|
11
|
+
# and "General ambiguous-medial-span veto") and spec/rules/apostrophe.md 3.4. One
|
|
12
|
+
# definition, used identically by quotes.rb and apostrophe.rb, so the two rules cannot
|
|
13
|
+
# drift apart on what counts as ambiguous (mirrors ref-js's src/rules/quote-ambiguity.ts,
|
|
14
|
+
# the Python port's _quote_ambiguity.py and the Go port's quote_ambiguity.go).
|
|
15
|
+
#
|
|
16
|
+
# The shape (quotes.md 3.2, "General ambiguous-medial-span veto"): a pair of straight ASCII
|
|
17
|
+
# single quotes (U+0027) enclosing 1-3 LETTER code points, with at least one INLINE-SPACE
|
|
18
|
+
# code point immediately outside each mark -- `rock 'n' roll`, `She chose 'A' today`. Only
|
|
19
|
+
# the single adjacent code point is tested on each side; a longer run of inline spaces
|
|
20
|
+
# further out does not invalidate the match (quotes.md 3.2's "at least one, deliberately
|
|
21
|
+
# not exactly one").
|
|
22
|
+
#
|
|
23
|
+
# Without a matching quotes.elisionIdioms entry, neither quotes nor apostrophe may touch
|
|
24
|
+
# either mark: quotes must not pair them as an ordinary quotation, and apostrophe's own case
|
|
25
|
+
# ladder (which would otherwise independently read the left mark as a leading elision and
|
|
26
|
+
# the right one as a trailing possessive/elision, apostrophe.md 3.3 cases 3/4) must not
|
|
27
|
+
# convert them either.
|
|
28
|
+
module QuoteAmbiguity
|
|
29
|
+
# STRAIGHT_APOSTROPHE is the GENERAL ambiguous-shape veto's own trigger glyph -- U+0027
|
|
30
|
+
# only. An already-curly U+2018/U+2019 pair is out of this predicate's scope by
|
|
31
|
+
# construction.
|
|
32
|
+
STRAIGHT_APOSTROPHE = 0x27
|
|
33
|
+
|
|
34
|
+
# NARROW -- quotes.md 3.1 NARROW -- every glyph an elision idiom's marks may appear as
|
|
35
|
+
# across pipeline passes (straight, or already curled by an earlier pass). Shared with
|
|
36
|
+
# quotes.rb so the two rules cannot define two slightly different NARROW sets.
|
|
37
|
+
NARROW = [0x27, 0x2018, 0x2019, 0x201A, 0x201B, 0x2039, 0x203A].freeze
|
|
38
|
+
|
|
39
|
+
# INLINE_SPACE -- quotes.md 3.1 INLINE-SPACE, deliberately excluding BREAK/MARKER/
|
|
40
|
+
# LINE_MARKER so this shape never crosses a line or span boundary (modes.md 3.3) -- the
|
|
41
|
+
# same anchor the elisionIdioms matcher already uses.
|
|
42
|
+
INLINE_SPACE = [0x20, 0x09, 0xA0, 0x202F, 0x2007, 0x2009, 0x200A].freeze
|
|
43
|
+
|
|
44
|
+
MIN_ENCLOSED = 1
|
|
45
|
+
MAX_ENCLOSED = 3
|
|
46
|
+
|
|
47
|
+
def self.narrow?(cp)
|
|
48
|
+
NARROW.include?(cp)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def self.inline_space?(cp)
|
|
52
|
+
INLINE_SPACE.include?(cp)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# at(cp, i) returns cp[i], or Engine::NONE if i is out of bounds -- the spec's own
|
|
56
|
+
# boundary value.
|
|
57
|
+
def self.at(cp, i)
|
|
58
|
+
return Engine::NONE if i.negative? || i >= cp.length
|
|
59
|
+
|
|
60
|
+
cp[i]
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def self.alnum?(cp)
|
|
64
|
+
(cp >= 0x30 && cp <= 0x39) || UnicodeUtil.letter?(cp)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# ascii_lower folds ASCII A-Z to a-z, ASCII-only -- the same convention nbsp's
|
|
68
|
+
# afterShortWords and the existing idiom matcher already use (ARCHITECTURE.md 4.4: never
|
|
69
|
+
# a platform locale case-fold).
|
|
70
|
+
def self.ascii_lower(cp)
|
|
71
|
+
cp >= 0x41 && cp <= 0x5A ? cp + 0x20 : cp
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# elided_matches? compares cp[start, elided.length] against elided exactly, code point
|
|
75
|
+
# for code point -- no case leniency, ever, on the elided content (quotes.md 3.2). A
|
|
76
|
+
# MARKER/LINE_MARKER/NONE sentinel (a negative value, not a valid code point) is never
|
|
77
|
+
# treated as if it could equal a real code point: every comparison here goes through
|
|
78
|
+
# Codepoints.valid_codepoint? first, and this method never assembles the candidate span
|
|
79
|
+
# into a Ruby String (no `.pack("U*")`, no `.chr`) -- comparison stays code point vs.
|
|
80
|
+
# code point throughout. This is the exact defensive check the Python port needed after a
|
|
81
|
+
# real crash (`ValueError: chr() arg not in range`) when a span straddling a span-
|
|
82
|
+
# boundary marker was packed into a string and compared to a literal.
|
|
83
|
+
def self.elided_matches?(cp, start, elided)
|
|
84
|
+
elided.each_with_index do |want, w|
|
|
85
|
+
c = at(cp, start + w)
|
|
86
|
+
return false unless Codepoints.valid_codepoint?(c)
|
|
87
|
+
return false unless c == want
|
|
88
|
+
end
|
|
89
|
+
true
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# word_ends_at? reports whether the word.length code points immediately before index
|
|
93
|
+
# `end_` (exclusive) match word exactly -- except the first code point, compared
|
|
94
|
+
# ASCII-case-insensitively -- and have a legal outer (left) word boundary: NONE, or not
|
|
95
|
+
# LETTER/DIGIT (quotes.md 3.2's Word definition). The caller has already verified the
|
|
96
|
+
# code point at `end_` itself is a legal right-hand boundary (a single INLINE-SPACE code
|
|
97
|
+
# point). Guarded against a marker exactly like elided_matches? above.
|
|
98
|
+
def self.word_ends_at?(cp, end_, word)
|
|
99
|
+
start = end_ - word.length
|
|
100
|
+
return false if start.negative?
|
|
101
|
+
|
|
102
|
+
word.each_with_index do |want, k|
|
|
103
|
+
c = at(cp, start + k)
|
|
104
|
+
return false unless Codepoints.valid_codepoint?(c)
|
|
105
|
+
|
|
106
|
+
if k.zero?
|
|
107
|
+
return false unless ascii_lower(c) == ascii_lower(want)
|
|
108
|
+
else
|
|
109
|
+
return false unless c == want
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
before = at(cp, start - 1)
|
|
113
|
+
before == Engine::NONE || !alnum?(before)
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# word_starts_at? is word_ends_at?'s mirror image: word must start exactly at `start`,
|
|
117
|
+
# first code point ASCII-case-insensitive, with a legal outer (right) word boundary
|
|
118
|
+
# immediately after it.
|
|
119
|
+
def self.word_starts_at?(cp, start, word)
|
|
120
|
+
n = cp.length
|
|
121
|
+
word.each_with_index do |want, k|
|
|
122
|
+
c = at(cp, start + k)
|
|
123
|
+
return false unless Codepoints.valid_codepoint?(c)
|
|
124
|
+
|
|
125
|
+
if k.zero?
|
|
126
|
+
return false unless ascii_lower(c) == ascii_lower(want)
|
|
127
|
+
else
|
|
128
|
+
return false unless c == want
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
after = start + word.length
|
|
132
|
+
return true if after >= n
|
|
133
|
+
|
|
134
|
+
!alnum?(cp[after])
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# compute_idiom_matched_indices is the listed elision veto (quotes.md 3.2, spec 0.4.0),
|
|
138
|
+
# locale data quotes.elisionIdioms. Bounded literal scan for `left, NARROW, elided,
|
|
139
|
+
# NARROW, right` (`rock 'n' roll`'s {left: "rock", elided: "n", right: "roll"}). Both
|
|
140
|
+
# marks of a match are returned (as Hash keys, used as a set). Matches on NARROW quote
|
|
141
|
+
# marks generally (U+0027 and already-curly U+2018/U+2019), not only straight ASCII --
|
|
142
|
+
# required for quotes' own idempotency (an idiom must still veto pairing on a second
|
|
143
|
+
# pipeline pass, after apostrophe has curled the marks).
|
|
144
|
+
#
|
|
145
|
+
# idioms is the locale's quotes.elisionIdioms array: a list of Hashes with String keys
|
|
146
|
+
# "left"/"elided"/"right", each a literal String (locale.schema.json).
|
|
147
|
+
def self.compute_idiom_matched_indices(cp, idioms)
|
|
148
|
+
vetoed = {}
|
|
149
|
+
return vetoed if idioms.nil? || idioms.empty?
|
|
150
|
+
|
|
151
|
+
n = cp.length
|
|
152
|
+
compiled = idioms.map do |idiom|
|
|
153
|
+
{
|
|
154
|
+
left: idiom["left"].codepoints,
|
|
155
|
+
elided: idiom["elided"].codepoints,
|
|
156
|
+
right: idiom["right"].codepoints
|
|
157
|
+
}
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
(0...n).each do |i|
|
|
161
|
+
g = cp[i]
|
|
162
|
+
next unless narrow?(g)
|
|
163
|
+
|
|
164
|
+
l_lit = at(cp, i - 1)
|
|
165
|
+
next if l_lit == Engine::NONE || !inline_space?(l_lit)
|
|
166
|
+
|
|
167
|
+
compiled.each do |idiom|
|
|
168
|
+
k = idiom[:elided].length
|
|
169
|
+
j = i + 1 + k
|
|
170
|
+
next if j >= n
|
|
171
|
+
next unless elided_matches?(cp, i + 1, idiom[:elided])
|
|
172
|
+
next unless narrow?(cp[j])
|
|
173
|
+
|
|
174
|
+
r_lit = at(cp, j + 1)
|
|
175
|
+
next if r_lit == Engine::NONE || !inline_space?(r_lit)
|
|
176
|
+
|
|
177
|
+
next unless word_ends_at?(cp, i - 1, idiom[:left])
|
|
178
|
+
next unless word_starts_at?(cp, j + 2, idiom[:right])
|
|
179
|
+
|
|
180
|
+
vetoed[i] = true
|
|
181
|
+
vetoed[j] = true
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
vetoed
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
# compute_ambiguous_shape_indices is the general ambiguous-medial-span shape,
|
|
189
|
+
# locale-independent (quotes.md 3.2, spec 0.5.0): a pair of straight ASCII single quotes
|
|
190
|
+
# (U+0027 only) enclosing 1-3 LETTER code points, with at least one INLINE-SPACE code
|
|
191
|
+
# point immediately outside each mark. Both mark positions are returned for every match.
|
|
192
|
+
# A superset of compute_idiom_matched_indices's output whenever an idiom's elided field
|
|
193
|
+
# is itself 1-3 letters (true of every idiom shipped so far), but computed independently
|
|
194
|
+
# rather than assumed, since a future idiom's elided field is not required to be that
|
|
195
|
+
# short.
|
|
196
|
+
def self.compute_ambiguous_shape_indices(cp)
|
|
197
|
+
ambiguous = {}
|
|
198
|
+
n = cp.length
|
|
199
|
+
|
|
200
|
+
(0...n).each do |i|
|
|
201
|
+
next unless at(cp, i) == STRAIGHT_APOSTROPHE
|
|
202
|
+
|
|
203
|
+
l_lit = at(cp, i - 1)
|
|
204
|
+
next if l_lit == Engine::NONE || !inline_space?(l_lit)
|
|
205
|
+
|
|
206
|
+
k = 0
|
|
207
|
+
k += 1 while k < MAX_ENCLOSED && UnicodeUtil.letter?(at(cp, i + 1 + k))
|
|
208
|
+
next if k < MIN_ENCLOSED
|
|
209
|
+
|
|
210
|
+
j = i + 1 + k
|
|
211
|
+
next unless at(cp, j) == STRAIGHT_APOSTROPHE
|
|
212
|
+
|
|
213
|
+
r_lit = at(cp, j + 1)
|
|
214
|
+
next if r_lit == Engine::NONE || !inline_space?(r_lit)
|
|
215
|
+
|
|
216
|
+
ambiguous[i] = true
|
|
217
|
+
ambiguous[j] = true
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
ambiguous
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
# compute_preserve_indices is the set of straight-ASCII-quote index positions
|
|
224
|
+
# apostrophe.md 3.4 requires `apostrophe` to skip byte-identically: ambiguous-shaped, but
|
|
225
|
+
# with no matching cited idiom. A position with a matching idiom is not in this set --
|
|
226
|
+
# apostrophe's ordinary case ladder still curls it, exactly as spec 0.4.0-0.4.1 did.
|
|
227
|
+
def self.compute_preserve_indices(cp, idioms)
|
|
228
|
+
ambiguous = compute_ambiguous_shape_indices(cp)
|
|
229
|
+
return ambiguous if ambiguous.empty?
|
|
230
|
+
|
|
231
|
+
idiom_matched = compute_idiom_matched_indices(cp, idioms)
|
|
232
|
+
return ambiguous if idiom_matched.empty?
|
|
233
|
+
|
|
234
|
+
ambiguous.each_with_object({}) do |(idx, _), preserve|
|
|
235
|
+
preserve[idx] = true unless idiom_matched.key?(idx)
|
|
236
|
+
end
|
|
237
|
+
end
|
|
238
|
+
end
|
|
239
|
+
end
|
|
240
|
+
end
|
|
241
|
+
end
|
|
@@ -0,0 +1,420 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "quote_ambiguity"
|
|
4
|
+
require_relative "../sentinels"
|
|
5
|
+
require_relative "../unicode_util"
|
|
6
|
+
require_relative "../edits"
|
|
7
|
+
require_relative "../registry"
|
|
8
|
+
|
|
9
|
+
module Polytypo
|
|
10
|
+
module Engine
|
|
11
|
+
module Rules
|
|
12
|
+
# spec/rules/quotes.md (spec 0.5.0), order 40.
|
|
13
|
+
#
|
|
14
|
+
# Mandate 1 (every existing quote glyph is a re-typesetting candidate) and mandate 2 (a
|
|
15
|
+
# space touching a quote mark is sloppiness, not evidence) are this rule's whole
|
|
16
|
+
# architecture. Five passes plus an emit; no backtracking inside a pass, no regular
|
|
17
|
+
# expression, no native-string indexing (ARCHITECTURE.md 4.1, 4.2).
|
|
18
|
+
#
|
|
19
|
+
# Pass 1 (collect_candidates) -- collect and classify candidates (quotes.md 3.2)
|
|
20
|
+
# Pass 2 (pair_candidates) -- pair the candidates, one stack per width (3.3)
|
|
21
|
+
# Pass 3 (depth_of) -- assign glyphs by depth, folded into the render plan (3.4)
|
|
22
|
+
# Pass 4 (certify) -- the certification gate (3.5)
|
|
23
|
+
# Pass 5 (emit) -- emit edits (3.6)
|
|
24
|
+
module Quotes
|
|
25
|
+
Candidate = Struct.new(:index, :wide, :can_open, :can_close)
|
|
26
|
+
Pair = Struct.new(:open, :close)
|
|
27
|
+
|
|
28
|
+
# WIDE is quotes.md 3.1 WIDE. NARROW is QuoteAmbiguity::NARROW, shared with apostrophe so
|
|
29
|
+
# the two rules cannot define two slightly different NARROW sets. WIDE and NARROW are
|
|
30
|
+
# disjoint and their union is QUOTEMARK.
|
|
31
|
+
WIDE = [0x22, 0xAB, 0xBB, 0x201C, 0x201D, 0x201E, 0x201F, 0x301D, 0x301E, 0x301F].freeze
|
|
32
|
+
|
|
33
|
+
BREAK = [0x0A, 0x0D, 0x0B, 0x0C, 0x85, 0x2028, 0x2029, Engine::LINE_MARKER].freeze
|
|
34
|
+
DASHISH = [0x2D, 0x2011, 0x2013, 0x2014].freeze
|
|
35
|
+
OPENISH_LITERAL = [0x28, 0x5B, 0x7B].freeze
|
|
36
|
+
CLOSEISH_LITERAL = [0x29, 0x5D, 0x7D, 0x2C, 0x2E, 0x3B, 0x3A, 0x21, 0x3F, 0x2026, 0x2013, 0x2014].freeze
|
|
37
|
+
|
|
38
|
+
def self.quote_mark?(cp)
|
|
39
|
+
WIDE.include?(cp) || QuoteAmbiguity.narrow?(cp)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def self.digit?(cp)
|
|
43
|
+
cp >= 0x30 && cp <= 0x39
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def self.alnum?(cp)
|
|
47
|
+
digit?(cp) || UnicodeUtil.letter?(cp)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def self.break?(cp)
|
|
51
|
+
BREAK.include?(cp)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# SPACELIKE = INLINE-SPACE ∪ BREAK.
|
|
55
|
+
def self.spacelike?(cp)
|
|
56
|
+
QuoteAmbiguity.inline_space?(cp) || break?(cp)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# OPENISH. QUOTEMARK is a member of both OPENISH and CLOSEISH, and is exempt from
|
|
60
|
+
# canOpen's closeish rejection -- Lemma A's entire mechanism (quotes.md 5) and the reason
|
|
61
|
+
# a candidate's verdict never depends on which quote glyph its neighbour is. Do not
|
|
62
|
+
# "simplify" this back to per-glyph lists. MARKER is a member too (modes.md 3.3).
|
|
63
|
+
def self.openish?(cp)
|
|
64
|
+
return true if cp == Engine::MARKER
|
|
65
|
+
|
|
66
|
+
OPENISH_LITERAL.include?(cp) || quote_mark?(cp)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# CLOSEISH -- see openish?'s dual-membership note.
|
|
70
|
+
def self.closeish?(cp)
|
|
71
|
+
return true if cp == Engine::MARKER
|
|
72
|
+
|
|
73
|
+
CLOSEISH_LITERAL.include?(cp) || quote_mark?(cp)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def self.dashish?(cp)
|
|
77
|
+
DASHISH.include?(cp)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# DELETE-LANDING is the largest landing class for which every earlier-ordered rule's own
|
|
81
|
+
# classes are unaffected by a quote glyph or a U+0020 (quotes.md 3.7, composition
|
|
82
|
+
# obligation).
|
|
83
|
+
def self.delete_landing?(cp)
|
|
84
|
+
alnum?(cp) || quote_mark?(cp)
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# V1ID (quotes.md 3.2, spec 0.4.1) -- a conservative over-approximation, not a claim that
|
|
88
|
+
# every U+0027 becomes U+2019. apostrophe only ever emits U+2019 for a U+0027, but its own
|
|
89
|
+
# case ladder leaves some U+0027s unedited (the prime guard, and "nothing inferable"), and
|
|
90
|
+
# quotes cannot know which without re-deriving apostrophe's verdict against quotes' own
|
|
91
|
+
# not-yet-final output -- circular. V1ID treats every U+0027 as possibly about to become
|
|
92
|
+
# U+2019, and every U+2019 as possibly a U+0027 that already did, so V1's comparison stays
|
|
93
|
+
# invariant across the two rules running in sequence on successive pipeline passes
|
|
94
|
+
# (quotes.md 5, Lemma A).
|
|
95
|
+
def self.v1_identity(cp)
|
|
96
|
+
cp == 0x27 ? 0x2019 : cp
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# glyph_codepoint decodes a locale-declared quote glyph (guaranteed by schema to be
|
|
100
|
+
# exactly one code point) to its code point value.
|
|
101
|
+
def self.glyph_codepoint(str)
|
|
102
|
+
str.ord
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# compute_skip_sets is quotes.md 3.1a -- locale-derived skip sets, computed once per call.
|
|
106
|
+
# These are exactly the positions at which nbsp's N8 can insert a space (nbsp.md 3.10),
|
|
107
|
+
# which is what makes Lemma B's coverage exact rather than a survey.
|
|
108
|
+
def self.compute_skip_sets(quotes_data)
|
|
109
|
+
space_right = {}
|
|
110
|
+
space_left = {}
|
|
111
|
+
[quotes_data["primary"], quotes_data["secondary"]].each do |pair|
|
|
112
|
+
next if pair["innerSpace"] == "none"
|
|
113
|
+
|
|
114
|
+
open_cp = glyph_codepoint(pair["open"])
|
|
115
|
+
close_cp = glyph_codepoint(pair["close"])
|
|
116
|
+
next if open_cp == close_cp
|
|
117
|
+
|
|
118
|
+
space_right[open_cp] = true
|
|
119
|
+
space_left[close_cp] = true
|
|
120
|
+
end
|
|
121
|
+
{ space_right: space_right, space_left: space_left }
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# skip_left/skip_right are the straight-line walk of quotes.md 3.2: step outward across a
|
|
125
|
+
# maximal INLINE-SPACE run. MARKER and every BREAK stop it, because neither is in
|
|
126
|
+
# INLINE-SPACE.
|
|
127
|
+
def self.skip_left(cp, i)
|
|
128
|
+
j = i - 1
|
|
129
|
+
j -= 1 while j >= 0 && QuoteAmbiguity.inline_space?(cp[j])
|
|
130
|
+
j >= 0 ? cp[j] : Engine::NONE
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def self.skip_right(cp, i)
|
|
134
|
+
n = cp.length
|
|
135
|
+
j = i + 1
|
|
136
|
+
j += 1 while j < n && QuoteAmbiguity.inline_space?(cp[j])
|
|
137
|
+
j < n ? cp[j] : Engine::NONE
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# collect_candidates is pass 1 (quotes.md 3.2) -- collect and classify candidates.
|
|
141
|
+
# can_open always skips right and can_close always skips left (mandate 2's inner-side
|
|
142
|
+
# skip); the outer side skips only when nbsp can reach it (the locale-derived
|
|
143
|
+
# space_right/space_left sets), which is what keeps every verdict inert to nbsp (Lemma B).
|
|
144
|
+
def self.collect_candidates(cp, skip_sets, idioms)
|
|
145
|
+
n = cp.length
|
|
146
|
+
candidates = []
|
|
147
|
+
|
|
148
|
+
# spec 0.5.0: the veto set is the UNION of the cited-idiom match (unchanged since 0.4.0)
|
|
149
|
+
# and the general ambiguous-medial-span shape (quotes.md 3.2a) -- quotes must decline
|
|
150
|
+
# pairing for both, so apostrophe's own case ladder never independently "fixes" a shape
|
|
151
|
+
# quotes left alone.
|
|
152
|
+
idiom_matched = QuoteAmbiguity.compute_idiom_matched_indices(cp, idioms)
|
|
153
|
+
ambiguous_shape = QuoteAmbiguity.compute_ambiguous_shape_indices(cp)
|
|
154
|
+
elision_vetoed = idiom_matched.merge(ambiguous_shape)
|
|
155
|
+
|
|
156
|
+
(0...n).each do |i|
|
|
157
|
+
g = cp[i]
|
|
158
|
+
next unless quote_mark?(g)
|
|
159
|
+
|
|
160
|
+
l_lit = QuoteAmbiguity.at(cp, i - 1)
|
|
161
|
+
r_lit = QuoteAmbiguity.at(cp, i + 1)
|
|
162
|
+
l_skip = skip_left(cp, i)
|
|
163
|
+
r_skip = skip_right(cp, i)
|
|
164
|
+
|
|
165
|
+
open_left = skip_sets[:space_left][g] ? l_skip : l_lit
|
|
166
|
+
close_right = skip_sets[:space_right][g] ? r_skip : r_lit
|
|
167
|
+
|
|
168
|
+
can_open = (open_left == Engine::NONE || spacelike?(open_left) || openish?(open_left) || dashish?(open_left)) &&
|
|
169
|
+
r_skip != Engine::NONE && !spacelike?(r_skip) &&
|
|
170
|
+
(!closeish?(r_skip) || quote_mark?(r_skip) || r_skip == Engine::MARKER)
|
|
171
|
+
|
|
172
|
+
can_close = l_skip != Engine::NONE && !spacelike?(l_skip) &&
|
|
173
|
+
(close_right == Engine::NONE || spacelike?(close_right) || closeish?(close_right) || dashish?(close_right))
|
|
174
|
+
|
|
175
|
+
# Medial-elision veto (quotes.md 3.2), NARROW marks only, literal reads: don't,
|
|
176
|
+
# l'été, O'Brien, 1990's -- and, on a second pipeline pass, don't with U+2019, because
|
|
177
|
+
# apostrophe has converted the mark and U+2019 is also NARROW.
|
|
178
|
+
if QuoteAmbiguity.narrow?(g) && l_lit != Engine::NONE && r_lit != Engine::NONE &&
|
|
179
|
+
alnum?(l_lit) && alnum?(r_lit)
|
|
180
|
+
can_open = false
|
|
181
|
+
can_close = false
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# Listed + general ambiguous-shape veto (quotes.md 3.2, spec 0.4.0/0.5.0): both
|
|
185
|
+
# capabilities forced false, overriding every other test in this loop.
|
|
186
|
+
if elision_vetoed.key?(i)
|
|
187
|
+
can_open = false
|
|
188
|
+
can_close = false
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
# V1 -- same-V1-identity adjacency veto (quotes.md 3.2), both widths: "", '', ««, ””,
|
|
192
|
+
# plus the same shape separated by exactly one INLINE-SPACE code point at a position
|
|
193
|
+
# nbsp can insert or remove (gap_insertable, scoped to Lemma B's two insertion sites).
|
|
194
|
+
g_v1 = v1_identity(g)
|
|
195
|
+
gap_insertable = skip_sets[:space_right][g] || skip_sets[:space_left][g]
|
|
196
|
+
left_vetoed = v1_identity(l_lit) == g_v1 ||
|
|
197
|
+
(l_lit != Engine::NONE && QuoteAmbiguity.inline_space?(l_lit) &&
|
|
198
|
+
v1_identity(l_skip) == g_v1 && gap_insertable)
|
|
199
|
+
right_vetoed = v1_identity(r_lit) == g_v1 ||
|
|
200
|
+
(r_lit != Engine::NONE && QuoteAmbiguity.inline_space?(r_lit) &&
|
|
201
|
+
v1_identity(r_skip) == g_v1 && gap_insertable)
|
|
202
|
+
if left_vetoed || right_vetoed
|
|
203
|
+
can_open = false
|
|
204
|
+
can_close = false
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
candidates << Candidate.new(i, WIDE.include?(g), can_open, can_close) if can_open || can_close
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
candidates
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
# vacuous? is quotes.md 3.3 vacuous(a, b). Vacuously true when b = a + 1 (Ruby's Range#all?
|
|
214
|
+
# on the empty range (a+1...a+1) is true, matching the spec directly).
|
|
215
|
+
def self.vacuous?(cp, a, b)
|
|
216
|
+
((a + 1)...b).all? { |k| QuoteAmbiguity.inline_space?(cp[k]) }
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
# pair_candidates is pass 2 (quotes.md 3.3) -- pair the candidates, one stack per width.
|
|
220
|
+
# Closing is tried before opening; a candidate reaches exactly one of three outcomes
|
|
221
|
+
# (paired, pushed, unmatched), and a closer that fails the vacuity condition falls through
|
|
222
|
+
# to step 2 and then step 3 rather than being discarded -- the exhaustive three-outcome
|
|
223
|
+
# shape the certification gate depends on.
|
|
224
|
+
def self.pair_candidates(cp, candidates)
|
|
225
|
+
wide_stack = []
|
|
226
|
+
narrow_stack = []
|
|
227
|
+
pairs = []
|
|
228
|
+
|
|
229
|
+
candidates.each do |c|
|
|
230
|
+
stack = c.wide ? wide_stack : narrow_stack
|
|
231
|
+
if c.can_close && !stack.empty?
|
|
232
|
+
top = stack.last
|
|
233
|
+
unless vacuous?(cp, top.index, c.index)
|
|
234
|
+
stack.pop
|
|
235
|
+
pairs << Pair.new(top.index, c.index)
|
|
236
|
+
next
|
|
237
|
+
end
|
|
238
|
+
end
|
|
239
|
+
stack.push(c) if c.can_open
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
pairs
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
# depth_of is pass 3's depth (quotes.md 3.4), computed over the accepted set, never the
|
|
246
|
+
# raw pass-2 output: on a second run the accepted set is the raw set, so a depth taken
|
|
247
|
+
# over the raw set on run 1 and the accepted set on run 2 would disagree whenever the gate
|
|
248
|
+
# declined anything.
|
|
249
|
+
def self.depth_of(pairs, p)
|
|
250
|
+
depth = 1
|
|
251
|
+
pairs.each do |q|
|
|
252
|
+
depth += 1 if q.open < p.open && p.close < q.close
|
|
253
|
+
end
|
|
254
|
+
depth
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
def self.pair_for(quotes_data, depth)
|
|
258
|
+
depth.odd? ? quotes_data["primary"] : quotes_data["secondary"]
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
# compute_render_plan is quotes.md 3.5 render's glyph/deletion plan, shared by the
|
|
262
|
+
# certification gate's hypothetical and the real emit (pass 5) -- the only difference
|
|
263
|
+
# between them is whether the plan is applied to a throwaway array or actually returned
|
|
264
|
+
# as edits.
|
|
265
|
+
def self.compute_render_plan(cp, accepted, quotes_data)
|
|
266
|
+
replace = {}
|
|
267
|
+
del = {}
|
|
268
|
+
n = cp.length
|
|
269
|
+
|
|
270
|
+
accepted.each do |p|
|
|
271
|
+
glyphs = pair_for(quotes_data, depth_of(accepted, p))
|
|
272
|
+
replace[p.open] = glyph_codepoint(glyphs["open"])
|
|
273
|
+
replace[p.close] = glyph_codepoint(glyphs["close"])
|
|
274
|
+
|
|
275
|
+
next unless glyphs["innerSpace"] == "none"
|
|
276
|
+
|
|
277
|
+
# Open-side run: the maximal INLINE-SPACE run starting at p.open + 1.
|
|
278
|
+
open_start = p.open + 1
|
|
279
|
+
open_end = open_start
|
|
280
|
+
open_end += 1 while open_end < n && QuoteAmbiguity.inline_space?(cp[open_end])
|
|
281
|
+
open_empty = open_end == open_start
|
|
282
|
+
open_landing = open_end < n ? cp[open_end] : Engine::NONE
|
|
283
|
+
|
|
284
|
+
# Close-side run: the maximal INLINE-SPACE run ending at p.close - 1.
|
|
285
|
+
close_end = p.close
|
|
286
|
+
close_start = close_end - 1
|
|
287
|
+
close_start -= 1 while close_start >= 0 && QuoteAmbiguity.inline_space?(cp[close_start])
|
|
288
|
+
close_start += 1
|
|
289
|
+
close_empty = close_start == close_end
|
|
290
|
+
close_landing = close_start - 1 >= 0 ? cp[close_start - 1] : Engine::NONE
|
|
291
|
+
|
|
292
|
+
# A run is deleted iff non-empty, its landing is in DELETE-LANDING, and it is not
|
|
293
|
+
# simultaneously both of the pair's runs -- a pair enclosing nothing but spaces
|
|
294
|
+
# deletes neither (quotes.md 3.5). Unreachable for an accepted pair given pass 2's
|
|
295
|
+
# vacuity condition, but the guard is cheap and the spec states it unconditionally.
|
|
296
|
+
same_run = !open_empty && !close_empty && open_start == close_start && open_end == close_end
|
|
297
|
+
|
|
298
|
+
if !open_empty && delete_landing?(open_landing) && !same_run
|
|
299
|
+
(open_start...open_end).each { |k| del[k] = true }
|
|
300
|
+
end
|
|
301
|
+
if !close_empty && delete_landing?(close_landing) && !same_run
|
|
302
|
+
(close_start...close_end).each { |k| del[k] = true }
|
|
303
|
+
end
|
|
304
|
+
end
|
|
305
|
+
|
|
306
|
+
{ replace: replace, del: del }
|
|
307
|
+
end
|
|
308
|
+
|
|
309
|
+
# apply_render_plan applies a render plan, returning the rendered array and the
|
|
310
|
+
# order-preserving index map from surviving input indices to output indices.
|
|
311
|
+
def self.apply_render_plan(cp, plan)
|
|
312
|
+
y = []
|
|
313
|
+
m = Array.new(cp.length, -1)
|
|
314
|
+
cp.each_with_index do |c, i|
|
|
315
|
+
next if plan[:del][i]
|
|
316
|
+
|
|
317
|
+
m[i] = y.length
|
|
318
|
+
y << (plan[:replace].key?(i) ? plan[:replace][i] : c)
|
|
319
|
+
end
|
|
320
|
+
[y, m]
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
# certify is pass 4, the certification gate (quotes.md 3.5): the accepted pairing is
|
|
324
|
+
# checked, not proved. Render the hypothetical output, re-run passes 1-2 on it, and
|
|
325
|
+
# decline pairs until the re-run reproduces the accepted set exactly. Declination is
|
|
326
|
+
# simultaneous per round, and when the intersection fails to shrink A, the pair with the
|
|
327
|
+
# greatest open index is forced out -- both clauses are normative, so two ports cannot
|
|
328
|
+
# disagree.
|
|
329
|
+
def self.certify(cp, initial_pairs, quotes_data, skip_sets, idioms)
|
|
330
|
+
accepted = initial_pairs.dup
|
|
331
|
+
# Each round accepts or strictly shrinks `accepted`; it is finite and the empty set
|
|
332
|
+
# accepts unconditionally, so the loop runs at most |A0| + 1 times (quotes.md 3.5). The
|
|
333
|
+
# bound below is a defensive safety net, not a normative one.
|
|
334
|
+
max_rounds = initial_pairs.length + 2
|
|
335
|
+
|
|
336
|
+
(0..max_rounds).each do
|
|
337
|
+
return accepted if accepted.empty?
|
|
338
|
+
|
|
339
|
+
plan = compute_render_plan(cp, accepted, quotes_data)
|
|
340
|
+
y, m = apply_render_plan(cp, plan)
|
|
341
|
+
rederived = pair_candidates(y, collect_candidates(y, skip_sets, idioms))
|
|
342
|
+
|
|
343
|
+
b_set = Set.new(rederived)
|
|
344
|
+
projected = accepted.map { |p| Pair.new(m[p.open], m[p.close]) }
|
|
345
|
+
proj_set = Set.new(projected)
|
|
346
|
+
|
|
347
|
+
return accepted if proj_set == b_set
|
|
348
|
+
|
|
349
|
+
survivors = []
|
|
350
|
+
accepted.each_with_index do |p, idx|
|
|
351
|
+
survivors << p if b_set.include?(projected[idx])
|
|
352
|
+
end
|
|
353
|
+
|
|
354
|
+
if survivors.length == accepted.length
|
|
355
|
+
remove_idx = 0
|
|
356
|
+
(1...accepted.length).each do |i|
|
|
357
|
+
remove_idx = i if accepted[i].open > accepted[remove_idx].open
|
|
358
|
+
end
|
|
359
|
+
accepted = accepted[0...remove_idx] + accepted[(remove_idx + 1)..]
|
|
360
|
+
else
|
|
361
|
+
accepted = survivors
|
|
362
|
+
end
|
|
363
|
+
end
|
|
364
|
+
|
|
365
|
+
# Unreachable given the termination argument; declines everything rather than looping.
|
|
366
|
+
[]
|
|
367
|
+
end
|
|
368
|
+
|
|
369
|
+
# emit is pass 5 (quotes.md 3.6). An edit whose replacement equals the span it replaces is
|
|
370
|
+
# never emitted -- the invisible-edit principle, applied per mark, not per pair. Hash
|
|
371
|
+
# iteration order is never relied on: both replacement and deletion indices are sorted
|
|
372
|
+
# before edits are built (ARCHITECTURE.md 4.5).
|
|
373
|
+
def self.emit(cp, accepted, quotes_data)
|
|
374
|
+
plan = compute_render_plan(cp, accepted, quotes_data)
|
|
375
|
+
edits = []
|
|
376
|
+
|
|
377
|
+
plan[:replace].keys.sort.each do |idx|
|
|
378
|
+
new_cp = plan[:replace][idx]
|
|
379
|
+
next if cp[idx] == new_cp
|
|
380
|
+
|
|
381
|
+
edits << Engine::Edit.new(idx, idx + 1, [new_cp], "quotes")
|
|
382
|
+
end
|
|
383
|
+
|
|
384
|
+
del_idx = plan[:del].keys.sort
|
|
385
|
+
i = 0
|
|
386
|
+
while i < del_idx.length
|
|
387
|
+
j = i
|
|
388
|
+
j += 1 while j + 1 < del_idx.length && del_idx[j + 1] == del_idx[j] + 1
|
|
389
|
+
edits << Engine::Edit.new(del_idx[i], del_idx[j] + 1, [], "quotes")
|
|
390
|
+
i = j + 1
|
|
391
|
+
end
|
|
392
|
+
|
|
393
|
+
edits.sort_by(&:start)
|
|
394
|
+
end
|
|
395
|
+
|
|
396
|
+
def self.scan(cp, locale_data, _ctx)
|
|
397
|
+
quotes_data = locale_data["quotes"]
|
|
398
|
+
idioms = quotes_data["elisionIdioms"] || []
|
|
399
|
+
skip_sets = compute_skip_sets(quotes_data)
|
|
400
|
+
|
|
401
|
+
candidates = collect_candidates(cp, skip_sets, idioms)
|
|
402
|
+
return [] if candidates.empty?
|
|
403
|
+
|
|
404
|
+
initial_pairs = pair_candidates(cp, candidates)
|
|
405
|
+
return [] if initial_pairs.empty?
|
|
406
|
+
|
|
407
|
+
accepted = certify(cp, initial_pairs, quotes_data, skip_sets, idioms)
|
|
408
|
+
return [] if accepted.empty?
|
|
409
|
+
|
|
410
|
+
emit(cp, accepted, quotes_data)
|
|
411
|
+
end
|
|
412
|
+
end
|
|
413
|
+
end
|
|
414
|
+
end
|
|
415
|
+
end
|
|
416
|
+
|
|
417
|
+
Polytypo::Engine::Registry.register(
|
|
418
|
+
"quotes",
|
|
419
|
+
->(cp, locale_data, ctx) { Polytypo::Engine::Rules::Quotes.scan(cp, locale_data, ctx) }
|
|
420
|
+
)
|