polytypo 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +109 -0
- data/lib/polytypo/data/README.md +20 -0
- data/lib/polytypo/data/UNICODE +1 -0
- data/lib/polytypo/data/VERSION +1 -0
- data/lib/polytypo/data/fixtures/de-CH.json +501 -0
- data/lib/polytypo/data/fixtures/de-DE.json +547 -0
- data/lib/polytypo/data/fixtures/el.json +239 -0
- data/lib/polytypo/data/fixtures/en-GB.json +1274 -0
- data/lib/polytypo/data/fixtures/en-US.json +1807 -0
- data/lib/polytypo/data/fixtures/fi.json +1306 -0
- data/lib/polytypo/data/fixtures/fr-CA.json +268 -0
- data/lib/polytypo/data/fixtures/fr.json +603 -0
- data/lib/polytypo/data/fixtures/locale-resolution.json +209 -0
- data/lib/polytypo/data/fixtures/ru.json +688 -0
- data/lib/polytypo/data/fixtures/sv.json +1290 -0
- data/lib/polytypo/data/locales/de-CH.json +77 -0
- data/lib/polytypo/data/locales/de-DE.json +76 -0
- data/lib/polytypo/data/locales/el.json +90 -0
- data/lib/polytypo/data/locales/en-GB.json +115 -0
- data/lib/polytypo/data/locales/en-US.json +133 -0
- data/lib/polytypo/data/locales/fi.json +136 -0
- data/lib/polytypo/data/locales/fr-CA.json +78 -0
- data/lib/polytypo/data/locales/fr.json +84 -0
- data/lib/polytypo/data/locales/registry.json +9 -0
- data/lib/polytypo/data/locales/ru.json +112 -0
- data/lib/polytypo/data/locales/sv.json +124 -0
- data/lib/polytypo/data/rules/dashes.md +1238 -0
- data/lib/polytypo/data/rules/order.json +78 -0
- data/lib/polytypo/data/schema/fixtures.schema.json +79 -0
- data/lib/polytypo/data/schema/locale.schema.json +235 -0
- data/lib/polytypo/data/schema/registry.schema.json +29 -0
- data/lib/polytypo/data/schema/resolution.schema.json +50 -0
- data/lib/polytypo/engine/codepoints.rb +24 -0
- data/lib/polytypo/engine/edits.rb +64 -0
- data/lib/polytypo/engine/locale.rb +138 -0
- data/lib/polytypo/engine/pipeline.rb +61 -0
- data/lib/polytypo/engine/registry.rb +47 -0
- data/lib/polytypo/engine/rules/apostrophe.rb +127 -0
- data/lib/polytypo/engine/rules/dash_shared.rb +342 -0
- data/lib/polytypo/engine/rules/dashes.rb +125 -0
- data/lib/polytypo/engine/rules/ellipsis.rb +100 -0
- data/lib/polytypo/engine/rules/hyphen.rb +207 -0
- data/lib/polytypo/engine/rules/nbsp.rb +616 -0
- data/lib/polytypo/engine/rules/quote_ambiguity.rb +241 -0
- data/lib/polytypo/engine/rules/quotes.rb +420 -0
- data/lib/polytypo/engine/rules/ranges.rb +124 -0
- data/lib/polytypo/engine/rules/spaces.rb +232 -0
- data/lib/polytypo/engine/rules/symbols.rb +291 -0
- data/lib/polytypo/engine/rules.rb +19 -0
- data/lib/polytypo/engine/sentinels.rb +23 -0
- data/lib/polytypo/engine/unicode_util.rb +390 -0
- data/lib/polytypo/errors.rb +24 -0
- data/lib/polytypo/modes/html.rb +233 -0
- data/lib/polytypo/modes/markdown.rb +187 -0
- data/lib/polytypo/modes/parse_error.rb +19 -0
- data/lib/polytypo/modes/runner.rb +57 -0
- data/lib/polytypo/modes/spans.rb +132 -0
- data/lib/polytypo/version.rb +5 -0
- data/lib/polytypo.rb +91 -0
- data/polytypo.gemspec +37 -0
- metadata +122 -0
|
@@ -0,0 +1,616 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../edits"
|
|
4
|
+
require_relative "../sentinels"
|
|
5
|
+
require_relative "../unicode_util"
|
|
6
|
+
require_relative "../registry"
|
|
7
|
+
require_relative "../../errors"
|
|
8
|
+
|
|
9
|
+
module Polytypo
|
|
10
|
+
module Engine
|
|
11
|
+
module Rules
|
|
12
|
+
# `nbsp` -- spec/rules/nbsp.md (spec 0.6.0), order 70 (last).
|
|
13
|
+
#
|
|
14
|
+
# Ten sub-rules, N1 through N10, evaluated in that fixed order (3.2); each produces
|
|
15
|
+
# candidate edits keyed by the index of the space (or insertion point) it claims, and the
|
|
16
|
+
# first sub-rule to claim an index wins. The claims table is a positional array indexed
|
|
17
|
+
# 0..cp.length, never a Hash -- a map-iteration implementation would resolve conflicts
|
|
18
|
+
# differently in Go (ARCHITECTURE.md section 4.5). No regex, no native-string indexing
|
|
19
|
+
# (ARCHITECTURE.md section 4.1, 4.2).
|
|
20
|
+
module Nbsp
|
|
21
|
+
SPACE = 0x20
|
|
22
|
+
TAB = 0x09
|
|
23
|
+
NBSP = 0xA0
|
|
24
|
+
NNBSP = 0x202F
|
|
25
|
+
FULL_STOP = 0x2E
|
|
26
|
+
|
|
27
|
+
DIGIT_ZERO = 0x30
|
|
28
|
+
DIGIT_NINE = 0x39
|
|
29
|
+
|
|
30
|
+
PAREN_OPEN = 0x28
|
|
31
|
+
SQUARE_OPEN = 0x5B
|
|
32
|
+
BRACE_OPEN = 0x7B
|
|
33
|
+
PAREN_CLOSE = 0x29
|
|
34
|
+
SQUARE_CLOSE = 0x5D
|
|
35
|
+
BRACE_CLOSE = 0x7D
|
|
36
|
+
|
|
37
|
+
EN_DASH = 0x2013
|
|
38
|
+
EM_DASH = 0x2014
|
|
39
|
+
ELLIPSIS = 0x2026
|
|
40
|
+
|
|
41
|
+
# cp[i], or NONE if i is out of bounds -- the spec's own boundary value.
|
|
42
|
+
def self.at(cp, i)
|
|
43
|
+
return NONE if i.negative? || i >= cp.length
|
|
44
|
+
|
|
45
|
+
cp[i]
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def self.digit?(cp)
|
|
49
|
+
cp >= DIGIT_ZERO && cp <= DIGIT_NINE
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def self.alnum?(cp)
|
|
53
|
+
digit?(cp) || UnicodeUtil.letter?(cp)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# BREAK (nbsp.md 3.1), including LINE_MARKER -- a member of BREAK for every rule,
|
|
57
|
+
# everywhere (modes.md 3.2).
|
|
58
|
+
#
|
|
59
|
+
# MARKER is deliberately NOT a member here, and openish?/closeish? below do not add it
|
|
60
|
+
# either. modes.md 3.3's table says nbsp's OPENISH/CLOSEISH include the span-boundary
|
|
61
|
+
# MARKER, the same way quotes' and apostrophe's do -- but the JS reference
|
|
62
|
+
# implementation's own isBreak/isOpenish/isCloseish never test for MARKER, only
|
|
63
|
+
# LINE_MARKER via isBreak. That is a documented, preserved discrepancy (tracked, not
|
|
64
|
+
# resolved, in the roadmap), carried forward here unchanged rather than "corrected"
|
|
65
|
+
# against the table, for cross-runtime consistency with the already-shipped JS, Python
|
|
66
|
+
# and Go ports.
|
|
67
|
+
def self.break?(cp)
|
|
68
|
+
cp == 0x0A || cp == 0x0D || cp == 0x0B || cp == 0x0C || cp == 0x85 ||
|
|
69
|
+
cp == 0x2028 || cp == 0x2029 || cp == LINE_MARKER
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def self.no_break?(cp)
|
|
73
|
+
cp == NBSP || cp == NNBSP
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# OTHER-SPACE (nbsp.md 3.1): the fixed-width spaces. A member of SPACELIKE for boundary
|
|
77
|
+
# purposes, but never converted and never an "already correct" state -- a thin or
|
|
78
|
+
# figure space the author placed stays exactly where it is.
|
|
79
|
+
def self.other_space?(cp)
|
|
80
|
+
(cp >= 0x2000 && cp <= 0x200A) || cp == 0x205F || cp == 0x3000
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# SPACELIKE, with NOBREAK included. Every boundary test in this rule uses this
|
|
84
|
+
# predicate and never U+0020 alone; that single decision is what makes the rule
|
|
85
|
+
# idempotent (nbsp.md 3.1).
|
|
86
|
+
def self.space_like?(cp)
|
|
87
|
+
cp == SPACE || no_break?(cp) || cp == TAB || other_space?(cp) || break?(cp)
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# SENTENCE-DASH (nbsp.md 3.1): U+2013 and U+2014 only, never a hyphen. A hyphen marks
|
|
91
|
+
# an intra-word position by construction, so the token after it is not a free-standing
|
|
92
|
+
# word -- without this exclusion "из-за дождя" would bind twice over, once by `hyphen`
|
|
93
|
+
# producing "из-за" and once by N3 reading the compound's tail "за" as a listed
|
|
94
|
+
# preposition (nbsp.md 3.5 step 2). An em or en dash does open a phrase, so
|
|
95
|
+
# "-- в Москве" still binds.
|
|
96
|
+
def self.sentence_dash?(cp)
|
|
97
|
+
cp == EN_DASH || cp == EM_DASH
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# One locale quote pair N8 owns: the open/close glyphs and the no-break space (or
|
|
101
|
+
# narrow no-break space) that belongs on their inner side.
|
|
102
|
+
QuoteTarget = Struct.new(:open, :close, :target)
|
|
103
|
+
|
|
104
|
+
# Locale data resolved to code points once per call. No module-level mutable state
|
|
105
|
+
# (ARCHITECTURE.md section 7): everything here is local to one scan call.
|
|
106
|
+
Prepared = Struct.new(
|
|
107
|
+
:before_punctuation, :narrow_before_punctuation, :short_words, :abbreviations,
|
|
108
|
+
:units, :before_number, :before_word, :symbols, :initial_binding, :opens, :closes,
|
|
109
|
+
:quote_pairs,
|
|
110
|
+
keyword_init: true,
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
def self.malformed(message)
|
|
114
|
+
raise Polytypo::Error.new(Polytypo::CODE_MALFORMED_LOCALE_DATA, message)
|
|
115
|
+
end
|
|
116
|
+
private_class_method :malformed
|
|
117
|
+
|
|
118
|
+
def self.single_code_point(entry, field)
|
|
119
|
+
cps = entry.codepoints
|
|
120
|
+
malformed("nbsp.#{field} entry #{entry.inspect} is not exactly one code point.") if cps.length != 1
|
|
121
|
+
cps[0]
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# Longest first, so each sub-rule's "longest match wins at a given a" (nbsp.md 3.5) is
|
|
125
|
+
# a linear search that returns on the first match.
|
|
126
|
+
def self.prepare_list(entries)
|
|
127
|
+
entries.map(&:codepoints).sort_by { |cps| -cps.length }
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# Resolves the locale's nbsp and quotes fields to code points once. nbsp.md 2 lists the
|
|
131
|
+
# fields; 2.1 explains why the mechanism (U+00A0 vs U+202F, convert-only vs insert)
|
|
132
|
+
# lives here and not in the locale file.
|
|
133
|
+
def self.prepare(locale_data)
|
|
134
|
+
data = locale_data["nbsp"]
|
|
135
|
+
before_punctuation = data["beforePunctuation"].map { |e| single_code_point(e, "beforePunctuation") }
|
|
136
|
+
narrow_before_punctuation =
|
|
137
|
+
data["narrowBeforePunctuation"].map { |e| single_code_point(e, "narrowBeforePunctuation") }
|
|
138
|
+
|
|
139
|
+
# nbsp.md 2 precondition: the two arrays must be disjoint. locale.schema.json does
|
|
140
|
+
# not enforce this; an implementation that finds a code point in both must raise
|
|
141
|
+
# POLYTYPO_MALFORMED_LOCALE_DATA rather than pick a winner silently.
|
|
142
|
+
before_punctuation.each do |cp|
|
|
143
|
+
next unless narrow_before_punctuation.include?(cp)
|
|
144
|
+
|
|
145
|
+
malformed(
|
|
146
|
+
format(
|
|
147
|
+
"nbsp.beforePunctuation and nbsp.narrowBeforePunctuation both list U+%04X; " \
|
|
148
|
+
"they must be disjoint (spec/rules/nbsp.md section 2).",
|
|
149
|
+
cp,
|
|
150
|
+
),
|
|
151
|
+
)
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
quotes = locale_data["quotes"]
|
|
155
|
+
primary = quotes["primary"]
|
|
156
|
+
secondary = quotes["secondary"]
|
|
157
|
+
opens = [
|
|
158
|
+
PAREN_OPEN, SQUARE_OPEN, BRACE_OPEN,
|
|
159
|
+
single_code_point(primary["open"], "quotes.primary.open"),
|
|
160
|
+
single_code_point(secondary["open"], "quotes.secondary.open"),
|
|
161
|
+
]
|
|
162
|
+
closes = [
|
|
163
|
+
PAREN_CLOSE, SQUARE_CLOSE, BRACE_CLOSE,
|
|
164
|
+
single_code_point(primary["close"], "quotes.primary.close"),
|
|
165
|
+
single_code_point(secondary["close"], "quotes.secondary.close"),
|
|
166
|
+
]
|
|
167
|
+
|
|
168
|
+
quote_pairs = []
|
|
169
|
+
[primary, secondary].each do |pair|
|
|
170
|
+
next if pair["innerSpace"] == "none"
|
|
171
|
+
|
|
172
|
+
open_cp = single_code_point(pair["open"], "quotes.open")
|
|
173
|
+
close_cp = single_code_point(pair["close"], "quotes.close")
|
|
174
|
+
# 3.10 sidedness precondition: an open glyph equal to its close glyph cannot be
|
|
175
|
+
# told apart without the pairing information only `quotes` has. Documented no-op
|
|
176
|
+
# (7.5).
|
|
177
|
+
next if open_cp == close_cp
|
|
178
|
+
|
|
179
|
+
target = pair["innerSpace"] == "nbsp" ? NBSP : NNBSP
|
|
180
|
+
quote_pairs << QuoteTarget.new(open_cp, close_cp, target)
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
Prepared.new(
|
|
184
|
+
before_punctuation: before_punctuation,
|
|
185
|
+
narrow_before_punctuation: narrow_before_punctuation,
|
|
186
|
+
short_words: prepare_list(data["afterShortWords"]),
|
|
187
|
+
abbreviations: prepare_list(data["abbreviations"]),
|
|
188
|
+
units: prepare_list(data["beforeUnits"]),
|
|
189
|
+
before_number: prepare_list(data["beforeNumber"]),
|
|
190
|
+
before_word: prepare_list(data["beforeWord"]),
|
|
191
|
+
symbols: prepare_list(data["afterSymbols"]),
|
|
192
|
+
initial_binding: data["initialBinding"],
|
|
193
|
+
opens: opens,
|
|
194
|
+
closes: closes,
|
|
195
|
+
quote_pairs: quote_pairs,
|
|
196
|
+
)
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
# OPENISH / CLOSEISH (nbsp.md 3.1): the ASCII brackets plus every locale quote glyph.
|
|
200
|
+
# See break?'s comment for why MARKER is not a member here.
|
|
201
|
+
def self.openish?(prep, cp)
|
|
202
|
+
prep.opens.include?(cp)
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
def self.closeish?(prep, cp)
|
|
206
|
+
prep.closes.include?(cp)
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def self.mark?(prep, cp)
|
|
210
|
+
prep.before_punctuation.include?(cp) || prep.narrow_before_punctuation.include?(cp)
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
# The only writers into the claims table. Both no-op if the index is already claimed,
|
|
214
|
+
# which is what makes sub-rule evaluation order equal first-claim-wins (nbsp.md 3.2).
|
|
215
|
+
def self.claim_conversion(claims, index, target)
|
|
216
|
+
return unless claims[index].nil?
|
|
217
|
+
|
|
218
|
+
claims[index] = Edit.new(index, index + 1, [target], "nbsp")
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
def self.claim_insertion(claims, index, target)
|
|
222
|
+
return unless claims[index].nil?
|
|
223
|
+
|
|
224
|
+
claims[index] = Edit.new(index, index, [target], "nbsp")
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
def self.match_exact?(cp, a, w)
|
|
228
|
+
return false if a + w.length > cp.length
|
|
229
|
+
|
|
230
|
+
w.each_with_index { |want, j| return false if cp[a + j] != want }
|
|
231
|
+
true
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
# nbsp.md 3.5 step 1: exact except that the pattern's first code point may also match
|
|
235
|
+
# its Unicode simple uppercase mapping -- a plain code-point-to-code-point table, never
|
|
236
|
+
# a locale-sensitive case operation (ARCHITECTURE.md section 4.4).
|
|
237
|
+
def self.match_first_char_lenient?(cp, a, w)
|
|
238
|
+
return false if w.empty? || a + w.length > cp.length
|
|
239
|
+
|
|
240
|
+
head = cp[a]
|
|
241
|
+
first = w[0]
|
|
242
|
+
return false if head != first && head != UnicodeUtil.simple_uppercase(first)
|
|
243
|
+
|
|
244
|
+
(1...w.length).each { |j| return false if cp[a + j] != w[j] }
|
|
245
|
+
true
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
# nbsp.md 3.6 step 1: exact except that a pattern U+0020 also matches an existing
|
|
249
|
+
# U+00A0 or U+202F in the input, so a previously-converted abbreviation still matches
|
|
250
|
+
# on a later run (the idempotency property nbsp.md 5 item 2 requires of N4).
|
|
251
|
+
def self.match_space_lenient?(cp, a, w)
|
|
252
|
+
return false if a + w.length > cp.length
|
|
253
|
+
|
|
254
|
+
w.each_with_index do |want, j|
|
|
255
|
+
got = cp[a + j]
|
|
256
|
+
next if got == want
|
|
257
|
+
next if want == SPACE && no_break?(got)
|
|
258
|
+
|
|
259
|
+
return false
|
|
260
|
+
end
|
|
261
|
+
true
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
# Returns the first pattern (from a longest-first-sorted list) that matches at a,
|
|
265
|
+
# implementing "longest match wins at a given a, with no backtracking" (nbsp.md 3.5)
|
|
266
|
+
# for every list-driven sub-rule: N3, N4, N5, N6, N9, N10.
|
|
267
|
+
def self.longest_match(patterns, cp, a, matcher)
|
|
268
|
+
patterns.each do |w|
|
|
269
|
+
return w if matcher.call(cp, a, w)
|
|
270
|
+
end
|
|
271
|
+
nil
|
|
272
|
+
end
|
|
273
|
+
|
|
274
|
+
# N1 (3.3, beforePunctuation -> U+00A0) and N2 (3.4, narrowBeforePunctuation ->
|
|
275
|
+
# U+202F): identical shape with target/other exchanged.
|
|
276
|
+
def self.punctuation_sub_rule(cp, prep, claims, marks, target, other)
|
|
277
|
+
return if marks.empty?
|
|
278
|
+
|
|
279
|
+
(0...cp.length).each do |i|
|
|
280
|
+
next unless marks.include?(cp[i])
|
|
281
|
+
|
|
282
|
+
left = at(cp, i - 1)
|
|
283
|
+
# Step 1 -- run guard: only the first mark of "?!" or "!!!" takes the space.
|
|
284
|
+
next if left != NONE && mark?(prep, left)
|
|
285
|
+
|
|
286
|
+
# Step 2 -- right-context guard: this is what protects "http://" and "12:30".
|
|
287
|
+
# U+2026 is accepted because the guard exists to catch punctuation *inside a
|
|
288
|
+
# token*, and an ellipsis after a question mark is not that (nbsp.md 3.3 step 2).
|
|
289
|
+
after = at(cp, i + 1)
|
|
290
|
+
if after != NONE && !space_like?(after) && !closeish?(prep, after) &&
|
|
291
|
+
after != ELLIPSIS && !mark?(prep, after)
|
|
292
|
+
next
|
|
293
|
+
end
|
|
294
|
+
|
|
295
|
+
# Step 3 -- quote-glyph guard. The space beside an opening quotation glyph is
|
|
296
|
+
# quotes.innerSpace and belongs to N8 alone; without this N2 and N8 alternate for
|
|
297
|
+
# ever on the French input "«?" (3.2, 3.10.1).
|
|
298
|
+
next if openish?(prep, left)
|
|
299
|
+
next if left != NONE && space_like?(left) && openish?(prep, at(cp, i - 2))
|
|
300
|
+
|
|
301
|
+
# Step 4.
|
|
302
|
+
next if left == target
|
|
303
|
+
|
|
304
|
+
if left == SPACE || left == other
|
|
305
|
+
claim_conversion(claims, i - 1, target)
|
|
306
|
+
next
|
|
307
|
+
end
|
|
308
|
+
# A fixed-width space stays as typed, and nothing is inserted beside it.
|
|
309
|
+
next if other_space?(left)
|
|
310
|
+
next if left == NONE || break?(left) || left == TAB
|
|
311
|
+
|
|
312
|
+
claim_insertion(claims, i, target)
|
|
313
|
+
end
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
# N3 -- nbsp.md 3.5 afterShortWords.
|
|
317
|
+
def self.short_words_sub_rule(cp, prep, claims)
|
|
318
|
+
return if prep.short_words.empty?
|
|
319
|
+
|
|
320
|
+
(0...cp.length).each do |a|
|
|
321
|
+
w = longest_match(prep.short_words, cp, a, method(:match_first_char_lenient?))
|
|
322
|
+
next if w.nil?
|
|
323
|
+
|
|
324
|
+
k = w.length
|
|
325
|
+
before = at(cp, a - 1)
|
|
326
|
+
unless before == NONE || space_like?(before) || openish?(prep, before) ||
|
|
327
|
+
sentence_dash?(before)
|
|
328
|
+
next
|
|
329
|
+
end
|
|
330
|
+
|
|
331
|
+
separator = at(cp, a + k)
|
|
332
|
+
next if separator == NBSP # already correct
|
|
333
|
+
next if separator != SPACE
|
|
334
|
+
|
|
335
|
+
following = at(cp, a + k + 1)
|
|
336
|
+
next unless alnum?(following) || openish?(prep, following)
|
|
337
|
+
|
|
338
|
+
claim_conversion(claims, a + k, NBSP)
|
|
339
|
+
end
|
|
340
|
+
end
|
|
341
|
+
|
|
342
|
+
# N4 -- nbsp.md 3.6 abbreviations, U+00A0 for every internal space.
|
|
343
|
+
def self.abbreviations_sub_rule(cp, prep, claims)
|
|
344
|
+
return if prep.abbreviations.empty?
|
|
345
|
+
|
|
346
|
+
(0...cp.length).each do |a|
|
|
347
|
+
w = longest_match(prep.abbreviations, cp, a, method(:match_space_lenient?))
|
|
348
|
+
next if w.nil?
|
|
349
|
+
|
|
350
|
+
k = w.length
|
|
351
|
+
next if alnum?(at(cp, a - 1))
|
|
352
|
+
next if alnum?(at(cp, a + k))
|
|
353
|
+
|
|
354
|
+
(0...k).each do |j|
|
|
355
|
+
next unless w[j] == SPACE
|
|
356
|
+
next if cp[a + j] == NBSP # already correct at this internal position
|
|
357
|
+
|
|
358
|
+
claim_conversion(claims, a + j, NBSP)
|
|
359
|
+
end
|
|
360
|
+
end
|
|
361
|
+
end
|
|
362
|
+
|
|
363
|
+
# N5 -- nbsp.md 3.7 beforeUnits. Converts an existing space; never inserts one (7.2).
|
|
364
|
+
def self.units_sub_rule(cp, prep, claims)
|
|
365
|
+
return if prep.units.empty?
|
|
366
|
+
|
|
367
|
+
(0...cp.length).each do |a|
|
|
368
|
+
w = longest_match(prep.units, cp, a, method(:match_exact?))
|
|
369
|
+
next if w.nil?
|
|
370
|
+
|
|
371
|
+
k = w.length
|
|
372
|
+
next if alnum?(at(cp, a + k))
|
|
373
|
+
|
|
374
|
+
left = at(cp, a - 1)
|
|
375
|
+
next if left == NBSP # already correct
|
|
376
|
+
next if left != SPACE
|
|
377
|
+
|
|
378
|
+
next unless digit?(at(cp, a - 2))
|
|
379
|
+
|
|
380
|
+
b = a - 2
|
|
381
|
+
b -= 1 while b - 1 >= 0 && digit?(cp[b - 1])
|
|
382
|
+
# The letter guard: "H2 O", "A4", "MP3" are not measurements.
|
|
383
|
+
next if UnicodeUtil.letter?(at(cp, b - 1))
|
|
384
|
+
|
|
385
|
+
claim_conversion(claims, a - 1, NBSP)
|
|
386
|
+
end
|
|
387
|
+
end
|
|
388
|
+
|
|
389
|
+
# N6 -- nbsp.md 3.8 afterSymbols. Conversion only.
|
|
390
|
+
def self.symbols_sub_rule(cp, prep, claims)
|
|
391
|
+
return if prep.symbols.empty?
|
|
392
|
+
|
|
393
|
+
(0...cp.length).each do |a|
|
|
394
|
+
w = longest_match(prep.symbols, cp, a, method(:match_exact?))
|
|
395
|
+
next if w.nil?
|
|
396
|
+
|
|
397
|
+
k = w.length
|
|
398
|
+
next if alnum?(at(cp, a - 1))
|
|
399
|
+
|
|
400
|
+
separator = at(cp, a + k)
|
|
401
|
+
next if separator == NBSP # already correct
|
|
402
|
+
next if separator != SPACE
|
|
403
|
+
|
|
404
|
+
next unless digit?(at(cp, a + k + 1))
|
|
405
|
+
|
|
406
|
+
claim_conversion(claims, a + k, NBSP)
|
|
407
|
+
end
|
|
408
|
+
end
|
|
409
|
+
|
|
410
|
+
# nbsp.md 3.9: one uppercase letter, one full stop, at a token start.
|
|
411
|
+
def self.initial_at?(cp, prep, p)
|
|
412
|
+
return false if p.negative?
|
|
413
|
+
return false unless UnicodeUtil.upper?(at(cp, p))
|
|
414
|
+
return false if at(cp, p + 1) != FULL_STOP
|
|
415
|
+
|
|
416
|
+
before = at(cp, p - 1)
|
|
417
|
+
before == NONE || space_like?(before) || openish?(prep, before)
|
|
418
|
+
end
|
|
419
|
+
|
|
420
|
+
# Guard C1-a (nbsp.md 3.9): an uppercase letter plus a dot that is itself preceded by
|
|
421
|
+
# a lower-case letter plus a dot is the second token of an abbreviation, not an
|
|
422
|
+
# initial. Without it the shipped de-DE data turns "z. B. Berlin" into a form with
|
|
423
|
+
# both the internal abbreviation space AND the space after "B." bound to U+00A0 -- a
|
|
424
|
+
# false positive on ordinary prose ("z. B." is correct, "B. Berlin" is not a name).
|
|
425
|
+
# "А. С. Пушкин" is unaffected: cp[p-3] there is uppercase.
|
|
426
|
+
def self.abbreviation_tail?(cp, p)
|
|
427
|
+
return false unless space_like?(at(cp, p - 1))
|
|
428
|
+
return false if at(cp, p - 2) != FULL_STOP
|
|
429
|
+
|
|
430
|
+
head = at(cp, p - 3)
|
|
431
|
+
UnicodeUtil.letter?(head) && !UnicodeUtil.upper?(head)
|
|
432
|
+
end
|
|
433
|
+
|
|
434
|
+
# The "chain" mode confirmation (nbsp.md 3.9): is the initial whose letter sits at p
|
|
435
|
+
# itself immediately preceded by another initial? Used only by "chain" mode's C1, to
|
|
436
|
+
# require Chicago's own "two or more initials" before the space leading into a
|
|
437
|
+
# following non-initial word (a candidate surname) is bound.
|
|
438
|
+
def self.preceding_initial?(cp, prep, p)
|
|
439
|
+
gap = at(cp, p - 1)
|
|
440
|
+
return false unless gap == SPACE || gap == NBSP
|
|
441
|
+
return false if at(cp, p - 2) != FULL_STOP
|
|
442
|
+
|
|
443
|
+
initial_at?(cp, prep, p - 3)
|
|
444
|
+
end
|
|
445
|
+
|
|
446
|
+
# N7 -- nbsp.md 3.9 initialBinding, skipped entirely when the locale's initialBinding
|
|
447
|
+
# is "none".
|
|
448
|
+
def self.initials_sub_rule(cp, prep, claims)
|
|
449
|
+
mode = prep.initial_binding
|
|
450
|
+
return if mode == "none"
|
|
451
|
+
|
|
452
|
+
(0...cp.length).each do |q|
|
|
453
|
+
here = cp[q]
|
|
454
|
+
next unless here == SPACE || here == NBSP
|
|
455
|
+
|
|
456
|
+
# C1 -- an initial on the left and an uppercase letter on the right, unless C1-a
|
|
457
|
+
# declines.
|
|
458
|
+
left_initial_p = q - 2
|
|
459
|
+
c1_shape = at(cp, q - 1) == FULL_STOP &&
|
|
460
|
+
initial_at?(cp, prep, left_initial_p) &&
|
|
461
|
+
UnicodeUtil.upper?(at(cp, q + 1)) &&
|
|
462
|
+
!abbreviation_tail?(cp, left_initial_p)
|
|
463
|
+
# "chain" mode additionally requires either that the right side is itself an
|
|
464
|
+
# initial (the between-initials case, e.g. "E.|B.", always safe) or that the left
|
|
465
|
+
# initial is itself preceded by another initial (a confirmed chain of two or more,
|
|
466
|
+
# e.g. "E. B.|White") before binding to a plain following word. "single" mode keeps
|
|
467
|
+
# the unconditional shape check -- the behaviour fr/fr-CA need for
|
|
468
|
+
# "N. Bourbaki"/"M. Dupont" (nbsp.md 3.9, Jacques Andre), structurally
|
|
469
|
+
# indistinguishable from a sentence-boundary collision.
|
|
470
|
+
c1 = c1_shape &&
|
|
471
|
+
(mode == "single" ||
|
|
472
|
+
initial_at?(cp, prep, q + 1) ||
|
|
473
|
+
preceding_initial?(cp, prep, left_initial_p))
|
|
474
|
+
|
|
475
|
+
# C2 -- a word on the left and two consecutive initials on the right
|
|
476
|
+
# ("Пушкин А. С."). Already requires two initials by construction, so it is
|
|
477
|
+
# unaffected by "chain" vs "single".
|
|
478
|
+
right_space = at(cp, q + 3)
|
|
479
|
+
c2 = UnicodeUtil.letter?(at(cp, q - 1)) &&
|
|
480
|
+
initial_at?(cp, prep, q + 1) &&
|
|
481
|
+
(right_space == SPACE || right_space == NBSP) &&
|
|
482
|
+
initial_at?(cp, prep, q + 4)
|
|
483
|
+
|
|
484
|
+
next unless c1 || c2
|
|
485
|
+
next if here == NBSP # already correct
|
|
486
|
+
|
|
487
|
+
claim_conversion(claims, q, NBSP)
|
|
488
|
+
end
|
|
489
|
+
end
|
|
490
|
+
|
|
491
|
+
# N8 -- nbsp.md 3.10 quotes.innerSpace. The only sub-rule besides N1/N2 that may
|
|
492
|
+
# insert.
|
|
493
|
+
def self.quotes_sub_rule(cp, prep, claims)
|
|
494
|
+
prep.quote_pairs.each do |pair|
|
|
495
|
+
(0...cp.length).each do |i|
|
|
496
|
+
here = cp[i]
|
|
497
|
+
|
|
498
|
+
if here == pair.open
|
|
499
|
+
right = at(cp, i + 1)
|
|
500
|
+
if right == pair.target
|
|
501
|
+
# already correct
|
|
502
|
+
elsif right == SPACE || no_break?(right)
|
|
503
|
+
claim_conversion(claims, i + 1, pair.target)
|
|
504
|
+
elsif right == NONE || break?(right)
|
|
505
|
+
# skip: never insert at a line boundary or the end of the text
|
|
506
|
+
else
|
|
507
|
+
claim_insertion(claims, i + 1, pair.target)
|
|
508
|
+
end
|
|
509
|
+
next
|
|
510
|
+
end
|
|
511
|
+
|
|
512
|
+
next unless here == pair.close
|
|
513
|
+
|
|
514
|
+
left = at(cp, i - 1)
|
|
515
|
+
if left == pair.target
|
|
516
|
+
# already correct
|
|
517
|
+
elsif left == SPACE || no_break?(left)
|
|
518
|
+
claim_conversion(claims, i - 1, pair.target)
|
|
519
|
+
elsif left == NONE || break?(left)
|
|
520
|
+
# skip
|
|
521
|
+
else
|
|
522
|
+
claim_insertion(claims, i, pair.target)
|
|
523
|
+
end
|
|
524
|
+
end
|
|
525
|
+
end
|
|
526
|
+
end
|
|
527
|
+
|
|
528
|
+
# N9 (nbsp.md 3.11, beforeNumber, wants_digit = true) and N10 (3.12, beforeWord,
|
|
529
|
+
# wants_digit = false). They share every guard except what must follow the separator:
|
|
530
|
+
# a digit for N9, a letter for N10.
|
|
531
|
+
def self.forward_binding_sub_rule(cp, prep, claims, patterns, wants_digit)
|
|
532
|
+
return if patterns.empty?
|
|
533
|
+
|
|
534
|
+
(0...cp.length).each do |a|
|
|
535
|
+
w = longest_match(patterns, cp, a, method(:match_exact?))
|
|
536
|
+
next if w.nil?
|
|
537
|
+
|
|
538
|
+
k = w.length
|
|
539
|
+
|
|
540
|
+
# G-D (3.12 step 5, N10 only): in a locale where N7 is active, an *uppercase*
|
|
541
|
+
# letter plus a dot is structurally an initial, and N7 owns that shape with better
|
|
542
|
+
# evidence (it inspects what follows for a second initial or a surname). The UPPER
|
|
543
|
+
# test is load-bearing: without it a lower-case entry such as "ул." would be inert
|
|
544
|
+
# in an initialBinding-active locale.
|
|
545
|
+
if !wants_digit && prep.initial_binding != "none" && k == 2 &&
|
|
546
|
+
UnicodeUtil.upper?(w[0]) && w[1] == FULL_STOP
|
|
547
|
+
next
|
|
548
|
+
end
|
|
549
|
+
|
|
550
|
+
# G-L -- stronger than "not ALNUM": it is what stops "S." matching inside
|
|
551
|
+
# "Fig.S. 3". A hyphen fails it, per 3.5 step 2 -- an abbreviation cannot begin
|
|
552
|
+
# immediately after an intra-word hyphen.
|
|
553
|
+
before = at(cp, a - 1)
|
|
554
|
+
unless before == NONE || space_like?(before) || openish?(prep, before) ||
|
|
555
|
+
sentence_dash?(before)
|
|
556
|
+
next
|
|
557
|
+
end
|
|
558
|
+
|
|
559
|
+
# G-S -- exactly one separator, and it must already be a space.
|
|
560
|
+
separator = at(cp, a + k)
|
|
561
|
+
next if separator == NBSP # already correct
|
|
562
|
+
next if separator != SPACE
|
|
563
|
+
|
|
564
|
+
following = at(cp, a + k + 1)
|
|
565
|
+
next if space_like?(following)
|
|
566
|
+
|
|
567
|
+
# G-W / "a following number": one code point, tested for membership. NONE fails
|
|
568
|
+
# both, which is also the line-boundary guard G-B.
|
|
569
|
+
if wants_digit
|
|
570
|
+
next unless digit?(following)
|
|
571
|
+
else
|
|
572
|
+
next unless UnicodeUtil.letter?(following)
|
|
573
|
+
end
|
|
574
|
+
|
|
575
|
+
claim_conversion(claims, a + k, NBSP)
|
|
576
|
+
end
|
|
577
|
+
end
|
|
578
|
+
|
|
579
|
+
# nbsp.md 3.2: N1 through N10, in that fixed order, first claim wins. The order is
|
|
580
|
+
# positional and total, never an artefact of Hash iteration.
|
|
581
|
+
#
|
|
582
|
+
# First-claim-wins only settles a conflict when both sub-rules actually emit an edit;
|
|
583
|
+
# an "already correct" branch emits nothing and therefore claims nothing, silently
|
|
584
|
+
# yielding the index to a lower-priority sub-rule. Sub-rules wanting *different* code
|
|
585
|
+
# points at a shared index are therefore made disjoint by construction elsewhere
|
|
586
|
+
# (N1/N2's quote-glyph guard, nbsp.md 3.10.1) rather than relying on ordering alone.
|
|
587
|
+
def self.scan(cp, locale_data, _ctx)
|
|
588
|
+
prep = prepare(locale_data)
|
|
589
|
+
claims = Array.new(cp.length + 1)
|
|
590
|
+
|
|
591
|
+
punctuation_sub_rule(cp, prep, claims, prep.before_punctuation, NBSP, NNBSP) # N1
|
|
592
|
+
punctuation_sub_rule(cp, prep, claims, prep.narrow_before_punctuation, NNBSP, NBSP) # N2
|
|
593
|
+
short_words_sub_rule(cp, prep, claims) # N3
|
|
594
|
+
abbreviations_sub_rule(cp, prep, claims) # N4
|
|
595
|
+
units_sub_rule(cp, prep, claims) # N5
|
|
596
|
+
symbols_sub_rule(cp, prep, claims) # N6
|
|
597
|
+
initials_sub_rule(cp, prep, claims) # N7
|
|
598
|
+
quotes_sub_rule(cp, prep, claims) # N8
|
|
599
|
+
forward_binding_sub_rule(cp, prep, claims, prep.before_number, true) # N9
|
|
600
|
+
forward_binding_sub_rule(cp, prep, claims, prep.before_word, false) # N10
|
|
601
|
+
|
|
602
|
+
edits = []
|
|
603
|
+
(0..cp.length).each do |i|
|
|
604
|
+
edits << claims[i] unless claims[i].nil?
|
|
605
|
+
end
|
|
606
|
+
edits
|
|
607
|
+
end
|
|
608
|
+
end
|
|
609
|
+
end
|
|
610
|
+
end
|
|
611
|
+
end
|
|
612
|
+
|
|
613
|
+
Polytypo::Engine::Registry.register(
|
|
614
|
+
"nbsp",
|
|
615
|
+
->(cp, locale_data, ctx) { Polytypo::Engine::Rules::Nbsp.scan(cp, locale_data, ctx) },
|
|
616
|
+
)
|