polytypo 1.17.4 → 1.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/polytypo/data/VERSION +1 -1
- data/lib/polytypo/data/fixtures/bg.json +1 -1
- data/lib/polytypo/data/fixtures/cs.json +1 -1
- data/lib/polytypo/data/fixtures/de-CH.json +1 -1
- data/lib/polytypo/data/fixtures/de-DE.json +1 -1
- data/lib/polytypo/data/fixtures/el.json +1 -1
- data/lib/polytypo/data/fixtures/en-GB.json +64 -1
- data/lib/polytypo/data/fixtures/en-US.json +64 -1
- data/lib/polytypo/data/fixtures/es.json +272 -7
- data/lib/polytypo/data/fixtures/fi.json +1 -1
- data/lib/polytypo/data/fixtures/fr-CA.json +1 -1
- data/lib/polytypo/data/fixtures/fr.json +1 -1
- data/lib/polytypo/data/fixtures/hr.json +1 -1
- data/lib/polytypo/data/fixtures/hu.json +1 -1
- data/lib/polytypo/data/fixtures/it.json +1 -1
- data/lib/polytypo/data/fixtures/locale-resolution.json +1 -1
- data/lib/polytypo/data/fixtures/lt.json +1 -1
- data/lib/polytypo/data/fixtures/nl.json +1 -1
- data/lib/polytypo/data/fixtures/pl.json +1 -1
- data/lib/polytypo/data/fixtures/pt-BR.json +1 -1
- data/lib/polytypo/data/fixtures/pt-PT.json +1 -1
- data/lib/polytypo/data/fixtures/ro.json +1 -1
- data/lib/polytypo/data/fixtures/ru.json +1 -1
- data/lib/polytypo/data/fixtures/sk.json +1 -1
- data/lib/polytypo/data/fixtures/sl.json +1 -1
- data/lib/polytypo/data/fixtures/sr.json +1 -1
- data/lib/polytypo/data/fixtures/sv.json +1 -1
- data/lib/polytypo/data/fixtures/tr.json +1 -1
- data/lib/polytypo/data/fixtures/uk.json +1 -1
- data/lib/polytypo/data/locales/el.json +1 -1
- data/lib/polytypo/data/locales/es.json +2 -2
- data/lib/polytypo/data/locales/registry.json +1 -1
- data/lib/polytypo/data/rules/dashes.md +108 -16
- data/lib/polytypo/data/rules/modes.md +49 -4
- data/lib/polytypo/data/rules/order.json +2 -2
- data/lib/polytypo/data/rules/pipeline-idempotency.md +7 -0
- data/lib/polytypo/data/schema/locale.schema.json +2 -2
- data/lib/polytypo/engine/rules/dash_shared.rb +32 -21
- data/lib/polytypo/engine/rules/dashes.rb +4 -1
- data/lib/polytypo/engine/rules/dashes_paired.rb +160 -0
- data/lib/polytypo/modes/spans.rb +53 -2
- data/lib/polytypo/version.rb +1 -1
- metadata +2 -1
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "dash_shared"
|
|
4
|
+
require_relative "../edits"
|
|
5
|
+
require_relative "../sentinels"
|
|
6
|
+
require_relative "../unicode_util"
|
|
7
|
+
|
|
8
|
+
module Polytypo
|
|
9
|
+
module Engine
|
|
10
|
+
module Rules
|
|
11
|
+
# `dash.parenthetical = "em-paired"` -- spec/rules/dashes.md 3.4a. The pair is written with a
|
|
12
|
+
# space outside it and none inside (`Llego —por fin— a casa`), which no per-token rule can
|
|
13
|
+
# express: which stroke opens and which closes is a property of the sentence, not of the
|
|
14
|
+
# token. A sentence is converted only when it holds exactly two counted strokes and both are
|
|
15
|
+
# eligible; anything else is left as typed. Mirrors dashes-paired.ts in the JS reference
|
|
16
|
+
# implementation.
|
|
17
|
+
module DashesPaired
|
|
18
|
+
# One counted DASH run of a segment: its span [s, e), outer spacing, the flanking indices
|
|
19
|
+
# after the joiner walk, and whether it may take part in a conversion.
|
|
20
|
+
Run = Struct.new(:s, :e, :lsp, :rsp, :left, :right, :eligible, keyword_init: true)
|
|
21
|
+
|
|
22
|
+
TERMINALS = [0x2e, 0x3f, 0x21, 0x2026].freeze
|
|
23
|
+
|
|
24
|
+
module_function
|
|
25
|
+
|
|
26
|
+
def terminal?(cp)
|
|
27
|
+
TERMINALS.include?(cp)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# A terminal ends a segment only before a space (U+0020 or a no-break space, which `nbsp`
|
|
31
|
+
# puts there later), a line break, a marker or the end of input.
|
|
32
|
+
def ends_segment?(cp, i)
|
|
33
|
+
return false unless terminal?(cp[i])
|
|
34
|
+
return true if i + 1 >= cp.length
|
|
35
|
+
|
|
36
|
+
nxt = cp[i + 1]
|
|
37
|
+
nxt == DashShared::SPACE || DashShared.no_break_space?(nxt) || DashShared.break?(nxt) ||
|
|
38
|
+
Polytypo::Engine.marker?(nxt)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def wants_space?(cp)
|
|
42
|
+
Polytypo::Engine::UnicodeUtil.letter?(cp) || DashShared.digit?(cp)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# dashes.md 3.2 steps 5-7: content on the same line, no dash or space beside the stroke's
|
|
46
|
+
# outer spacing, and not part of a cluster holding another dash run.
|
|
47
|
+
def neighbours_clean?(cp, s, e, left, right)
|
|
48
|
+
left_cp = cp[left]
|
|
49
|
+
right_cp = cp[right]
|
|
50
|
+
return false if DashShared.break?(left_cp) || DashShared.break?(right_cp)
|
|
51
|
+
return false if DashShared.inert_dash?(left_cp) || DashShared.inert_dash?(right_cp)
|
|
52
|
+
return false if DashShared.dash?(left_cp) || DashShared.dash?(right_cp)
|
|
53
|
+
return false if left_cp == DashShared::SPACE || DashShared.no_break_space?(left_cp)
|
|
54
|
+
return false if right_cp == DashShared::SPACE || DashShared.no_break_space?(right_cp)
|
|
55
|
+
|
|
56
|
+
!DashShared.cluster_inert?(cp, s, e)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Returns nil when the run is not a stroke at all, otherwise a Run (eligible or not).
|
|
60
|
+
def classify(cp, s, e)
|
|
61
|
+
n = cp.length
|
|
62
|
+
k = e - s
|
|
63
|
+
return nil if k > 3
|
|
64
|
+
|
|
65
|
+
lsp = s.positive? && cp[s - 1] == DashShared::SPACE ? 1 : 0
|
|
66
|
+
rsp = e < n && cp[e] == DashShared::SPACE ? 1 : 0
|
|
67
|
+
first = cp[s]
|
|
68
|
+
is_hyphen_shaped = first == DashShared::HYPHEN_MINUS || first == DashShared::HYPHEN ||
|
|
69
|
+
first == DashShared::MINUS_SIGN
|
|
70
|
+
# A hyphen inside a word is a compound, not a stroke.
|
|
71
|
+
return nil if k == 1 && is_hyphen_shaped && lsp.zero? && rsp.zero?
|
|
72
|
+
# The hyphen of `+/-`, which `symbols` (order 80) turns into a plus-minus sign: counting
|
|
73
|
+
# it would let that later rule change the number of strokes in the segment.
|
|
74
|
+
return nil if k == 1 && first == DashShared::HYPHEN_MINUS && s >= 2 && cp[s - 1] == 0x2f && cp[s - 2] == 0x2b
|
|
75
|
+
|
|
76
|
+
left = s - 1 - lsp
|
|
77
|
+
right = e + rsp
|
|
78
|
+
counted = lambda do |eligible|
|
|
79
|
+
Run.new(s: s, e: e, lsp: lsp, rsp: rsp, left: left, right: right, eligible: eligible)
|
|
80
|
+
end
|
|
81
|
+
return counted.call(false) if left.negative? || right >= n
|
|
82
|
+
|
|
83
|
+
join_start = left + 1
|
|
84
|
+
join_end = right
|
|
85
|
+
left -= 1 while left >= 0 && cp[left] == DashShared::WORD_JOINER
|
|
86
|
+
right += 1 while right < n && cp[right] == DashShared::WORD_JOINER
|
|
87
|
+
return counted.call(false) if left.negative? || right >= n
|
|
88
|
+
|
|
89
|
+
crossed_joiner = left + 1 != join_start || right != join_end
|
|
90
|
+
|
|
91
|
+
return nil unless DashShared.range_flanks(cp, left, right).nil?
|
|
92
|
+
return counted.call(false) if crossed_joiner
|
|
93
|
+
|
|
94
|
+
return counted.call(false) unless neighbours_clean?(cp, s, e, left, right)
|
|
95
|
+
|
|
96
|
+
# A stroke spaced on one side only is eligible solely as the one em dash that is already
|
|
97
|
+
# half of the target form; `-mente`, `pre-`, `--force` and `- item` are not dashes.
|
|
98
|
+
return counted.call(false) if lsp != rsp && !(k == 1 && first == DashShared::EM_DASH)
|
|
99
|
+
# P4: a tight stroke between two Roman numerals is a range already in its correct form.
|
|
100
|
+
return counted.call(false) if lsp.zero? && rsp.zero? && Dashes.roman_flanked?(cp, left, right)
|
|
101
|
+
# P5: an authored lone en dash is declined in every locale.
|
|
102
|
+
return counted.call(false) if k == 1 && first == DashShared::EN_DASH
|
|
103
|
+
|
|
104
|
+
counted.call(true)
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def scan(cp)
|
|
108
|
+
n = cp.length
|
|
109
|
+
edits = []
|
|
110
|
+
segment = []
|
|
111
|
+
i = 0
|
|
112
|
+
|
|
113
|
+
while i < n
|
|
114
|
+
if DashShared.dash?(cp[i])
|
|
115
|
+
s = i
|
|
116
|
+
e = s
|
|
117
|
+
e += 1 while e < n && DashShared.dash?(cp[e])
|
|
118
|
+
i = e
|
|
119
|
+
run = classify(cp, s, e)
|
|
120
|
+
segment << run unless run.nil?
|
|
121
|
+
next
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
if ends_segment?(cp, i)
|
|
125
|
+
emit_pair(cp, segment[0], segment[1], edits) if segment.length == 2
|
|
126
|
+
segment = []
|
|
127
|
+
end
|
|
128
|
+
i += 1
|
|
129
|
+
end
|
|
130
|
+
emit_pair(cp, segment[0], segment[1], edits) if segment.length == 2
|
|
131
|
+
|
|
132
|
+
edits
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def emit_pair(cp, a, b, edits)
|
|
136
|
+
return unless a.eligible && b.eligible
|
|
137
|
+
return if a.lsp.zero? && a.rsp == 1 # the first stroke is shaped like a closer
|
|
138
|
+
return if b.lsp == 1 && b.rsp.zero? # the second stroke is shaped like an opener
|
|
139
|
+
return if (a.e...b.s).any? { |j| DashShared.break?(cp[j]) }
|
|
140
|
+
|
|
141
|
+
insert_before = a.lsp.zero? && wants_space?(cp[a.left])
|
|
142
|
+
insert_after = b.rsp.zero? && wants_space?(cp[b.right])
|
|
143
|
+
return if insert_before && DashShared.spacing_transition_blocked_left?(cp, a.left)
|
|
144
|
+
return if insert_after && DashShared.spacing_transition_blocked_right?(cp, b.right)
|
|
145
|
+
|
|
146
|
+
opener = a.lsp == 1 || insert_before ? [DashShared::SPACE, DashShared::EM_DASH] : [DashShared::EM_DASH]
|
|
147
|
+
closer = b.rsp == 1 || insert_after ? [DashShared::EM_DASH, DashShared::SPACE] : [DashShared::EM_DASH]
|
|
148
|
+
|
|
149
|
+
[[a, opener], [b, closer]].each do |run, replacement|
|
|
150
|
+
span_start = run.s - run.lsp
|
|
151
|
+
span_end = run.e + run.rsp
|
|
152
|
+
next if DashShared.same_content?(cp, span_start, span_end, replacement)
|
|
153
|
+
|
|
154
|
+
edits << Polytypo::Engine::Edit.new(span_start, span_end, replacement, "dashes")
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
end
|
|
160
|
+
end
|
data/lib/polytypo/modes/spans.rb
CHANGED
|
@@ -189,6 +189,56 @@ module Polytypo
|
|
|
189
189
|
end
|
|
190
190
|
private_class_method :creates_reference?
|
|
191
191
|
|
|
192
|
+
# modes.md 3.4 bracket-shape rule (markdown only, spec 1.17.5): a deletion that joins a
|
|
193
|
+
# task-list marker or a footnote label. A marker or a position outside the array is in no
|
|
194
|
+
# class. Ruby indexes negatively, so every read goes through +code_point_at+.
|
|
195
|
+
def self.code_point_at(cp, i)
|
|
196
|
+
i.negative? ? nil : cp[i]
|
|
197
|
+
end
|
|
198
|
+
private_class_method :code_point_at
|
|
199
|
+
|
|
200
|
+
def self.skip_spaces(cp, i, step)
|
|
201
|
+
k = i
|
|
202
|
+
k += step while k >= 0 && k < cp.length && cp[k] == SPACE
|
|
203
|
+
k
|
|
204
|
+
end
|
|
205
|
+
private_class_method :skip_spaces
|
|
206
|
+
|
|
207
|
+
def self.x_letter?(c)
|
|
208
|
+
c == 0x78 || c == 0x58
|
|
209
|
+
end
|
|
210
|
+
private_class_method :x_letter?
|
|
211
|
+
|
|
212
|
+
def self.label_character?(c)
|
|
213
|
+
!c.nil? && c >= 0 && c != SPACE && c != 0x5B && c != 0x5D
|
|
214
|
+
end
|
|
215
|
+
private_class_method :label_character?
|
|
216
|
+
|
|
217
|
+
def self.creates_bracket_shape?(cp, p, q, replacement)
|
|
218
|
+
is_collapse = replacement.length == 1 && replacement[0] == SPACE
|
|
219
|
+
return false unless replacement.empty? || is_collapse
|
|
220
|
+
|
|
221
|
+
a = skip_spaces(cp, p - 1, -1)
|
|
222
|
+
b = skip_spaces(cp, q + 1, 1)
|
|
223
|
+
at_a = code_point_at(cp, a)
|
|
224
|
+
at_b = code_point_at(cp, b)
|
|
225
|
+
return at_a == 0x5B && at_b == 0x5D if is_collapse
|
|
226
|
+
return true if at_a == 0x5B && at_b == 0x5D
|
|
227
|
+
if at_a == 0x5B
|
|
228
|
+
return true if at_b == 0x5E
|
|
229
|
+
|
|
230
|
+
return x_letter?(at_b) && code_point_at(cp, skip_spaces(cp, b + 1, 1)) == 0x5D
|
|
231
|
+
end
|
|
232
|
+
return false if at_b != 0x5D
|
|
233
|
+
return true if x_letter?(at_a) && code_point_at(cp, skip_spaces(cp, a - 1, -1)) == 0x5B
|
|
234
|
+
|
|
235
|
+
run = a
|
|
236
|
+
run -= 1 while run >= 0 && label_character?(cp[run])
|
|
237
|
+
first = run + 1
|
|
238
|
+
a - first >= 1 && cp[first] == 0x5E && code_point_at(cp, skip_spaces(cp, run, -1)) == 0x5B
|
|
239
|
+
end
|
|
240
|
+
private_class_method :creates_bracket_shape?
|
|
241
|
+
|
|
192
242
|
# modes.md 3.4, two safety nets, both pure functions of (p, q, r, s0, s1):
|
|
193
243
|
#
|
|
194
244
|
# 1. No edit may contain a marker -- one that does is a bug, discarded rather than
|
|
@@ -202,8 +252,8 @@ module Polytypo
|
|
|
202
252
|
# point any rule emits whose meaning comes from its position rather than from itself
|
|
203
253
|
# (modes.md 5 item 2).
|
|
204
254
|
#
|
|
205
|
-
# +mode+ selects the neighbour rules of modes.md 3.4 -- delimiter-neighbour
|
|
206
|
-
# for markdown, the reference rule for markdown and html -- and +guarded+ holds, per span,
|
|
255
|
+
# +mode+ selects the neighbour rules of modes.md 3.4 -- delimiter-neighbour, the escape rule
|
|
256
|
+
# and the bracket-shape rule for markdown, the reference rule for markdown and html -- and +guarded+ holds, per span,
|
|
207
257
|
# whether its first code point is the terminator of a bare web address (modes.md 3.7.3).
|
|
208
258
|
def self.filter_boundary_edits(cp, edits, ranges, mode: nil, guarded: [])
|
|
209
259
|
edits.select do |edit|
|
|
@@ -216,6 +266,7 @@ module Polytypo
|
|
|
216
266
|
markdown = mode == "markdown"
|
|
217
267
|
next false if markdown && changes_delimiter_flank?(cp, p, q, edit.replacement)
|
|
218
268
|
next false if markdown && creates_escape?(cp, p, q, edit.replacement)
|
|
269
|
+
next false if markdown && creates_bracket_shape?(cp, p, q, edit.replacement)
|
|
219
270
|
next false if (markdown || mode == "html") && creates_reference?(cp, p, q, edit.replacement)
|
|
220
271
|
|
|
221
272
|
span = span_containing(ranges, p)
|
data/lib/polytypo/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: polytypo
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.18.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Iurii Rogulia
|
|
@@ -125,6 +125,7 @@ files:
|
|
|
125
125
|
- lib/polytypo/engine/rules/apostrophe.rb
|
|
126
126
|
- lib/polytypo/engine/rules/dash_shared.rb
|
|
127
127
|
- lib/polytypo/engine/rules/dashes.rb
|
|
128
|
+
- lib/polytypo/engine/rules/dashes_paired.rb
|
|
128
129
|
- lib/polytypo/engine/rules/ellipsis.rb
|
|
129
130
|
- lib/polytypo/engine/rules/hyphen.rb
|
|
130
131
|
- lib/polytypo/engine/rules/nbsp.rb
|