polytypo 1.17.4 → 1.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +4 -4
  2. data/lib/polytypo/data/VERSION +1 -1
  3. data/lib/polytypo/data/fixtures/bg.json +1 -1
  4. data/lib/polytypo/data/fixtures/cs.json +1 -1
  5. data/lib/polytypo/data/fixtures/de-CH.json +1 -1
  6. data/lib/polytypo/data/fixtures/de-DE.json +1 -1
  7. data/lib/polytypo/data/fixtures/el.json +1 -1
  8. data/lib/polytypo/data/fixtures/en-GB.json +64 -1
  9. data/lib/polytypo/data/fixtures/en-US.json +64 -1
  10. data/lib/polytypo/data/fixtures/es.json +272 -7
  11. data/lib/polytypo/data/fixtures/fi.json +1 -1
  12. data/lib/polytypo/data/fixtures/fr-CA.json +1 -1
  13. data/lib/polytypo/data/fixtures/fr.json +1 -1
  14. data/lib/polytypo/data/fixtures/hr.json +1 -1
  15. data/lib/polytypo/data/fixtures/hu.json +1 -1
  16. data/lib/polytypo/data/fixtures/it.json +1 -1
  17. data/lib/polytypo/data/fixtures/locale-resolution.json +1 -1
  18. data/lib/polytypo/data/fixtures/lt.json +1 -1
  19. data/lib/polytypo/data/fixtures/nl.json +1 -1
  20. data/lib/polytypo/data/fixtures/pl.json +1 -1
  21. data/lib/polytypo/data/fixtures/pt-BR.json +1 -1
  22. data/lib/polytypo/data/fixtures/pt-PT.json +1 -1
  23. data/lib/polytypo/data/fixtures/ro.json +1 -1
  24. data/lib/polytypo/data/fixtures/ru.json +1 -1
  25. data/lib/polytypo/data/fixtures/sk.json +1 -1
  26. data/lib/polytypo/data/fixtures/sl.json +1 -1
  27. data/lib/polytypo/data/fixtures/sr.json +1 -1
  28. data/lib/polytypo/data/fixtures/sv.json +1 -1
  29. data/lib/polytypo/data/fixtures/tr.json +1 -1
  30. data/lib/polytypo/data/fixtures/uk.json +1 -1
  31. data/lib/polytypo/data/locales/el.json +1 -1
  32. data/lib/polytypo/data/locales/es.json +2 -2
  33. data/lib/polytypo/data/locales/registry.json +1 -1
  34. data/lib/polytypo/data/rules/dashes.md +108 -16
  35. data/lib/polytypo/data/rules/modes.md +49 -4
  36. data/lib/polytypo/data/rules/order.json +2 -2
  37. data/lib/polytypo/data/rules/pipeline-idempotency.md +7 -0
  38. data/lib/polytypo/data/schema/locale.schema.json +2 -2
  39. data/lib/polytypo/engine/rules/dash_shared.rb +32 -21
  40. data/lib/polytypo/engine/rules/dashes.rb +4 -1
  41. data/lib/polytypo/engine/rules/dashes_paired.rb +160 -0
  42. data/lib/polytypo/modes/spans.rb +53 -2
  43. data/lib/polytypo/version.rb +1 -1
  44. metadata +2 -1
@@ -0,0 +1,160 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "dash_shared"
4
+ require_relative "../edits"
5
+ require_relative "../sentinels"
6
+ require_relative "../unicode_util"
7
+
8
+ module Polytypo
9
+ module Engine
10
+ module Rules
11
+ # `dash.parenthetical = "em-paired"` -- spec/rules/dashes.md 3.4a. The pair is written with a
12
+ # space outside it and none inside (`Llego —por fin— a casa`), which no per-token rule can
13
+ # express: which stroke opens and which closes is a property of the sentence, not of the
14
+ # token. A sentence is converted only when it holds exactly two counted strokes and both are
15
+ # eligible; anything else is left as typed. Mirrors dashes-paired.ts in the JS reference
16
+ # implementation.
17
+ module DashesPaired
18
+ # One counted DASH run of a segment: its span [s, e), outer spacing, the flanking indices
19
+ # after the joiner walk, and whether it may take part in a conversion.
20
+ Run = Struct.new(:s, :e, :lsp, :rsp, :left, :right, :eligible, keyword_init: true)
21
+
22
+ TERMINALS = [0x2e, 0x3f, 0x21, 0x2026].freeze
23
+
24
+ module_function
25
+
26
+ def terminal?(cp)
27
+ TERMINALS.include?(cp)
28
+ end
29
+
30
+ # A terminal ends a segment only before a space (U+0020 or a no-break space, which `nbsp`
31
+ # puts there later), a line break, a marker or the end of input.
32
+ def ends_segment?(cp, i)
33
+ return false unless terminal?(cp[i])
34
+ return true if i + 1 >= cp.length
35
+
36
+ nxt = cp[i + 1]
37
+ nxt == DashShared::SPACE || DashShared.no_break_space?(nxt) || DashShared.break?(nxt) ||
38
+ Polytypo::Engine.marker?(nxt)
39
+ end
40
+
41
+ def wants_space?(cp)
42
+ Polytypo::Engine::UnicodeUtil.letter?(cp) || DashShared.digit?(cp)
43
+ end
44
+
45
+ # dashes.md 3.2 steps 5-7: content on the same line, no dash or space beside the stroke's
46
+ # outer spacing, and not part of a cluster holding another dash run.
47
+ def neighbours_clean?(cp, s, e, left, right)
48
+ left_cp = cp[left]
49
+ right_cp = cp[right]
50
+ return false if DashShared.break?(left_cp) || DashShared.break?(right_cp)
51
+ return false if DashShared.inert_dash?(left_cp) || DashShared.inert_dash?(right_cp)
52
+ return false if DashShared.dash?(left_cp) || DashShared.dash?(right_cp)
53
+ return false if left_cp == DashShared::SPACE || DashShared.no_break_space?(left_cp)
54
+ return false if right_cp == DashShared::SPACE || DashShared.no_break_space?(right_cp)
55
+
56
+ !DashShared.cluster_inert?(cp, s, e)
57
+ end
58
+
59
+ # Returns nil when the run is not a stroke at all, otherwise a Run (eligible or not).
60
+ def classify(cp, s, e)
61
+ n = cp.length
62
+ k = e - s
63
+ return nil if k > 3
64
+
65
+ lsp = s.positive? && cp[s - 1] == DashShared::SPACE ? 1 : 0
66
+ rsp = e < n && cp[e] == DashShared::SPACE ? 1 : 0
67
+ first = cp[s]
68
+ is_hyphen_shaped = first == DashShared::HYPHEN_MINUS || first == DashShared::HYPHEN ||
69
+ first == DashShared::MINUS_SIGN
70
+ # A hyphen inside a word is a compound, not a stroke.
71
+ return nil if k == 1 && is_hyphen_shaped && lsp.zero? && rsp.zero?
72
+ # The hyphen of `+/-`, which `symbols` (order 80) turns into a plus-minus sign: counting
73
+ # it would let that later rule change the number of strokes in the segment.
74
+ return nil if k == 1 && first == DashShared::HYPHEN_MINUS && s >= 2 && cp[s - 1] == 0x2f && cp[s - 2] == 0x2b
75
+
76
+ left = s - 1 - lsp
77
+ right = e + rsp
78
+ counted = lambda do |eligible|
79
+ Run.new(s: s, e: e, lsp: lsp, rsp: rsp, left: left, right: right, eligible: eligible)
80
+ end
81
+ return counted.call(false) if left.negative? || right >= n
82
+
83
+ join_start = left + 1
84
+ join_end = right
85
+ left -= 1 while left >= 0 && cp[left] == DashShared::WORD_JOINER
86
+ right += 1 while right < n && cp[right] == DashShared::WORD_JOINER
87
+ return counted.call(false) if left.negative? || right >= n
88
+
89
+ crossed_joiner = left + 1 != join_start || right != join_end
90
+
91
+ return nil unless DashShared.range_flanks(cp, left, right).nil?
92
+ return counted.call(false) if crossed_joiner
93
+
94
+ return counted.call(false) unless neighbours_clean?(cp, s, e, left, right)
95
+
96
+ # A stroke spaced on one side only is eligible solely as the one em dash that is already
97
+ # half of the target form; `-mente`, `pre-`, `--force` and `- item` are not dashes.
98
+ return counted.call(false) if lsp != rsp && !(k == 1 && first == DashShared::EM_DASH)
99
+ # P4: a tight stroke between two Roman numerals is a range already in its correct form.
100
+ return counted.call(false) if lsp.zero? && rsp.zero? && Dashes.roman_flanked?(cp, left, right)
101
+ # P5: an authored lone en dash is declined in every locale.
102
+ return counted.call(false) if k == 1 && first == DashShared::EN_DASH
103
+
104
+ counted.call(true)
105
+ end
106
+
107
+ def scan(cp)
108
+ n = cp.length
109
+ edits = []
110
+ segment = []
111
+ i = 0
112
+
113
+ while i < n
114
+ if DashShared.dash?(cp[i])
115
+ s = i
116
+ e = s
117
+ e += 1 while e < n && DashShared.dash?(cp[e])
118
+ i = e
119
+ run = classify(cp, s, e)
120
+ segment << run unless run.nil?
121
+ next
122
+ end
123
+
124
+ if ends_segment?(cp, i)
125
+ emit_pair(cp, segment[0], segment[1], edits) if segment.length == 2
126
+ segment = []
127
+ end
128
+ i += 1
129
+ end
130
+ emit_pair(cp, segment[0], segment[1], edits) if segment.length == 2
131
+
132
+ edits
133
+ end
134
+
135
+ def emit_pair(cp, a, b, edits)
136
+ return unless a.eligible && b.eligible
137
+ return if a.lsp.zero? && a.rsp == 1 # the first stroke is shaped like a closer
138
+ return if b.lsp == 1 && b.rsp.zero? # the second stroke is shaped like an opener
139
+ return if (a.e...b.s).any? { |j| DashShared.break?(cp[j]) }
140
+
141
+ insert_before = a.lsp.zero? && wants_space?(cp[a.left])
142
+ insert_after = b.rsp.zero? && wants_space?(cp[b.right])
143
+ return if insert_before && DashShared.spacing_transition_blocked_left?(cp, a.left)
144
+ return if insert_after && DashShared.spacing_transition_blocked_right?(cp, b.right)
145
+
146
+ opener = a.lsp == 1 || insert_before ? [DashShared::SPACE, DashShared::EM_DASH] : [DashShared::EM_DASH]
147
+ closer = b.rsp == 1 || insert_after ? [DashShared::EM_DASH, DashShared::SPACE] : [DashShared::EM_DASH]
148
+
149
+ [[a, opener], [b, closer]].each do |run, replacement|
150
+ span_start = run.s - run.lsp
151
+ span_end = run.e + run.rsp
152
+ next if DashShared.same_content?(cp, span_start, span_end, replacement)
153
+
154
+ edits << Polytypo::Engine::Edit.new(span_start, span_end, replacement, "dashes")
155
+ end
156
+ end
157
+ end
158
+ end
159
+ end
160
+ end
@@ -189,6 +189,56 @@ module Polytypo
189
189
  end
190
190
  private_class_method :creates_reference?
191
191
 
192
+ # modes.md 3.4 bracket-shape rule (markdown only, spec 1.17.5): a deletion that joins a
193
+ # task-list marker or a footnote label. A marker or a position outside the array is in no
194
+ # class. Ruby indexes negatively, so every read goes through +code_point_at+.
195
+ def self.code_point_at(cp, i)
196
+ i.negative? ? nil : cp[i]
197
+ end
198
+ private_class_method :code_point_at
199
+
200
+ def self.skip_spaces(cp, i, step)
201
+ k = i
202
+ k += step while k >= 0 && k < cp.length && cp[k] == SPACE
203
+ k
204
+ end
205
+ private_class_method :skip_spaces
206
+
207
+ def self.x_letter?(c)
208
+ c == 0x78 || c == 0x58
209
+ end
210
+ private_class_method :x_letter?
211
+
212
+ def self.label_character?(c)
213
+ !c.nil? && c >= 0 && c != SPACE && c != 0x5B && c != 0x5D
214
+ end
215
+ private_class_method :label_character?
216
+
217
+ def self.creates_bracket_shape?(cp, p, q, replacement)
218
+ is_collapse = replacement.length == 1 && replacement[0] == SPACE
219
+ return false unless replacement.empty? || is_collapse
220
+
221
+ a = skip_spaces(cp, p - 1, -1)
222
+ b = skip_spaces(cp, q + 1, 1)
223
+ at_a = code_point_at(cp, a)
224
+ at_b = code_point_at(cp, b)
225
+ return at_a == 0x5B && at_b == 0x5D if is_collapse
226
+ return true if at_a == 0x5B && at_b == 0x5D
227
+ if at_a == 0x5B
228
+ return true if at_b == 0x5E
229
+
230
+ return x_letter?(at_b) && code_point_at(cp, skip_spaces(cp, b + 1, 1)) == 0x5D
231
+ end
232
+ return false if at_b != 0x5D
233
+ return true if x_letter?(at_a) && code_point_at(cp, skip_spaces(cp, a - 1, -1)) == 0x5B
234
+
235
+ run = a
236
+ run -= 1 while run >= 0 && label_character?(cp[run])
237
+ first = run + 1
238
+ a - first >= 1 && cp[first] == 0x5E && code_point_at(cp, skip_spaces(cp, run, -1)) == 0x5B
239
+ end
240
+ private_class_method :creates_bracket_shape?
241
+
192
242
  # modes.md 3.4, two safety nets, both pure functions of (p, q, r, s0, s1):
193
243
  #
194
244
  # 1. No edit may contain a marker -- one that does is a bug, discarded rather than
@@ -202,8 +252,8 @@ module Polytypo
202
252
  # point any rule emits whose meaning comes from its position rather than from itself
203
253
  # (modes.md 5 item 2).
204
254
  #
205
- # +mode+ selects the neighbour rules of modes.md 3.4 -- delimiter-neighbour and the escape rule
206
- # for markdown, the reference rule for markdown and html -- and +guarded+ holds, per span,
255
+ # +mode+ selects the neighbour rules of modes.md 3.4 -- delimiter-neighbour, the escape rule
256
+ # and the bracket-shape rule for markdown, the reference rule for markdown and html -- and +guarded+ holds, per span,
207
257
  # whether its first code point is the terminator of a bare web address (modes.md 3.7.3).
208
258
  def self.filter_boundary_edits(cp, edits, ranges, mode: nil, guarded: [])
209
259
  edits.select do |edit|
@@ -216,6 +266,7 @@ module Polytypo
216
266
  markdown = mode == "markdown"
217
267
  next false if markdown && changes_delimiter_flank?(cp, p, q, edit.replacement)
218
268
  next false if markdown && creates_escape?(cp, p, q, edit.replacement)
269
+ next false if markdown && creates_bracket_shape?(cp, p, q, edit.replacement)
219
270
  next false if (markdown || mode == "html") && creates_reference?(cp, p, q, edit.replacement)
220
271
 
221
272
  span = span_containing(ranges, p)
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Polytypo
4
- VERSION = "1.17.4"
4
+ VERSION = "1.18.0"
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: polytypo
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.17.4
4
+ version: 1.18.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Iurii Rogulia
@@ -125,6 +125,7 @@ files:
125
125
  - lib/polytypo/engine/rules/apostrophe.rb
126
126
  - lib/polytypo/engine/rules/dash_shared.rb
127
127
  - lib/polytypo/engine/rules/dashes.rb
128
+ - lib/polytypo/engine/rules/dashes_paired.rb
128
129
  - lib/polytypo/engine/rules/ellipsis.rb
129
130
  - lib/polytypo/engine/rules/hyphen.rb
130
131
  - lib/polytypo/engine/rules/nbsp.rb