js_regex 3.13.0 → 3.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. checksums.yaml +4 -4
  2. data/lib/js_regex/conversion.rb +2 -1
  3. data/lib/js_regex/converter/anchor_converter.rb +4 -2
  4. data/lib/js_regex/converter/assertion_converter.rb +2 -0
  5. data/lib/js_regex/converter/backreference_converter.rb +33 -12
  6. data/lib/js_regex/converter/base.rb +38 -27
  7. data/lib/js_regex/converter/conditional_converter.rb +12 -4
  8. data/lib/js_regex/converter/context.rb +54 -8
  9. data/lib/js_regex/converter/escape_converter.rb +6 -30
  10. data/lib/js_regex/converter/freespace_converter.rb +2 -0
  11. data/lib/js_regex/converter/group_converter.rb +18 -16
  12. data/lib/js_regex/converter/keep_converter.rb +2 -0
  13. data/lib/js_regex/converter/literal_converter.rb +16 -46
  14. data/lib/js_regex/converter/meta_converter.rb +15 -13
  15. data/lib/js_regex/converter/property_converter.rb +4 -20
  16. data/lib/js_regex/converter/set_converter.rb +10 -37
  17. data/lib/js_regex/converter/subexpression_converter.rb +2 -0
  18. data/lib/js_regex/converter/type_converter.rb +45 -39
  19. data/lib/js_regex/converter/unsupported_token_converter.rb +2 -0
  20. data/lib/js_regex/converter.rb +2 -0
  21. data/lib/js_regex/error.rb +2 -0
  22. data/lib/js_regex/node/checks.rb +70 -0
  23. data/lib/js_regex/node/conditional_parts.rb +17 -0
  24. data/lib/js_regex/node/quantifier.rb +134 -0
  25. data/lib/js_regex/node/stringification.rb +14 -0
  26. data/lib/js_regex/node/transformations.rb +56 -0
  27. data/lib/js_regex/node/traversal.rb +24 -0
  28. data/lib/js_regex/node.rb +49 -40
  29. data/lib/js_regex/second_pass/alternate_conditional_permutations.rb +89 -0
  30. data/lib/js_regex/second_pass/delimit_backrefs.rb +31 -0
  31. data/lib/js_regex/second_pass/expand_bounded_capture_loops.rb +79 -0
  32. data/lib/js_regex/second_pass/expand_constant_capture_loops.rb +287 -0
  33. data/lib/js_regex/second_pass/expand_nullable_captures.rb +327 -0
  34. data/lib/js_regex/second_pass/handle_non_participating_backrefs.rb +69 -0
  35. data/lib/js_regex/second_pass/preserve_repeated_captures.rb +370 -0
  36. data/lib/js_regex/second_pass/resolve_capture_references.rb +116 -0
  37. data/lib/js_regex/second_pass/substitute_root_level_keep_mark.rb +26 -0
  38. data/lib/js_regex/second_pass.rb +364 -107
  39. data/lib/js_regex/target.rb +2 -0
  40. data/lib/js_regex/utils/captures.rb +46 -0
  41. data/lib/js_regex/utils/char_sets.rb +34 -0
  42. data/lib/js_regex/utils/char_types.rb +18 -0
  43. data/lib/js_regex/utils/escapes.rb +32 -0
  44. data/lib/js_regex/utils/literals.rb +53 -0
  45. data/lib/js_regex/utils/properties.rb +19 -0
  46. data/lib/js_regex/{converter → utils}/property_map.csv +31 -5
  47. data/lib/js_regex/utils.rb +8 -0
  48. data/lib/js_regex/version.rb +3 -1
  49. data/lib/js_regex.rb +5 -3
  50. metadata +29 -10
@@ -0,0 +1,327 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Separate choices that skip a capture from choices that capture an empty string.
4
+ # Copy the rest of the pattern into each choice so references and conditionals
5
+ # use that choice's capture state without changing greedy or lazy match order.
6
+ # For nullable loops, keep the final empty capture outside the repeated body.
7
+ # Example: /(a*)?\1/ => /(?:(a*)\1|(a*){0}(?!))/.
8
+ # The absent choice fails; the present choice may still capture an empty string.
9
+ class JsRegex::SecondPass::ExpandNullableCaptures < JsRegex::SecondPass
10
+ def call
11
+ return unless tree.walk.any? do |node|
12
+ node.type == :captured_group && !node.marked_nonempty?
13
+ end
14
+
15
+ loop do
16
+ @counts = reference_counts(tree)
17
+ break if @counts.empty? || !expand(tree)
18
+
19
+ renumber_captures
20
+ end
21
+ end
22
+
23
+ private
24
+
25
+ # /(a)(?:b(c))/ => the capture nodes numbered [1, 2].
26
+ def captures(node)
27
+ node.walk.select { |part| part.type == :captured_group }
28
+ end
29
+
30
+ # In /b(a*)?\1c/, replace /(a*)?\1c/ together, so each choice gets its
31
+ # own continuation; return true after that one replacement.
32
+ def expand(node, repeated = false)
33
+ return false unless node.is_a?(Node)
34
+
35
+ repeated ||= node.repeated?
36
+ return true if node.children.any? { |child| expand(child, repeated) }
37
+ return false if repeated
38
+
39
+ # Compound atomic wrappers cannot be treated as ordinary sequences.
40
+ finish = sequence_end(node)
41
+ return false unless finish
42
+
43
+ node.children.each.with_index do |child, index|
44
+ next unless child.is_a?(Node)
45
+
46
+ groups = captures(child)
47
+ candidates = groups.reject do |capture|
48
+ capture.marked_nonempty? || @counts[capture.reference].zero?
49
+ end
50
+ next if candidates.empty?
51
+
52
+ following = Node.new(*node.children[(index + 1)...finish])
53
+ suffix_counts = reference_counts(following)
54
+ next unless candidates.any? do |capture|
55
+ suffix_counts[capture.reference] == @counts[capture.reference]
56
+ end
57
+
58
+ replacement = expand_choice(child, following, groups)
59
+ next unless replacement
60
+
61
+ node.update(children: [*node.children[0...index], replacement, *node.children[finish..-1]])
62
+ return true
63
+ end
64
+ false
65
+ end
66
+
67
+ # For /(?:(a*)?)\1/, move the continuation into the passive group:
68
+ # /(?:(a*)?\1)/, allowing a subsequent expansion of the inner optional capture.
69
+ def expand_choice(child, following, groups)
70
+ if groups.length == 1
71
+ if greedy_nullable_loop?(child)
72
+ return expand_nullable_loop(child, following)
73
+ elsif child.quantifier&.reluctant?
74
+ collapsed = collapse_lazy_character_run(child, following)
75
+ return collapsed if collapsed
76
+ end
77
+ end
78
+
79
+ if child.quantifier&.unbounded? && child.quantifier.minimum_at_most_one? && !child.quantifier.reluctant?
80
+ replacement = expand_nullable_alternation(child, following, groups)
81
+ return replacement if replacement
82
+ end
83
+
84
+ choices = choices_for(child)
85
+ if choices
86
+ original_groups = groups.map(&:reference)
87
+ alternate(choices.map do |choice|
88
+ continuation = following.clone
89
+ specialize(continuation, capture_states(choice, original_groups))
90
+ Node.new(choice, continuation, type: :branch)
91
+ end)
92
+ elsif passive_sequence?(child)
93
+ position = child.children.first == '(?:' ? child.children.length - 1 : child.children.length
94
+ child.children.insert(position, *following.children)
95
+ child
96
+ end
97
+ end
98
+
99
+ # /(a*)+?\1/ => /a*()/: the greedy run first consumes every a. Its reference
100
+ # then fails unless the run was empty, so Ruby takes one final empty iteration.
101
+ # Restrict this to a single-character run and a lone required reference;
102
+ # a longer continuation can force backtracking to a different capture value.
103
+ def collapse_lazy_character_run(capture, following)
104
+ quantity = capture.quantifier
105
+ return unless capture.type == :captured_group && quantity&.reluctant? &&
106
+ quantity.unbounded? && quantity.minimum_at_most_one?
107
+
108
+ reference = following.children.first
109
+ return unless following.children.one? && reference.is_a?(Node) && reference.type == :backref &&
110
+ reference.reference == capture.reference && !reference.quantifier
111
+
112
+ body = Node.new(*capture.children[1...-1])
113
+ expression = single_expression(Regexp::Parser.parse(body.to_s))
114
+ repeat = Node::Quantifier.from_expression(expression)
115
+ return unless repeat&.zero_or_more? && !repeat.reluctant?
116
+
117
+ expression.quantifier = nil
118
+ expression = single_expression(expression)
119
+ return unless CODEPOINT_TYPES[expression.type] &&
120
+ (ml = expression.match_length).fixed? && ml.min == 1
121
+
122
+ Node.new(body, empty_capture(capture))
123
+ end
124
+
125
+ CODEPOINT_TYPES = %i[literal set type property nonproperty escape].to_h { |k| [k, true] }.freeze
126
+
127
+ # /(?:a)/ => the a expression; /(?:ab)*/ stays wrapped because it is quantified.
128
+ def single_expression(expression)
129
+ while !expression.quantifier &&
130
+ [:group, :expression].include?(expression.type) &&
131
+ expression.expressions.one?
132
+ expression = expression.expressions.first
133
+ end
134
+ expression
135
+ end
136
+
137
+ # A nullable alternative can end the loop before a later, consuming
138
+ # alternative is tried. Test that stopping choice before each nonempty step.
139
+ # For /(?:(a*)|b)+\1/ on 'b', try the final empty capture before consuming b.
140
+ # Shape: guarded nonempty iterations, then /()\1/ or a retained-capture tail.
141
+ def expand_nullable_alternation(loop_node, following, groups)
142
+ referenced = groups.select { |group| @counts[group.reference] > 0 }
143
+ return unless referenced.length == 1 && referenced.first.nullable?
144
+ return unless reference_counts(loop_node).empty?
145
+
146
+ capture = referenced.first
147
+ iteration = loop_node.with(quantifier: nil)
148
+ choices = split_alternatives(iteration)
149
+ return unless choices && choices.length > 1
150
+
151
+ empty_choices = choices.select { |choice| captures(choice).any? { |group| group.reference == capture.reference } }
152
+ return unless empty_choices.length == 1 && capture_only?(empty_choices.first, capture.reference)
153
+ return if (choices - empty_choices).all? { |choice| may_match_empty?(choice) }
154
+
155
+ present_tail = following.clone
156
+ specialize(present_tail, capture.reference => true)
157
+ probe_body = Node.new('(', iteration.clone, ')', reference: capture.reference)
158
+ stop = stop_before_empty_capture(probe_body, present_tail)
159
+ position = temporary_capture('[\\s\\S]*')
160
+ mark_nonempty_capture(iteration, capture.reference)
161
+ prefix = Node.new('(?:', stop, Node.new('(?=', position, ')'), iteration,
162
+ require_progress(position.reference), ')')
163
+ prefix.quantify(0, -1)
164
+
165
+ empty = empty_choices.first
166
+ empty.walk do |part|
167
+ part.update(children: ['(', ')'], quantifier: nil) if part.type == :captured_group && part.reference == capture.reference
168
+ end
169
+ retained_tail = following.clone
170
+ retained_tail.walk do |part|
171
+ make_condition_dynamic(part) if part.conditional? && part.reference == capture.reference
172
+ end
173
+ unless loop_node.optional?
174
+ start = temporary_capture('[\\s\\S]*')
175
+ prefix = Node.new(Node.new('(?=', start, ')'), prefix)
176
+ retained_tail = Node.new(require_progress(start.reference), retained_tail)
177
+ end
178
+ Node.new(prefix, alternate([Node.new(empty, present_tail), retained_tail]))
179
+ end
180
+
181
+ # For capture 1, /(?:(a*))/ => true; /(?:b(a*))/ => false.
182
+ def capture_only?(node, reference)
183
+ return ['(', '(?:', ')', ''].include?(node) if node.is_a?(String)
184
+ return !node.quantifier if node.type == :captured_group && node.reference == reference
185
+ return false if node.quantifier
186
+
187
+ node.children.all? { |child| capture_only?(child, reference) }
188
+ end
189
+
190
+ # Inside a guarded consuming iteration, (a*) keeps its source but changes
191
+ # its nonempty annotation from false to true.
192
+ def mark_nonempty_capture(node, reference)
193
+ node.children.map! do |child|
194
+ next child if child.is_a?(String)
195
+
196
+ if child.type == :captured_group && child.reference == reference
197
+ child.with(children: child.children, marked_nonempty: true, nullable: false)
198
+ else
199
+ mark_nonempty_capture(child, reference)
200
+ child
201
+ end
202
+ end
203
+ end
204
+
205
+ # /(a*)+/ => true; /(a*)+?/ => false.
206
+ def greedy_nullable_loop?(node)
207
+ node.type == :captured_group && node.nullable? && !!node.quantifier&.unbounded? && node.quantifier.minimum_at_most_one? && !node.quantifier.reluctant?
208
+ end
209
+
210
+ # /(?:a(b))/ => true; /(?:a(b))*/ => false because it is repeated.
211
+ def passive_sequence?(node)
212
+ return false unless node.type == :plain && !node.quantifier && node.children.any? { |child| child.is_a?(Node) }
213
+
214
+ node.children.first == '(?:' || !node.children.any? { |child| child.is_a?(String) }
215
+ end
216
+
217
+ # /(a*)?/ => [/(a*)/, /(a*){0}/]; /(a*)??/ gives the reverse order.
218
+ def choices_for(node)
219
+ if node.zero_or_one?
220
+ present = node.with(quantifier: nil)
221
+ absent = node.clone
222
+ discard_contents(absent)
223
+ node.quantifier.reluctant? ? [absent, present] : [present, absent]
224
+ elsif !node.quantifier
225
+ split_alternatives(node)
226
+ end
227
+ end
228
+
229
+ # Shape: /(a*)+\1/ => guarded nonempty iterations followed by either
230
+ # /()\1/ or /\1/ using the last nonempty capture. The empty ending is tried first.
231
+ def expand_nullable_loop(capture, following)
232
+ present_tail = following.clone
233
+ specialize(present_tail, capture.reference => true)
234
+ prefix = nonempty_iterations(capture, present_tail)
235
+ ending = alternate([
236
+ Node.new(empty_capture(capture), present_tail),
237
+ last_nonempty_tail(capture, following),
238
+ ])
239
+ Node.new(prefix, ending)
240
+ end
241
+
242
+ # For the repeated body (a*), consuming 'aaa' is allowed; consuming '' is
243
+ # blocked by a position check and left to the separate empty ending.
244
+ def nonempty_iterations(capture, present_tail)
245
+ stop = stop_before_empty_capture(capture, present_tail)
246
+ remainder = temporary_capture('[\\s\\S]*')
247
+ participating = capture.with(quantifier: nil, marked_nonempty: true, nullable: false)
248
+ prefix = Node.new('(?:', stop, Node.new('(?=', remainder, ')'), participating,
249
+ require_progress(remainder.reference), ')')
250
+ prefix.quantify(0, -1)
251
+ end
252
+
253
+ # With capture /(a*)/ and tail /\1b/, input 'b' selects the empty ending;
254
+ # the emitted negative lookahead prevents another loop iteration there.
255
+ def stop_before_empty_capture(capture, following)
256
+ # Prefer Ruby's final empty iteration when the body first chooses empty
257
+ # and the rest of the pattern succeeds with that value.
258
+ probe = temporary_capture(*capture.children[1...-1].map(&:clone))
259
+ tail = following.clone
260
+ tail.walk do |part|
261
+ next unless part.reference == capture.reference && part.type != :captured_group
262
+
263
+ part.renumber(probe.reference)
264
+ end
265
+ Node.new('(?!', Node.new('(?=', probe, ')'), nonempty_capture_check(probe.reference, false), tail, ')')
266
+ end
267
+
268
+ # For /(a*)+/ followed by /\1/, emit /(?=[\s\S]*(?![\s\S])(?!\1))\1/,
269
+ # so this ending requires a previous nonempty capture.
270
+ def last_nonempty_tail(capture, following)
271
+ tail = following.clone
272
+ tail.walk do |part|
273
+ make_condition_dynamic(part) if part.conditional? && part.reference == capture.reference
274
+ end
275
+ capture.optional? ? tail : Node.new(nonempty_capture_check(capture.reference, true), tail)
276
+ end
277
+
278
+ # Choice /(a*)/ from /(?:(a*)|(b*))/ => {1 => true, 2 => false}.
279
+ def capture_states(node, original_groups)
280
+ found = captures(node).map(&:reference)
281
+ states = (original_groups - found).to_h { |ref| [ref, false] }
282
+ collect_states(node, states)
283
+ states
284
+ end
285
+
286
+ # /(a*)(b*)?(c*){0}/ => {1 => true, 3 => false}; capture 2 stays unknown.
287
+ def collect_states(node, states, required = true, excluded = false)
288
+ return unless node.is_a?(Node)
289
+
290
+ excluded ||= node.zero_quantified?
291
+ required &&= !node.optional? && !node.negative_assertion?
292
+ if node.type == :captured_group
293
+ states[node.reference] = false if excluded
294
+ states[node.reference] = true if required && !excluded
295
+ end
296
+ required &&= !node.conditional? &&
297
+ !node.alternation?
298
+ node.children.each { |child| collect_states(child, states, required, excluded) }
299
+ end
300
+
301
+ # With {1 => false}, \1 => /(?!)/ and (?(1)a|b) selects b.
302
+ # With {1 => true}, the same condition selects a, even for an empty capture.
303
+ def specialize(node, states)
304
+ return unless node.is_a?(Node)
305
+
306
+ case node.type
307
+ when :backref
308
+ node.make_unmatchable if states[node.reference] == false
309
+ when :multiplex_backref, :capture_backref
310
+ children = node.children.reject { |child| states[child.reference] == false }
311
+ if children.empty?
312
+ node.make_unmatchable
313
+ else
314
+ node.update(children: children)
315
+ end
316
+ when :conditional
317
+ select_conditional_branch(node, states[node.reference]) if states.key?(node.reference)
318
+ when :capture_conditional
319
+ references = node.condition_references
320
+ present = references.any? { |reference| states[reference] }
321
+ if present || references.all? { |reference| states[reference] == false }
322
+ select_conditional_branch(node, present)
323
+ end
324
+ end
325
+ node.children.each { |child| specialize(child, states) }
326
+ end
327
+ end
@@ -0,0 +1,69 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Make backreferences to skipped groups fail, as in Ruby, instead of matching
4
+ # an empty string as in JavaScript. Check capture values when they cannot be
5
+ # empty; otherwise, require the group to participate where the reference is required.
6
+ # Example: /\1(a)/ => /(?!)(a)/: the forward reference cannot succeed.
7
+ class JsRegex::SecondPass::HandleNonParticipatingBackrefs < JsRegex::SecondPass
8
+ def call
9
+ nodes, paths = reference_nodes_with_paths
10
+ groups = {}
11
+ nodes.each do |node|
12
+ if node.type == :captured_group
13
+ groups[node.reference] = node
14
+ elsif node.type == :backref
15
+ guard_backreference(node, groups[node.reference], paths)
16
+ end
17
+ end
18
+ end
19
+
20
+ private
21
+
22
+ # /(a)|b\1/ => /(a)|b(?!)/: capture and reference are in different branches.
23
+ def guard_backreference(node, capture, paths)
24
+ path = paths[node]
25
+ capture_path = capture && paths[capture]
26
+ if !capture || path.include?(capture) || mutually_exclusive?(capture_path, path)
27
+ node.make_unmatchable
28
+ return
29
+ end
30
+
31
+ skipped = capture_path.select { |ancestor| skippable_scope?(ancestor) } - path
32
+ if skipped.any? { |ancestor| ancestor.negative_assertion? }
33
+ node.make_unmatchable
34
+ elsif skipped.any?
35
+ require_participation(node, capture, capture_path, path)
36
+ end
37
+ end
38
+
39
+ # /(a)?\1/ => /(a)?(?:(?=[\s\S]*(?![\s\S])(?!\1))\1)/,
40
+ # checking the capture at the reference without forcing the optional group.
41
+ def require_participation(node, capture, capture_path, path)
42
+ if capture.marked_nonempty?
43
+ reference = node.with(quantifier: nil)
44
+ node.update(type: :plain, reference: nil, children: ['(?:', nonempty_capture_check(reference.reference, true), reference, ')'])
45
+ else
46
+ require_nullable_capture(node, capture_path, path)
47
+ end
48
+ end
49
+
50
+ # /(?:(a*)?\1)?/ => /(?:(a*)\1)?/: require the capture inside the shared
51
+ # scope, while keeping that entire scope optional.
52
+ def require_nullable_capture(backref, group_path, path)
53
+ return if path[-2].walk.any? do |node|
54
+ node.type == :capture_check && node.reference == backref.reference
55
+ end
56
+
57
+ common_length = common_path_length(group_path, path)
58
+ common = group_path[common_length - 1]
59
+ remaining_path = path[common_length..-1]
60
+ return if remaining_path.any? { |ancestor| skippable_scope?(ancestor) }
61
+
62
+ # Every match through this node uses the reference, so paths that skip its
63
+ # capture cannot succeed. Keep the enclosing node's own optionality intact.
64
+ quantifier = common.quantifier
65
+ common.update(quantifier: nil)
66
+ require_group_participation(common, backref.reference)
67
+ common.update(quantifier: quantifier)
68
+ end
69
+ end