regexp_parser 2.7.0 → 2.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +7 -5
  3. data/LICENSE +1 -1
  4. data/Rakefile +5 -3
  5. data/lib/regexp_parser/error.rb +2 -0
  6. data/lib/regexp_parser/expression/base.rb +2 -7
  7. data/lib/regexp_parser/expression/classes/alternation.rb +3 -1
  8. data/lib/regexp_parser/expression/classes/anchor.rb +2 -0
  9. data/lib/regexp_parser/expression/classes/backreference.rb +7 -26
  10. data/lib/regexp_parser/expression/classes/character_set/intersection.rb +2 -0
  11. data/lib/regexp_parser/expression/classes/character_set/range.rb +4 -7
  12. data/lib/regexp_parser/expression/classes/character_set.rb +6 -8
  13. data/lib/regexp_parser/expression/classes/character_type.rb +2 -0
  14. data/lib/regexp_parser/expression/classes/conditional.rb +4 -28
  15. data/lib/regexp_parser/expression/classes/escape_sequence.rb +29 -96
  16. data/lib/regexp_parser/expression/classes/free_space.rb +5 -1
  17. data/lib/regexp_parser/expression/classes/group.rb +2 -22
  18. data/lib/regexp_parser/expression/classes/keep.rb +3 -1
  19. data/lib/regexp_parser/expression/classes/literal.rb +2 -0
  20. data/lib/regexp_parser/expression/classes/posix_class.rb +7 -5
  21. data/lib/regexp_parser/expression/classes/root.rb +2 -0
  22. data/lib/regexp_parser/expression/classes/unicode_property.rb +13 -11
  23. data/lib/regexp_parser/expression/methods/construct.rb +4 -4
  24. data/lib/regexp_parser/expression/methods/escape_sequence_char.rb +7 -0
  25. data/lib/regexp_parser/expression/methods/escape_sequence_codepoint.rb +76 -0
  26. data/lib/regexp_parser/expression/methods/human_name.rb +2 -0
  27. data/lib/regexp_parser/expression/methods/match.rb +2 -0
  28. data/lib/regexp_parser/expression/methods/match_length.rb +2 -0
  29. data/lib/regexp_parser/expression/methods/negative.rb +22 -0
  30. data/lib/regexp_parser/expression/methods/options.rb +2 -0
  31. data/lib/regexp_parser/expression/methods/parts.rb +25 -0
  32. data/lib/regexp_parser/expression/methods/printing.rb +28 -0
  33. data/lib/regexp_parser/expression/methods/referenced_expressions.rb +30 -0
  34. data/lib/regexp_parser/expression/methods/strfregexp.rb +2 -0
  35. data/lib/regexp_parser/expression/methods/tests.rb +42 -3
  36. data/lib/regexp_parser/expression/methods/traverse.rb +35 -20
  37. data/lib/regexp_parser/expression/quantifier.rb +33 -18
  38. data/lib/regexp_parser/expression/sequence.rb +7 -9
  39. data/lib/regexp_parser/expression/sequence_operation.rb +6 -9
  40. data/lib/regexp_parser/expression/shared.rb +40 -24
  41. data/lib/regexp_parser/expression/subexpression.rb +22 -18
  42. data/lib/regexp_parser/expression.rb +39 -31
  43. data/lib/regexp_parser/lexer.rb +17 -7
  44. data/lib/regexp_parser/parser.rb +101 -95
  45. data/lib/regexp_parser/scanner/errors/premature_end_error.rb +10 -0
  46. data/lib/regexp_parser/scanner/errors/scanner_error.rb +8 -0
  47. data/lib/regexp_parser/scanner/errors/validation_error.rb +65 -0
  48. data/lib/regexp_parser/scanner/properties/long.csv +61 -0
  49. data/lib/regexp_parser/scanner/properties/short.csv +15 -0
  50. data/lib/regexp_parser/scanner/property.rl +1 -1
  51. data/lib/regexp_parser/scanner/scanner.rl +90 -140
  52. data/lib/regexp_parser/scanner.rb +1336 -1450
  53. data/lib/regexp_parser/syntax/any.rb +2 -0
  54. data/lib/regexp_parser/syntax/base.rb +2 -0
  55. data/lib/regexp_parser/syntax/token/anchor.rb +5 -3
  56. data/lib/regexp_parser/syntax/token/assertion.rb +4 -2
  57. data/lib/regexp_parser/syntax/token/backreference.rb +11 -6
  58. data/lib/regexp_parser/syntax/token/character_set.rb +6 -1
  59. data/lib/regexp_parser/syntax/token/character_type.rb +6 -4
  60. data/lib/regexp_parser/syntax/token/conditional.rb +5 -3
  61. data/lib/regexp_parser/syntax/token/escape.rb +12 -8
  62. data/lib/regexp_parser/syntax/token/group.rb +8 -6
  63. data/lib/regexp_parser/syntax/token/keep.rb +3 -1
  64. data/lib/regexp_parser/syntax/token/meta.rb +11 -2
  65. data/lib/regexp_parser/syntax/token/posix_class.rb +4 -2
  66. data/lib/regexp_parser/syntax/token/quantifier.rb +8 -6
  67. data/lib/regexp_parser/syntax/token/unicode_property.rb +164 -70
  68. data/lib/regexp_parser/syntax/token/virtual.rb +13 -0
  69. data/lib/regexp_parser/syntax/token.rb +18 -16
  70. data/lib/regexp_parser/syntax/version_lookup.rb +4 -2
  71. data/lib/regexp_parser/syntax/versions/1.8.6.rb +2 -0
  72. data/lib/regexp_parser/syntax/versions/1.9.1.rb +2 -0
  73. data/lib/regexp_parser/syntax/versions/1.9.3.rb +2 -0
  74. data/lib/regexp_parser/syntax/versions/2.0.0.rb +2 -0
  75. data/lib/regexp_parser/syntax/versions/2.2.0.rb +2 -0
  76. data/lib/regexp_parser/syntax/versions/2.3.0.rb +2 -0
  77. data/lib/regexp_parser/syntax/versions/2.4.0.rb +2 -0
  78. data/lib/regexp_parser/syntax/versions/2.4.1.rb +2 -0
  79. data/lib/regexp_parser/syntax/versions/2.5.0.rb +2 -0
  80. data/lib/regexp_parser/syntax/versions/2.6.0.rb +2 -0
  81. data/lib/regexp_parser/syntax/versions/2.6.2.rb +2 -0
  82. data/lib/regexp_parser/syntax/versions/2.6.3.rb +2 -0
  83. data/lib/regexp_parser/syntax/versions/3.1.0.rb +2 -0
  84. data/lib/regexp_parser/syntax/versions/3.2.0.rb +2 -0
  85. data/lib/regexp_parser/syntax/versions/3.5.0.rb +4 -0
  86. data/lib/regexp_parser/syntax/versions/4.0.0.rb +4 -0
  87. data/lib/regexp_parser/syntax/versions.rb +3 -1
  88. data/lib/regexp_parser/syntax.rb +3 -1
  89. data/lib/regexp_parser/token.rb +2 -0
  90. data/lib/regexp_parser/version.rb +3 -1
  91. data/lib/regexp_parser.rb +8 -6
  92. data/regexp_parser.gemspec +7 -5
  93. metadata +19 -11
  94. data/CHANGELOG.md +0 -632
  95. data/README.md +0 -503
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Regexp::Expression
2
4
  class Subexpression < Regexp::Expression::Base
3
5
  include Enumerable
@@ -11,16 +13,15 @@ module Regexp::Expression
11
13
 
12
14
  # Override base method to clone the expressions as well.
13
15
  def initialize_copy(orig)
14
- self.expressions = orig.expressions.map(&:clone)
16
+ self.expressions = orig.expressions.map do |exp|
17
+ exp.clone.tap { |copy| copy.parent = self }
18
+ end
15
19
  super
16
20
  end
17
21
 
18
22
  def <<(exp)
19
- if exp.is_a?(WhiteSpace) && last && last.is_a?(WhiteSpace)
20
- last.merge(exp)
21
- else
22
- expressions << exp
23
- end
23
+ exp.parent = self
24
+ expressions << exp
24
25
  end
25
26
 
26
27
  %w[[] at each empty? fetch index join last length values_at].each do |method|
@@ -38,11 +39,7 @@ module Regexp::Expression
38
39
  end
39
40
 
40
41
  def te
41
- ts + to_s.length
42
- end
43
-
44
- def parts
45
- expressions
42
+ ts + base_length
46
43
  end
47
44
 
48
45
  def to_h
@@ -52,14 +49,21 @@ module Regexp::Expression
52
49
  )
53
50
  end
54
51
 
55
- def terminal?
56
- false
57
- end
58
-
59
- private
52
+ def extract_quantifier_target(quantifier_description)
53
+ pre_quantifier_decorations = []
54
+ target = expressions.reverse.find do |exp|
55
+ if exp.decorative?
56
+ exp.custom_to_s_handling = true
57
+ pre_quantifier_decorations << exp.text
58
+ next
59
+ end
60
+ exp
61
+ end
62
+ target or raise Regexp::Parser::ParserError,
63
+ "No valid target found for '#{quantifier_description}' quantifier"
60
64
 
61
- def intersperse(expressions, separator)
62
- expressions.flat_map { |exp| [exp, separator] }.slice(0...-1)
65
+ target.pre_quantifier_decorations = pre_quantifier_decorations
66
+ target
63
67
  end
64
68
  end
65
69
  end
@@ -1,34 +1,42 @@
1
- require 'regexp_parser/error'
1
+ # frozen_string_literal: true
2
2
 
3
- require 'regexp_parser/expression/shared'
4
- require 'regexp_parser/expression/base'
5
- require 'regexp_parser/expression/quantifier'
6
- require 'regexp_parser/expression/subexpression'
7
- require 'regexp_parser/expression/sequence'
8
- require 'regexp_parser/expression/sequence_operation'
3
+ require_relative 'error'
9
4
 
10
- require 'regexp_parser/expression/classes/alternation'
11
- require 'regexp_parser/expression/classes/anchor'
12
- require 'regexp_parser/expression/classes/backreference'
13
- require 'regexp_parser/expression/classes/character_set'
14
- require 'regexp_parser/expression/classes/character_set/intersection'
15
- require 'regexp_parser/expression/classes/character_set/range'
16
- require 'regexp_parser/expression/classes/character_type'
17
- require 'regexp_parser/expression/classes/conditional'
18
- require 'regexp_parser/expression/classes/escape_sequence'
19
- require 'regexp_parser/expression/classes/free_space'
20
- require 'regexp_parser/expression/classes/group'
21
- require 'regexp_parser/expression/classes/keep'
22
- require 'regexp_parser/expression/classes/literal'
23
- require 'regexp_parser/expression/classes/posix_class'
24
- require 'regexp_parser/expression/classes/root'
25
- require 'regexp_parser/expression/classes/unicode_property'
5
+ require_relative 'expression/shared'
6
+ require_relative 'expression/base'
7
+ require_relative 'expression/quantifier'
8
+ require_relative 'expression/subexpression'
9
+ require_relative 'expression/sequence'
10
+ require_relative 'expression/sequence_operation'
26
11
 
27
- require 'regexp_parser/expression/methods/construct'
28
- require 'regexp_parser/expression/methods/human_name'
29
- require 'regexp_parser/expression/methods/match'
30
- require 'regexp_parser/expression/methods/match_length'
31
- require 'regexp_parser/expression/methods/options'
32
- require 'regexp_parser/expression/methods/strfregexp'
33
- require 'regexp_parser/expression/methods/tests'
34
- require 'regexp_parser/expression/methods/traverse'
12
+ require_relative 'expression/classes/alternation'
13
+ require_relative 'expression/classes/anchor'
14
+ require_relative 'expression/classes/backreference'
15
+ require_relative 'expression/classes/character_set'
16
+ require_relative 'expression/classes/character_set/intersection'
17
+ require_relative 'expression/classes/character_set/range'
18
+ require_relative 'expression/classes/character_type'
19
+ require_relative 'expression/classes/conditional'
20
+ require_relative 'expression/classes/escape_sequence'
21
+ require_relative 'expression/classes/free_space'
22
+ require_relative 'expression/classes/group'
23
+ require_relative 'expression/classes/keep'
24
+ require_relative 'expression/classes/literal'
25
+ require_relative 'expression/classes/posix_class'
26
+ require_relative 'expression/classes/root'
27
+ require_relative 'expression/classes/unicode_property'
28
+
29
+ require_relative 'expression/methods/construct'
30
+ require_relative 'expression/methods/escape_sequence_char'
31
+ require_relative 'expression/methods/escape_sequence_codepoint'
32
+ require_relative 'expression/methods/human_name'
33
+ require_relative 'expression/methods/match'
34
+ require_relative 'expression/methods/match_length'
35
+ require_relative 'expression/methods/negative'
36
+ require_relative 'expression/methods/options'
37
+ require_relative 'expression/methods/parts'
38
+ require_relative 'expression/methods/printing'
39
+ require_relative 'expression/methods/referenced_expressions'
40
+ require_relative 'expression/methods/strfregexp'
41
+ require_relative 'expression/methods/tests'
42
+ require_relative 'expression/methods/traverse'
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  # A very thin wrapper around the scanner that breaks quantified literal runs,
2
4
  # collects emitted tokens into an array, calculates their nesting depth, and
3
5
  # normalizes tokens for the parser, and checks if they are implemented by the
@@ -6,7 +8,7 @@ class Regexp::Lexer
6
8
 
7
9
  OPENING_TOKENS = %i[
8
10
  capture passive lookahead nlookahead lookbehind nlookbehind
9
- atomic options options_switch named absence
11
+ atomic options options_switch named absence open
10
12
  ].freeze
11
13
 
12
14
  CLOSING_TOKENS = %i[close].freeze
@@ -89,24 +91,32 @@ class Regexp::Lexer
89
91
  :nesting, :set_nesting, :conditional_nesting, :shift
90
92
 
91
93
  def ascend(type, token)
94
+ return unless CLOSING_TOKENS.include?(token)
95
+
92
96
  case type
93
97
  when :group, :assertion
94
- self.nesting = nesting - 1 if CLOSING_TOKENS.include?(token)
98
+ self.nesting = nesting - 1
95
99
  when :set
96
- self.set_nesting = set_nesting - 1 if token == :close
100
+ self.set_nesting = set_nesting - 1
97
101
  when :conditional
98
- self.conditional_nesting = conditional_nesting - 1 if token == :close
102
+ self.conditional_nesting = conditional_nesting - 1
103
+ else
104
+ raise "unhandled nesting type #{type}"
99
105
  end
100
106
  end
101
107
 
102
108
  def descend(type, token)
109
+ return unless OPENING_TOKENS.include?(token)
110
+
103
111
  case type
104
112
  when :group, :assertion
105
- self.nesting = nesting + 1 if OPENING_TOKENS.include?(token)
113
+ self.nesting = nesting + 1
106
114
  when :set
107
- self.set_nesting = set_nesting + 1 if token == :open
115
+ self.set_nesting = set_nesting + 1
108
116
  when :conditional
109
- self.conditional_nesting = conditional_nesting + 1 if token == :open
117
+ self.conditional_nesting = conditional_nesting + 1
118
+ else
119
+ raise "unhandled nesting type #{type}"
110
120
  end
111
121
  end
112
122
 
@@ -1,5 +1,7 @@
1
- require 'regexp_parser/error'
2
- require 'regexp_parser/expression'
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'error'
4
+ require_relative 'expression'
3
5
 
4
6
  class Regexp::Parser
5
7
  include Regexp::Expression
@@ -232,7 +234,7 @@ class Regexp::Parser
232
234
  node << Backreference::NameRecursionLevel.new(token, active_opts)
233
235
  when :name_call
234
236
  node << Backreference::NameCall.new(token, active_opts)
235
- when :number, :number_ref
237
+ when :number, :number_ref # TODO: split in v3.0.0
236
238
  node << Backreference::Number.new(token, active_opts)
237
239
  when :number_recursion_ref
238
240
  node << Backreference::NumberRecursionLevel.new(token, active_opts).tap do |exp|
@@ -272,9 +274,9 @@ class Regexp::Parser
272
274
  nest_conditional(Conditional::Expression.new(token, active_opts))
273
275
  when :condition
274
276
  conditional_nesting.last.condition = Conditional::Condition.new(token, active_opts)
275
- conditional_nesting.last.add_sequence(active_opts)
277
+ conditional_nesting.last.add_sequence(active_opts, { ts: token.te })
276
278
  when :separator
277
- conditional_nesting.last.add_sequence(active_opts)
279
+ conditional_nesting.last.add_sequence(active_opts, { ts: token.te })
278
280
  self.node = conditional_nesting.last.branches.last
279
281
  when :close
280
282
  conditional_nesting.pop
@@ -319,9 +321,11 @@ class Regexp::Parser
319
321
  when :codepoint_list; node << EscapeSequence::CodepointList.new(token, active_opts)
320
322
  when :hex; node << EscapeSequence::Hex.new(token, active_opts)
321
323
  when :octal; node << EscapeSequence::Octal.new(token, active_opts)
324
+ when :utf8_hex; node << EscapeSequence::UTF8Hex.new(token, active_opts)
322
325
 
323
326
  when :control
324
327
  if token.text =~ /\A(?:\\C-\\M|\\c\\M)/
328
+ # TODO: emit :meta_control_sequence token in v3.0.0
325
329
  node << EscapeSequence::MetaControl.new(token, active_opts)
326
330
  else
327
331
  node << EscapeSequence::Control.new(token, active_opts)
@@ -329,6 +333,7 @@ class Regexp::Parser
329
333
 
330
334
  when :meta_sequence
331
335
  if token.text =~ /\A\\M-\\[Cc]/
336
+ # TODO: emit :meta_control_sequence token in v3.0.0:
332
337
  node << EscapeSequence::MetaControl.new(token, active_opts)
333
338
  else
334
339
  node << EscapeSequence::Meta.new(token, active_opts)
@@ -349,11 +354,7 @@ class Regexp::Parser
349
354
  when :comment
350
355
  node << Comment.new(token, active_opts)
351
356
  when :whitespace
352
- if node.last.is_a?(WhiteSpace)
353
- node.last.merge(WhiteSpace.new(token, active_opts))
354
- else
355
- node << WhiteSpace.new(token, active_opts)
356
- end
357
+ node << WhiteSpace.new(token, active_opts)
357
358
  else
358
359
  raise UnknownTokenError.new('FreeSpace', token)
359
360
  end
@@ -381,96 +382,97 @@ class Regexp::Parser
381
382
  def sequence_operation(klass, token)
382
383
  unless node.instance_of?(klass)
383
384
  operator = klass.new(token, active_opts)
384
- sequence = operator.add_sequence(active_opts)
385
+ sequence = operator.add_sequence(active_opts, { ts: token.ts })
385
386
  sequence.expressions = node.expressions
386
387
  node.expressions = []
387
388
  nest(operator)
388
389
  end
389
- node.add_sequence(active_opts)
390
+ node.add_sequence(active_opts, { ts: token.te })
390
391
  end
391
392
 
392
393
  def posixclass(token)
393
394
  node << PosixClass.new(token, active_opts)
394
395
  end
395
396
 
396
- include Regexp::Expression::UnicodeProperty
397
- UPTokens = Regexp::Syntax::Token::UnicodeProperty
397
+ UP = Regexp::Expression::Property
398
+ UPTokens = Regexp::Syntax::Token::Property
398
399
 
399
400
  def property(token)
400
401
  case token.token
401
- when :alnum; node << Alnum.new(token, active_opts)
402
- when :alpha; node << Alpha.new(token, active_opts)
403
- when :ascii; node << Ascii.new(token, active_opts)
404
- when :blank; node << Blank.new(token, active_opts)
405
- when :cntrl; node << Cntrl.new(token, active_opts)
406
- when :digit; node << Digit.new(token, active_opts)
407
- when :graph; node << Graph.new(token, active_opts)
408
- when :lower; node << Lower.new(token, active_opts)
409
- when :print; node << Print.new(token, active_opts)
410
- when :punct; node << Punct.new(token, active_opts)
411
- when :space; node << Space.new(token, active_opts)
412
- when :upper; node << Upper.new(token, active_opts)
413
- when :word; node << Word.new(token, active_opts)
414
- when :xdigit; node << Xdigit.new(token, active_opts)
415
- when :xposixpunct; node << XPosixPunct.new(token, active_opts)
402
+ when :alnum; node << UP::Alnum.new(token, active_opts)
403
+ when :alpha; node << UP::Alpha.new(token, active_opts)
404
+ when :ascii; node << UP::Ascii.new(token, active_opts)
405
+ when :blank; node << UP::Blank.new(token, active_opts)
406
+ when :cntrl; node << UP::Cntrl.new(token, active_opts)
407
+ when :digit; node << UP::Digit.new(token, active_opts)
408
+ when :graph; node << UP::Graph.new(token, active_opts)
409
+ when :lower; node << UP::Lower.new(token, active_opts)
410
+ when :print; node << UP::Print.new(token, active_opts)
411
+ when :punct; node << UP::Punct.new(token, active_opts)
412
+ when :space; node << UP::Space.new(token, active_opts)
413
+ when :upper; node << UP::Upper.new(token, active_opts)
414
+ when :word; node << UP::Word.new(token, active_opts)
415
+ when :xdigit; node << UP::Xdigit.new(token, active_opts)
416
+ when :xposixpunct; node << UP::XPosixPunct.new(token, active_opts)
416
417
 
417
418
  # only in Oniguruma (old rubies)
418
- when :newline; node << Newline.new(token, active_opts)
419
-
420
- when :any; node << Any.new(token, active_opts)
421
- when :assigned; node << Assigned.new(token, active_opts)
422
-
423
- when :letter; node << Letter::Any.new(token, active_opts)
424
- when :cased_letter; node << Letter::Cased.new(token, active_opts)
425
- when :uppercase_letter; node << Letter::Uppercase.new(token, active_opts)
426
- when :lowercase_letter; node << Letter::Lowercase.new(token, active_opts)
427
- when :titlecase_letter; node << Letter::Titlecase.new(token, active_opts)
428
- when :modifier_letter; node << Letter::Modifier.new(token, active_opts)
429
- when :other_letter; node << Letter::Other.new(token, active_opts)
430
-
431
- when :mark; node << Mark::Any.new(token, active_opts)
432
- when :combining_mark; node << Mark::Combining.new(token, active_opts)
433
- when :nonspacing_mark; node << Mark::Nonspacing.new(token, active_opts)
434
- when :spacing_mark; node << Mark::Spacing.new(token, active_opts)
435
- when :enclosing_mark; node << Mark::Enclosing.new(token, active_opts)
436
-
437
- when :number; node << Number::Any.new(token, active_opts)
438
- when :decimal_number; node << Number::Decimal.new(token, active_opts)
439
- when :letter_number; node << Number::Letter.new(token, active_opts)
440
- when :other_number; node << Number::Other.new(token, active_opts)
441
-
442
- when :punctuation; node << Punctuation::Any.new(token, active_opts)
443
- when :connector_punctuation; node << Punctuation::Connector.new(token, active_opts)
444
- when :dash_punctuation; node << Punctuation::Dash.new(token, active_opts)
445
- when :open_punctuation; node << Punctuation::Open.new(token, active_opts)
446
- when :close_punctuation; node << Punctuation::Close.new(token, active_opts)
447
- when :initial_punctuation; node << Punctuation::Initial.new(token, active_opts)
448
- when :final_punctuation; node << Punctuation::Final.new(token, active_opts)
449
- when :other_punctuation; node << Punctuation::Other.new(token, active_opts)
450
-
451
- when :separator; node << Separator::Any.new(token, active_opts)
452
- when :space_separator; node << Separator::Space.new(token, active_opts)
453
- when :line_separator; node << Separator::Line.new(token, active_opts)
454
- when :paragraph_separator; node << Separator::Paragraph.new(token, active_opts)
455
-
456
- when :symbol; node << Symbol::Any.new(token, active_opts)
457
- when :math_symbol; node << Symbol::Math.new(token, active_opts)
458
- when :currency_symbol; node << Symbol::Currency.new(token, active_opts)
459
- when :modifier_symbol; node << Symbol::Modifier.new(token, active_opts)
460
- when :other_symbol; node << Symbol::Other.new(token, active_opts)
461
-
462
- when :other; node << Codepoint::Any.new(token, active_opts)
463
- when :control; node << Codepoint::Control.new(token, active_opts)
464
- when :format; node << Codepoint::Format.new(token, active_opts)
465
- when :surrogate; node << Codepoint::Surrogate.new(token, active_opts)
466
- when :private_use; node << Codepoint::PrivateUse.new(token, active_opts)
467
- when :unassigned; node << Codepoint::Unassigned.new(token, active_opts)
468
-
469
- when *UPTokens::Age; node << Age.new(token, active_opts)
470
- when *UPTokens::Derived; node << Derived.new(token, active_opts)
471
- when *UPTokens::Emoji; node << Emoji.new(token, active_opts)
472
- when *UPTokens::Script; node << Script.new(token, active_opts)
473
- when *UPTokens::UnicodeBlock; node << Block.new(token, active_opts)
419
+ when :newline; node << UP::Newline.new(token, active_opts)
420
+
421
+ when :any; node << UP::Any.new(token, active_opts)
422
+ when :assigned; node << UP::Assigned.new(token, active_opts)
423
+
424
+ when :letter; node << UP::Letter::Any.new(token, active_opts)
425
+ when :cased_letter; node << UP::Letter::Cased.new(token, active_opts)
426
+ when :uppercase_letter; node << UP::Letter::Uppercase.new(token, active_opts)
427
+ when :lowercase_letter; node << UP::Letter::Lowercase.new(token, active_opts)
428
+ when :titlecase_letter; node << UP::Letter::Titlecase.new(token, active_opts)
429
+ when :modifier_letter; node << UP::Letter::Modifier.new(token, active_opts)
430
+ when :other_letter; node << UP::Letter::Other.new(token, active_opts)
431
+
432
+ when :mark; node << UP::Mark::Any.new(token, active_opts)
433
+ when :combining_mark; node << UP::Mark::Combining.new(token, active_opts)
434
+ when :nonspacing_mark; node << UP::Mark::Nonspacing.new(token, active_opts)
435
+ when :spacing_mark; node << UP::Mark::Spacing.new(token, active_opts)
436
+ when :enclosing_mark; node << UP::Mark::Enclosing.new(token, active_opts)
437
+
438
+ when :number; node << UP::Number::Any.new(token, active_opts)
439
+ when :decimal_number; node << UP::Number::Decimal.new(token, active_opts)
440
+ when :letter_number; node << UP::Number::Letter.new(token, active_opts)
441
+ when :other_number; node << UP::Number::Other.new(token, active_opts)
442
+
443
+ when :punctuation; node << UP::Punctuation::Any.new(token, active_opts)
444
+ when :connector_punctuation; node << UP::Punctuation::Connector.new(token, active_opts)
445
+ when :dash_punctuation; node << UP::Punctuation::Dash.new(token, active_opts)
446
+ when :open_punctuation; node << UP::Punctuation::Open.new(token, active_opts)
447
+ when :close_punctuation; node << UP::Punctuation::Close.new(token, active_opts)
448
+ when :initial_punctuation; node << UP::Punctuation::Initial.new(token, active_opts)
449
+ when :final_punctuation; node << UP::Punctuation::Final.new(token, active_opts)
450
+ when :other_punctuation; node << UP::Punctuation::Other.new(token, active_opts)
451
+
452
+ when :separator; node << UP::Separator::Any.new(token, active_opts)
453
+ when :space_separator; node << UP::Separator::Space.new(token, active_opts)
454
+ when :line_separator; node << UP::Separator::Line.new(token, active_opts)
455
+ when :paragraph_separator; node << UP::Separator::Paragraph.new(token, active_opts)
456
+
457
+ when :symbol; node << UP::Symbol::Any.new(token, active_opts)
458
+ when :math_symbol; node << UP::Symbol::Math.new(token, active_opts)
459
+ when :currency_symbol; node << UP::Symbol::Currency.new(token, active_opts)
460
+ when :modifier_symbol; node << UP::Symbol::Modifier.new(token, active_opts)
461
+ when :other_symbol; node << UP::Symbol::Other.new(token, active_opts)
462
+
463
+ when :other; node << UP::Codepoint::Any.new(token, active_opts)
464
+ when :control; node << UP::Codepoint::Control.new(token, active_opts)
465
+ when :format; node << UP::Codepoint::Format.new(token, active_opts)
466
+ when :surrogate; node << UP::Codepoint::Surrogate.new(token, active_opts)
467
+ when :private_use; node << UP::Codepoint::PrivateUse.new(token, active_opts)
468
+ when :unassigned; node << UP::Codepoint::Unassigned.new(token, active_opts)
469
+
470
+ when *UPTokens::Age; node << UP::Age.new(token, active_opts)
471
+ when *UPTokens::Derived; node << UP::Derived.new(token, active_opts)
472
+ when *UPTokens::Emoji; node << UP::Emoji.new(token, active_opts)
473
+ when *UPTokens::Enumerated; node << UP::Enumerated.new(token, active_opts)
474
+ when *UPTokens::Script; node << UP::Script.new(token, active_opts)
475
+ when *UPTokens::UnicodeBlock; node << UP::Block.new(token, active_opts)
474
476
 
475
477
  else
476
478
  raise UnknownTokenError.new('UnicodeProperty', token)
@@ -478,8 +480,7 @@ class Regexp::Parser
478
480
  end
479
481
 
480
482
  def quantifier(token)
481
- target_node = node.expressions.reverse.find { |exp| !exp.is_a?(FreeSpace) }
482
- target_node or raise ParserError, "No valid target found for '#{token.text}'"
483
+ target_node = node.extract_quantifier_target(token.text)
483
484
 
484
485
  # in case of chained quantifiers, wrap target in an implicit passive group
485
486
  # description of the problem: https://github.com/ammar/regexp_parser/issues/3
@@ -527,6 +528,8 @@ class Regexp::Parser
527
528
  end
528
529
 
529
530
  def open_set(token)
531
+ # TODO: this and Quantifier are the only cases where Expression#token
532
+ # does not match the scanner/lexer output. Fix in v3.0.0.
530
533
  token.token = :character
531
534
  nest(CharacterSet.new(token, active_opts))
532
535
  end
@@ -575,22 +578,25 @@ class Regexp::Parser
575
578
  options_stack.last
576
579
  end
577
580
 
578
- # Assigns referenced expressions to refering expressions, e.g. if there is
581
+ # Assigns referenced expressions to referring expressions, e.g. if there is
579
582
  # an instance of Backreference::Number, its #referenced_expression is set to
580
583
  # the instance of Group::Capture that it refers to via its number.
581
584
  def assign_referenced_expressions
582
- # find all referencable and refering expressions
583
- targets = { 0 => root }
585
+ # find all referenceable and referring expressions
586
+ targets = { 0 => [root] }
584
587
  referrers = []
585
588
  root.each_expression do |exp|
586
- exp.is_a?(Group::Capture) && targets[exp.identifier] = exp
587
- referrers << exp if exp.referential?
589
+ if exp.referential?
590
+ referrers << exp
591
+ elsif exp.is_a?(Group::Capture)
592
+ (targets[exp.identifier] ||= []) << exp
593
+ end
588
594
  end
589
- # assign reference expression to refering expressions
595
+ # assign referenced expressions to referring expressions
590
596
  # (in a second iteration because there might be forward references)
591
597
  referrers.each do |exp|
592
- exp.referenced_expression = targets[exp.reference] ||
593
- raise(ParserError, "Invalid reference: #{exp.reference}")
598
+ exp.referenced_expressions = targets[exp.reference] ||
599
+ raise(ParserError, "Invalid reference #{exp.reference} at pos #{exp.ts}")
594
600
  end
595
601
  end
596
602
  end # module Regexp::Parser
@@ -0,0 +1,10 @@
1
+ # frozen_string_literal: true
2
+
3
+ class Regexp::Scanner
4
+ # Unexpected end of pattern
5
+ class PrematureEndError < ScannerError
6
+ def initialize(where = '')
7
+ super "Premature end of pattern at #{where}"
8
+ end
9
+ end
10
+ end
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../../../regexp_parser/error'
4
+
5
+ class Regexp::Scanner
6
+ # General scanner error (catch all)
7
+ class ScannerError < Regexp::Parser::Error; end
8
+ end
@@ -0,0 +1,65 @@
1
+ # frozen_string_literal: true
2
+
3
+ class Regexp::Scanner
4
+ # Base for all scanner validation errors
5
+ class ValidationError < ScannerError
6
+ # Centralizes and unifies the handling of validation related errors.
7
+ def self.for(type, problem, reason = nil)
8
+ types.fetch(type).new(problem, reason)
9
+ end
10
+
11
+ def self.types
12
+ @types ||= {
13
+ backref: InvalidBackrefError,
14
+ group: InvalidGroupError,
15
+ group_option: InvalidGroupOption,
16
+ posix_class: UnknownPosixClassError,
17
+ property: UnknownUnicodePropertyError,
18
+ sequence: InvalidSequenceError,
19
+ }
20
+ end
21
+ end
22
+
23
+ # Invalid sequence format. Used for escape sequences, mainly.
24
+ class InvalidSequenceError < ValidationError
25
+ def initialize(what = 'sequence', where = '')
26
+ super "Invalid #{what} at #{where}"
27
+ end
28
+ end
29
+
30
+ # Invalid group. Used for named groups.
31
+ class InvalidGroupError < ValidationError
32
+ def initialize(what, reason)
33
+ super "Invalid #{what}, #{reason}."
34
+ end
35
+ end
36
+
37
+ # Invalid groupOption. Used for inline options.
38
+ # TODO: should become InvalidGroupOptionError in v3.0.0 for consistency
39
+ class InvalidGroupOption < ValidationError
40
+ def initialize(option, text)
41
+ super "Invalid group option #{option} in #{text}"
42
+ end
43
+ end
44
+
45
+ # Invalid back reference. Used for name a number refs/calls.
46
+ class InvalidBackrefError < ValidationError
47
+ def initialize(what, reason)
48
+ super "Invalid back reference #{what}, #{reason}"
49
+ end
50
+ end
51
+
52
+ # The property name was not recognized by the scanner.
53
+ class UnknownUnicodePropertyError < ValidationError
54
+ def initialize(name, _)
55
+ super "Unknown unicode character property name #{name}"
56
+ end
57
+ end
58
+
59
+ # The POSIX class name was not recognized by the scanner.
60
+ class UnknownPosixClassError < ValidationError
61
+ def initialize(text, _)
62
+ super "Unknown POSIX class #{text}"
63
+ end
64
+ end
65
+ end