flexr 1.0.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +241 -72
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/RELEASING.md +3 -2
  25. data/docs/explanation/backends.md +30 -10
  26. data/docs/explanation/matching-semantics.md +4 -2
  27. data/docs/explanation/security-model.md +10 -3
  28. data/docs/explanation/unicode-and-encoding.md +3 -1
  29. data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
  30. data/docs/how-to/generate-a-lexer.md +3 -3
  31. data/docs/how-to/handle-errors.md +4 -1
  32. data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
  33. data/docs/how-to/track-token-locations.md +2 -1
  34. data/docs/how-to/tune-performance.md +15 -3
  35. data/docs/migration-v2.md +74 -0
  36. data/docs/perf-log.md +212 -0
  37. data/docs/reference/README.md +2 -0
  38. data/docs/reference/actions.md +5 -2
  39. data/docs/reference/cli.md +71 -5
  40. data/docs/reference/compatibility.md +10 -4
  41. data/docs/reference/diagnostics.md +346 -0
  42. data/docs/reference/dsl.md +30 -9
  43. data/docs/reference/errors.md +34 -3
  44. data/docs/reference/generated-artifacts.md +71 -6
  45. data/docs/reference/public-api.md +37 -7
  46. data/docs/reference/regexp.md +27 -13
  47. data/docs/reference/ruby-lsp.md +34 -0
  48. data/docs/reference/runtime.md +21 -4
  49. data/docs/reference/testing.md +57 -0
  50. data/docs/reference/tokens-and-locations.md +14 -4
  51. data/examples/graphql/README.md +14 -0
  52. data/examples/graphql/lexer.flexr.rb +29 -0
  53. data/examples/json/lexer.flexr.rb +6 -6
  54. data/examples/sql/README.md +16 -0
  55. data/examples/sql/lexer.flexr.rb +23 -0
  56. data/examples/toml/README.md +14 -0
  57. data/examples/toml/lexer.flexr.rb +23 -0
  58. data/gemfiles/ruby_lsp.gemfile +4 -0
  59. data/gemfiles/types.gemfile +7 -0
  60. data/lib/flexr/action_resolver.rb +15 -0
  61. data/lib/flexr/artifact_writer.rb +61 -0
  62. data/lib/flexr/automaton/accel.rb +78 -5
  63. data/lib/flexr/automaton/analysis.rb +202 -7
  64. data/lib/flexr/automaton/backend_cost_model.rb +40 -0
  65. data/lib/flexr/automaton/compiler.rb +229 -96
  66. data/lib/flexr/automaton/dfa.rb +106 -19
  67. data/lib/flexr/automaton/minimizer.rb +79 -31
  68. data/lib/flexr/automaton/nfa.rb +45 -13
  69. data/lib/flexr/automaton/types.rb +14 -0
  70. data/lib/flexr/cli.rb +128 -22
  71. data/lib/flexr/codegen/direct.rb +5 -41
  72. data/lib/flexr/codegen/firstmatch.rb +1 -7
  73. data/lib/flexr/codegen/table.rb +181 -69
  74. data/lib/flexr/codegen.rb +8 -0
  75. data/lib/flexr/configuration.rb +45 -0
  76. data/lib/flexr/diagnostics.rb +39 -3
  77. data/lib/flexr/dsl.rb +116 -47
  78. data/lib/flexr/errors.rb +7 -6
  79. data/lib/flexr/generated.rb +120 -46
  80. data/lib/flexr/generator.rb +102 -109
  81. data/lib/flexr/grammar.rb +41 -0
  82. data/lib/flexr/importer.rb +4 -40
  83. data/lib/flexr/ir.rb +2 -2
  84. data/lib/flexr/migration.rb +31 -0
  85. data/lib/flexr/minitest.rb +24 -0
  86. data/lib/flexr/options.rb +9 -5
  87. data/lib/flexr/rake_task.rb +6 -1
  88. data/lib/flexr/regexp/ast.rb +5 -2
  89. data/lib/flexr/regexp/matcher.rb +26 -0
  90. data/lib/flexr/regexp/normalizer.rb +63 -21
  91. data/lib/flexr/regexp/parser.rb +241 -60
  92. data/lib/flexr/regexp/tokenizer.rb +172 -47
  93. data/lib/flexr/rspec.rb +51 -0
  94. data/lib/flexr/runtime/buffer.rb +115 -9
  95. data/lib/flexr/runtime/core.rb +248 -51
  96. data/lib/flexr/runtime/errors.rb +66 -4
  97. data/lib/flexr/runtime/interpreter.rb +213 -97
  98. data/lib/flexr/runtime/location.rb +35 -12
  99. data/lib/flexr/runtime/record.rb +14 -0
  100. data/lib/flexr/runtime/token.rb +7 -1
  101. data/lib/flexr/runtime.rb +75 -0
  102. data/lib/flexr/source/passthrough.rb +70 -14
  103. data/lib/flexr/source/prism_reader.rb +217 -51
  104. data/lib/flexr/source/static_eval.rb +40 -4
  105. data/lib/flexr/source.rb +6 -0
  106. data/lib/flexr/testing.rb +47 -0
  107. data/lib/flexr/trace.rb +197 -0
  108. data/lib/flexr/unicode/data/properties.rb +1 -1
  109. data/lib/flexr/unicode/data.rb +11 -0
  110. data/lib/flexr/unicode/property.rb +4 -8
  111. data/lib/flexr/unicode/reference_regexp.rb +11 -9
  112. data/lib/flexr/unicode/version.rb +7 -0
  113. data/lib/flexr/version.rb +1 -1
  114. data/lib/flexr.rb +10 -76
  115. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  116. data/sig/flexr.rbs +191 -0
  117. data/site/README.md +54 -3
  118. data/site/astro.config.mjs +3 -1
  119. data/site/package.json +12 -5
  120. data/site/playwright.config.mjs +16 -0
  121. data/site/pnpm-lock.yaml +218 -88
  122. data/site/ruby/playground.rb +57 -0
  123. data/site/scripts/prepare-playground.mjs +35 -0
  124. data/site/scripts/verify-site.mjs +5 -3
  125. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  126. data/site/src/content/docs/concepts/security-model.md +9 -1
  127. data/site/src/content/docs/learn/generation.md +1 -1
  128. data/site/src/content/docs/reference/dsl.md +2 -2
  129. data/site/src/content/docs/reference/public-api.md +4 -2
  130. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  131. data/site/src/layouts/SiteLayout.astro +11 -3
  132. data/site/src/pages/index.astro +69 -155
  133. data/site/src/pages/playground.astro +40 -38
  134. data/site/src/playground/client.js +211 -0
  135. data/site/src/playground/graph-worker.mjs +11 -0
  136. data/site/src/playground/presets.mjs +33 -0
  137. data/site/src/playground/runtime.mjs +34 -0
  138. data/site/src/playground/share.mjs +30 -0
  139. data/site/src/playground/worker.mjs +29 -0
  140. data/site/src/styles/custom.css +12 -287
  141. data/site/src/styles/landing.css +239 -0
  142. data/site/tests/browser/playground.spec.mjs +99 -0
  143. data/site/tests/playground-runtime.test.mjs +85 -0
  144. data/test/ruby_lsp_addon_spec.rb +135 -0
  145. data/tools/coverage.rb +6 -1
  146. data/tools/docs_verify.rb +3 -2
  147. data/tools/golden.rb +54 -0
  148. data/tools/verification_failure.rb +119 -0
  149. metadata +66 -12
  150. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  151. data/benchmark/golden/json_lexer.sha256 +0 -1
  152. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  153. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  154. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  155. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  156. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  157. data/site/public/playground.js +0 -189
@@ -18,7 +18,7 @@ module Flexr
18
18
 
19
19
  def normalize_node(node)
20
20
  case node
21
- when AST::Empty, AST::ByteRange, AST::CodepointRange, AST::Anchor
21
+ when AST::Empty, AST::Fail, AST::ByteRange, AST::CodepointRange, AST::Anchor
22
22
  normalize_leaf(node)
23
23
  when AST::CharClass
24
24
  char_class(node)
@@ -27,7 +27,10 @@ module Flexr
27
27
  when AST::Alt
28
28
  alternatives(node.children.map { |child| normalize_node(child) })
29
29
  when AST::Star
30
- AST::Star.new(child: normalize_node(node.child), loc: node.loc)
30
+ child = normalize_node(node.child)
31
+ child.is_a?(AST::Fail) ? AST::Empty.new(loc: node.loc) : AST::Star.new(child: child, loc: node.loc)
32
+ when AST::Repeat
33
+ repeat(node)
31
34
  else
32
35
  raise CompileError, "unknown regexp AST node: #{node.class}"
33
36
  end
@@ -36,7 +39,7 @@ module Flexr
36
39
  def normalize_leaf(node)
37
40
  case node
38
41
  when AST::CodepointRange
39
- byte_sequences_for_ranges(casefold_ranges(node.lo, node.hi))
42
+ byte_sequences_for_ranges([[node.lo, node.hi]], loc: node.loc)
40
43
  else
41
44
  node
42
45
  end
@@ -45,44 +48,76 @@ module Flexr
45
48
  def char_class(node)
46
49
  ranges = node.ranges.flat_map do |range|
47
50
  if range.first == AST::Property
48
- Unicode::Property.ranges(range.last, negate: range[1])
51
+ property_ranges = Unicode::Property.ranges(range[2])
52
+ property_ranges = property_ranges.flat_map { |lo, hi| Unicode::CaseFold.ranges(lo, hi) } if range[3]
53
+ range[1] ? complement(property_ranges) : property_ranges
49
54
  else
50
- casefold_ranges(*range)
55
+ [range]
51
56
  end
52
57
  end
53
58
  ranges = complement(ranges) if node.negated
54
- byte_sequences_for_ranges(ranges)
59
+ byte_sequences_for_ranges(ranges, loc: node.loc)
55
60
  end
56
61
 
57
- def byte_sequences_for_ranges(ranges)
62
+ def byte_sequences_for_ranges(ranges, loc:)
58
63
  if @byte_mode
59
64
  byte_ranges = ranges.filter_map do |lo, hi|
60
65
  next if lo > 255
61
66
 
62
67
  [lo, [hi, 255].min]
63
68
  end
64
- return AST::Empty.new(loc: nil) if byte_ranges.empty?
69
+ return AST::Fail.new(loc: loc) if byte_ranges.empty?
65
70
 
66
- return alternatives(byte_ranges.map { |lo, hi| AST::ByteRange.new(lo: lo, hi: hi, loc: nil) })
71
+ return alternatives(byte_ranges.map { |lo, hi| AST::ByteRange.new(lo: lo, hi: hi, loc: loc) })
67
72
  end
68
73
 
69
74
  sequences = ranges.flat_map { |lo, hi| Unicode::Utf8Splitter.split(lo, hi) }
75
+ @trie_cache = {}
70
76
  trie(sequences, 0)
71
77
  end
72
78
 
73
79
  def trie(sequences, index)
74
- return AST::Empty.new(loc: nil) if sequences.empty?
75
- return AST::Empty.new(loc: nil) if sequences.first.length == index
76
-
77
- groups = sequences.group_by { |sequence| sequence[index] }
78
- alternatives(groups.map do |(lo, hi), group|
79
- child = trie(group, index + 1)
80
- AST::Seq.new(children: [AST::ByteRange.new(lo: lo, hi: hi, loc: nil), child], loc: nil)
80
+ return AST::Fail.new(loc: nil) if sequences.empty?
81
+
82
+ finished, continuing = sequences.partition { |sequence| sequence.length == index }
83
+ nodes = []
84
+ nodes << AST::Empty.new(loc: nil) unless finished.empty?
85
+ return nodes.first if continuing.empty?
86
+
87
+ key = [index, continuing]
88
+ return alternatives(nodes + [@trie_cache[key]]) if @trie_cache.key?(key)
89
+
90
+ endpoints = continuing.flat_map do |sequence|
91
+ lo, hi = sequence.fetch(index)
92
+ [lo, hi + 1]
93
+ end.uniq.sort
94
+ partitions = endpoints.each_cons(2).filter_map do |lo, after_hi|
95
+ covered = continuing.select do |sequence|
96
+ range = sequence.fetch(index)
97
+ range.first <= lo && range.last >= after_hi - 1
98
+ end
99
+ [lo, after_hi - 1, covered] unless covered.empty?
100
+ end
101
+ branches = partitions.map { |lo, hi, covered| [lo, hi, trie(covered, index + 1)] }
102
+ merged = branches.each_with_object([]) do |(lo, hi, child), result|
103
+ if !result.empty? && result.last[1] + 1 == lo && result.last[2] == child
104
+ result.last[1] = hi
105
+ else
106
+ result << [lo, hi, child]
107
+ end
108
+ end
109
+ tree = alternatives(merged.map do |lo, hi, child|
110
+ sequence([AST::ByteRange.new(lo: lo, hi: hi, loc: nil), child])
81
111
  end)
112
+ @trie_cache[key] = tree
113
+ alternatives(nodes + [tree])
82
114
  end
83
115
 
84
116
  def sequence(children)
85
117
  flattened = children.flat_map { |child| child.is_a?(AST::Seq) ? child.children : [child] }
118
+ return AST::Fail.new(loc: nil) if flattened.any?(AST::Fail)
119
+
120
+ flattened.reject! { |child| child.is_a?(AST::Empty) }
86
121
  return AST::Empty.new(loc: nil) if flattened.empty?
87
122
  return flattened.first if flattened.length == 1
88
123
 
@@ -91,12 +126,23 @@ module Flexr
91
126
 
92
127
  def alternatives(children)
93
128
  flattened = children.flat_map { |child| child.is_a?(AST::Alt) ? child.children : [child] }
94
- return AST::Empty.new(loc: nil) if flattened.empty?
129
+ flattened.reject! { |child| child.is_a?(AST::Fail) }
130
+ flattened.uniq!
131
+ return AST::Fail.new(loc: nil) if flattened.empty?
95
132
  return flattened.first if flattened.length == 1
96
133
 
97
134
  AST::Alt.new(children: flattened, loc: nil)
98
135
  end
99
136
 
137
+ def repeat(node)
138
+ child = normalize_node(node.child)
139
+ return AST::Empty.new(loc: node.loc) if node.maximum&.zero? || child.is_a?(AST::Empty)
140
+ return node.minimum.zero? ? AST::Empty.new(loc: node.loc) : AST::Fail.new(loc: node.loc) if
141
+ child.is_a?(AST::Fail)
142
+
143
+ AST::Repeat.new(child: child, minimum: node.minimum, maximum: node.maximum, loc: node.loc)
144
+ end
145
+
100
146
  def complement(ranges)
101
147
  result = []
102
148
  cursor = 0
@@ -108,10 +154,6 @@ module Flexr
108
154
  result
109
155
  end
110
156
 
111
- def casefold_ranges(lo, hi)
112
- return [[lo, hi]] if @options.nobits?(::Regexp::IGNORECASE)
113
- Unicode::CaseFold.ranges(lo, hi)
114
- end
115
157
  end
116
158
  end
117
159
  end
@@ -12,6 +12,7 @@ module Flexr
12
12
  "word" => "Word", "space" => "Space"
13
13
  }.freeze
14
14
  POSIX_CLASSES = %w[alnum alpha blank cntrl digit graph lower print punct space upper xdigit].freeze
15
+ LITERAL_ESCAPES = (%w[. [ ] { } ( ) * + ? | ^ $ \\ / - #] + [" "]).freeze
15
16
 
16
17
  attr_reader :source
17
18
 
@@ -22,6 +23,12 @@ module Flexr
22
23
  @unicode = unicode
23
24
  @index = 0
24
25
  @class_depth = 0
26
+ offset = 0
27
+ @byte_offsets = [0]
28
+ source.each_char do |character|
29
+ offset += character.bytesize
30
+ @byte_offsets << offset
31
+ end
25
32
  end
26
33
 
27
34
  def parse
@@ -34,43 +41,46 @@ module Flexr
34
41
  private
35
42
 
36
43
  def parse_expression
44
+ starting = @index
37
45
  branches = [parse_sequence]
38
46
  branches << parse_sequence while consume?("|")
39
47
  return branches.first if branches.length == 1
40
48
 
41
- AST::Alt.new(children: branches, loc: nil)
49
+ AST::Alt.new(children: branches, loc: span(starting))
42
50
  end
43
51
 
44
52
  def parse_sequence
53
+ starting = @index
45
54
  children = []
46
55
  children << parse_quantified until eof? || [")", "|"].include?(current)
47
- return AST::Empty.new(loc: nil) if children.empty?
56
+ return AST::Empty.new(loc: span(starting)) if children.empty?
48
57
  return children.first if children.length == 1
49
58
 
50
- AST::Seq.new(children: children, loc: nil)
59
+ AST::Seq.new(children: children, loc: span(starting))
51
60
  end
52
61
 
53
62
  def parse_quantified
63
+ starting = @index
54
64
  atom = parse_atom
55
65
  return atom unless ["*", "+", "?", "{"].include?(current)
56
66
 
57
67
  if consume?("*")
58
68
  reject_postfix_quantifier
59
- return AST::Star.new(child: atom, loc: nil)
69
+ return AST::Repeat.new(child: atom, minimum: 0, maximum: nil, loc: span(starting))
60
70
  end
61
71
  if consume?("+")
62
72
  reject_postfix_quantifier
63
- return AST::Seq.new(children: [atom, AST::Star.new(child: atom, loc: nil)], loc: nil)
73
+ return AST::Repeat.new(child: atom, minimum: 1, maximum: nil, loc: span(starting))
64
74
  end
65
75
  if consume?("?")
66
76
  reject_postfix_quantifier
67
- return AST::Alt.new(children: [atom, AST::Empty.new(loc: nil)], loc: nil)
77
+ return AST::Repeat.new(child: atom, minimum: 0, maximum: 1, loc: span(starting))
68
78
  end
69
79
 
70
- parse_repetition(atom)
80
+ parse_repetition(atom, starting)
71
81
  end
72
82
 
73
- def parse_repetition(atom)
83
+ def parse_repetition(atom, starting)
74
84
  consume?("{")
75
85
  min = read_number
76
86
  max = if consume?(",")
@@ -80,8 +90,7 @@ module Flexr
80
90
  end
81
91
  expect("}")
82
92
  raise_syntax("invalid repetition") if min.nil? || (!max.nil? && max < min)
83
- raise_syntax("open repetition is not supported") if max.nil?
84
- if max > 1000
93
+ if (max || min) > 1000
85
94
  raise_diagnostic(
86
95
  diagnostic("FLEXR-E007", "repetition limit exceeds 1000",
87
96
  help: "split the rule or use a smaller bounded repetition")
@@ -89,19 +98,13 @@ module Flexr
89
98
  end
90
99
  reject_postfix_quantifier
91
100
 
92
- required = Array.new(min) { atom }
93
- optional = Array.new(max - min) do
94
- AST::Alt.new(children: [atom, AST::Empty.new(loc: nil)], loc: nil)
95
- end
96
- children = required + optional
97
- return AST::Empty.new(loc: nil) if children.empty?
98
- return children.first if children.length == 1
99
-
100
- AST::Seq.new(children: children, loc: nil)
101
+ AST::Repeat.new(child: atom, minimum: min, maximum: max, loc: span(starting))
101
102
  end
102
103
 
103
104
  def parse_atom
105
+ starting = @index
104
106
  @escaped_value = false
107
+ @escaped_byte = false
105
108
  @last_ranges = nil
106
109
  return parse_group if consume?("(")
107
110
  return parse_class if consume?("[")
@@ -109,25 +112,29 @@ module Flexr
109
112
 
110
113
  if consume?(".")
111
114
  upper = @options.nobits?(::Regexp::MULTILINE) ? 0x0a - 1 : 0x10ffff
112
- return AST::CharClass.new(ranges: [[0, upper], [0x0b, 0x10ffff]], negated: false, loc: nil)
115
+ return AST::CharClass.new(
116
+ ranges: [[0, upper], [0x0b, 0x10ffff]], negated: false, loc: span(starting)
117
+ )
113
118
  end
114
119
 
115
120
  if consume?("\\")
116
121
  parse_escape
117
- return AST::CharClass.new(ranges: @last_ranges, negated: false, loc: nil) if @last_ranges
118
- return codepoint_node(read_codepoint) if @escaped_value
122
+ return AST::CharClass.new(ranges: @last_ranges, negated: false, loc: span(starting)) if @last_ranges
123
+ return AST::ByteRange.new(lo: @last_byte, hi: @last_byte, loc: span(starting)) if @escaped_byte
124
+ return codepoint_node(read_codepoint, span(starting)) if @escaped_value
119
125
  end
120
126
 
121
127
  char = advance
122
128
  raise_syntax("unexpected end of expression") unless char
123
- codepoint_node(char.ord)
129
+ codepoint_node(char.ord, span(starting))
124
130
  end
125
131
 
126
132
  def parse_group
133
+ starting = @index - 1
127
134
  saved_options = @options
128
135
  if consume?("?")
129
136
  prefix = parse_group_prefix
130
- return AST::Empty.new(loc: nil) if prefix == :global
137
+ return AST::Empty.new(loc: span(starting)) if prefix == :global
131
138
  else
132
139
  warn_capture
133
140
  end
@@ -164,8 +171,30 @@ module Flexr
164
171
  end
165
172
  raise_syntax("invalid inline option group")
166
173
  end
167
- raise unsupported("look-around", "use followed_by: or a state instead") if peek_prefix?("=") || peek_prefix?("!") || peek_prefix?("<=") || peek_prefix?("<!")
168
- raise unsupported("atomic groups", "rewrite the expression as a DFA-compatible expression") if consume?(">")
174
+ if peek_prefix?("=")
175
+ raise unsupported("look-around",
176
+ "for a trailing positive lookahead, rewrite rule(/a(?=b)/) as rule(/a/, followed_by: /b/); keep the action")
177
+ end
178
+ if peek_prefix?("!") || peek_prefix?("<=") || peek_prefix?("<!")
179
+ raise unsupported("look-around",
180
+ "use separate rules or states; followed_by: cannot express negative lookahead or lookbehind")
181
+ end
182
+ if consume?(">")
183
+ raise unsupported("atomic groups",
184
+ 'for unescaped single-line quoted text, a DFA-compatible rewrite is rule(/"[^"\n]*"/); keep the action; removing atomic groups can change matches')
185
+ end
186
+ if peek_prefix?("<") || peek_prefix?("'")
187
+ raise unsupported("named captures",
188
+ "when only grouping is needed, rewrite rule(/(?<word>ab)/) as rule(/(?:ab)/); keep the action; capture-dependent patterns need a separate rewrite")
189
+ end
190
+ if peek_prefix?("(")
191
+ raise unsupported("conditional groups",
192
+ "for this optional-prefix conditional, rewrite rule(/(a)?(?(1)b|c)/) as rule(/(?:ab|c)/); keep the action; other conditions need separate rules or state checks")
193
+ end
194
+ if peek_prefix?("~")
195
+ raise unsupported("absence groups",
196
+ "when excluding a single literal, rewrite rule(/x(?~a)/) as rule(/x[^a]*/); keep the action; larger absence expressions need a separate rewrite")
197
+ end
169
198
  raise unsupported("unsupported group syntax", "use a non-capturing group (?:...)")
170
199
  end
171
200
 
@@ -174,10 +203,11 @@ module Flexr
174
203
  @options = enabled ? (@options | bit) : (@options & ~bit)
175
204
  end
176
205
 
177
- def codepoint_node(codepoint)
178
- return AST::CodepointRange.new(lo: codepoint, hi: codepoint, loc: nil) if @options.nobits?(::Regexp::IGNORECASE)
206
+ def codepoint_node(codepoint, location)
207
+ return AST::CodepointRange.new(lo: codepoint, hi: codepoint, loc: location) if
208
+ @options.nobits?(::Regexp::IGNORECASE)
179
209
 
180
- AST::CharClass.new(ranges: fold_ranges([[codepoint, codepoint]]), negated: false, loc: nil)
210
+ AST::CharClass.new(ranges: fold_ranges([[codepoint, codepoint]]), negated: false, loc: location)
181
211
  end
182
212
 
183
213
  def fold_ranges(ranges)
@@ -190,29 +220,51 @@ module Flexr
190
220
  end
191
221
 
192
222
  def parse_class
223
+ starting = @index - 1
224
+ @class_depth += 1
193
225
  negated = consume?("^")
226
+ ranges = parse_class_union
227
+ while peek_prefix?("&&")
228
+ @index += 2
229
+ ranges = materialize_class_ranges(ranges, negated: negated)
230
+ negated = false
231
+ right = if consume?("[")
232
+ nested = parse_class
233
+ materialize_class_ranges(nested.ranges, negated: nested.negated)
234
+ else
235
+ materialize_class_ranges(parse_class_union, negated: false)
236
+ end
237
+ ranges = intersect_ranges(ranges, right)
238
+ end
239
+ expect("]")
240
+ ranges = merge_ranges(ranges)
241
+ ranges = fold_ranges(ranges) if @options.anybits?(::Regexp::IGNORECASE)
242
+ AST::CharClass.new(ranges: ranges, negated: negated, loc: span(starting))
243
+ ensure
244
+ @class_depth -= 1
245
+ end
246
+
247
+ def parse_class_union
194
248
  ranges = []
195
- @class_depth += 1
196
- until eof? || current == "]"
249
+ until eof? || current == "]" || peek_prefix?("&&")
197
250
  if current == "[" && @source[@index, 2] == "[:"
198
251
  ranges.concat(parse_posix_class)
199
252
  next
200
253
  end
201
254
  first = parse_class_atom
202
- if consume?("-") && current != "]"
203
- last = parse_class_atom
204
- ranges.concat(expand_class_range(first, last))
255
+ if consume?("-")
256
+ if current == "]"
257
+ ranges.concat(first)
258
+ ranges << [45, 45]
259
+ else
260
+ last = parse_class_atom
261
+ ranges.concat(expand_class_range(first, last))
262
+ end
205
263
  else
206
264
  ranges.concat(first)
207
265
  end
208
266
  end
209
- expect("]")
210
- @class_depth -= 1
211
- ranges = merge_ranges(ranges)
212
- ranges = fold_ranges(ranges) if @options.anybits?(::Regexp::IGNORECASE)
213
- AST::CharClass.new(ranges: ranges, negated: negated, loc: nil)
214
- ensure
215
- @class_depth -= 1 if @class_depth.positive? && current != "]"
267
+ ranges
216
268
  end
217
269
 
218
270
  def parse_posix_class
@@ -220,7 +272,7 @@ module Flexr
220
272
  raw_name = read_until(":]")
221
273
  inner_negated = raw_name.start_with?("^")
222
274
  name = raw_name.delete_prefix("^")
223
- return [[AST::Property, inner_negated, "POSIX_#{name}"]] if
275
+ return [[AST::Property, inner_negated, "POSIX_#{name}", ignorecase?]] if
224
276
  @encoding != Encoding::BINARY && POSIX_CLASSES.include?(name)
225
277
 
226
278
  ranges = case name
@@ -245,6 +297,7 @@ module Flexr
245
297
  def parse_class_atom
246
298
  if consume?("\\")
247
299
  parse_escape
300
+ return [[@last_byte, @last_byte]] if @escaped_byte
248
301
  return [[@last_codepoint, @last_codepoint]] if @escaped_value
249
302
  return @last_ranges
250
303
  end
@@ -261,16 +314,59 @@ module Flexr
261
314
  [[lo, hi]]
262
315
  end
263
316
 
317
+ def materialize_class_ranges(ranges, negated:)
318
+ concrete = ranges.flat_map do |range|
319
+ if range.first == AST::Property
320
+ property = Unicode::Property.ranges(range[2])
321
+ property = fold_ranges(property) if range[3]
322
+ range[1] ? complement_ranges(property) : property
323
+ else
324
+ [range]
325
+ end
326
+ end
327
+ concrete = merge_ranges(concrete)
328
+ negated ? complement_ranges(concrete) : concrete
329
+ end
330
+
331
+ def intersect_ranges(left, right)
332
+ result = []
333
+ left_index = 0
334
+ right_index = 0
335
+ while left_index < left.length && right_index < right.length
336
+ left_range = left[left_index]
337
+ right_range = right[right_index]
338
+ lo = [left_range.first, right_range.first].max
339
+ hi = [left_range.last, right_range.last].min
340
+ result << [lo, hi] if lo <= hi
341
+ if left_range.last < right_range.last
342
+ left_index += 1
343
+ else
344
+ right_index += 1
345
+ end
346
+ end
347
+ result
348
+ end
349
+
264
350
  def parse_escape
265
351
  @escaped_value = false
352
+ @escaped_byte = false
266
353
  @last_ranges = nil
267
354
  char = advance_raw
268
355
  raise_syntax("trailing backslash") unless char
269
356
  if ESCAPES.key?(char)
357
+ if char == "0" && current&.match?(/[0-9]/)
358
+ raise unsupported("octal escapes",
359
+ 'use an explicit hexadecimal escape: rewrite rule(/\011/) as rule(/\x09/) for a tab; keep the action')
360
+ end
361
+
270
362
  @escaped_value = true
271
363
  @last_codepoint = ESCAPES.fetch(char)
272
364
  return
273
365
  end
366
+ if @class_depth.positive? && %w[A B G z Z K g X].include?(char)
367
+ raise unsupported("\\#{char}",
368
+ "inside a character class this is a literal character: rewrite rule(/[\\#{char}]/) as rule(/[#{char}]/); keep the action")
369
+ end
274
370
  case char
275
371
  when "d", "D", "w", "W", "s", "S", "h", "H"
276
372
  ranges = shorthand_ranges(char)
@@ -279,16 +375,15 @@ module Flexr
279
375
  when "p", "P"
280
376
  expect("{")
281
377
  name = read_until("}")
282
- ranges = [[AST::Property, char == "P", name]]
378
+ ranges = [[AST::Property, char == "P", name, ignorecase?]]
283
379
  @last_ranges = ranges
284
380
  nil
285
381
  when "x"
286
- digits = if consume?("{")
287
- read_until("}")
382
+ if consume?("{")
383
+ assign_codepoint(read_until("}"))
288
384
  else
289
- read_exact(2)
385
+ assign_hex_byte(read_exact(2))
290
386
  end
291
- assign_codepoint(digits)
292
387
  nil
293
388
  when "u"
294
389
  digits = if consume?("{")
@@ -299,28 +394,77 @@ module Flexr
299
394
  end
300
395
  assign_codepoint(digits)
301
396
  nil
302
- when "G", "K", "b", "B", "A", "z", "Z", "1", "2", "3", "4", "5", "6", "7", "8", "9"
303
- raise unsupported("\\#{char}", "use a state or followed_by: instead")
304
- when "k"
305
- read_until(">") if consume?("<")
306
- raise unsupported("backreferences", "split the rule into DFA-compatible states")
307
- else
397
+ when "b"
398
+ unless @class_depth.positive?
399
+ raise unsupported("\\b",
400
+ 'when surrounding literals already establish the boundary, rewrite rule(/foo\b:/) as rule(/foo:/); keep the action; token edges need separate rules or state checks')
401
+ end
402
+
403
+ @escaped_value = true
404
+ @last_codepoint = 0x08
405
+ when "B"
406
+ raise unsupported("\\B",
407
+ 'when surrounding literals already establish the non-boundary, rewrite rule(/foo\Bbar/) as rule(/foobar/); keep the action; token edges need separate rules or state checks')
408
+ when "G"
409
+ raise unsupported("\\G",
410
+ 'at the start of a rule, rewrite rule(/\Ga/) as rule(/a/); keep the action; rules already match at the current lexer position; interior \G needs a separate rewrite')
411
+ when "A"
412
+ raise unsupported("\\A",
413
+ 'for input without line feeds, rewrite rule(/\Aa/) as rule(/^a/); keep the action; line boundaries differ from string boundaries')
414
+ when "z", "Z"
415
+ raise unsupported("\\#{char}",
416
+ "for input without line feeds, rewrite rule(/a\\#{char}/) as rule(/a$/); keep the action; line boundaries differ from string boundaries")
417
+ when "R"
418
+ example = if @encoding == Encoding::BINARY
419
+ 'rule(/\r\n|[\n\v\f\r]/n)'
420
+ else
421
+ 'rule(/\r\n|[\n\v\f\r\u0085\u2028\u2029]/)'
422
+ end
423
+ raise unsupported("\\R", "for a standalone line-break token, use #{example}; keep the action")
424
+ when "K"
425
+ raise unsupported("\\K",
426
+ 'for /a\Kb/ in :initial, use rule(/a/, followed_by: /b/) { begin_state :suffix }; state(:suffix) { rule(/b/) { begin_state :initial; emit :B } }; keep the action on the suffix rule after returning to :initial; choose an unused state name')
427
+ when "g"
428
+ raise unsupported("\\g",
429
+ 'for this non-recursive call, rewrite rule(/(a|b)\g<1>/) as rule(/[ab]{2}/); keep the action; recursive calls need a parser or state stack')
430
+ when "X"
431
+ raise unsupported("\\X",
432
+ 'for a standalone grapheme token with ASCII-only input, rewrite rule(/\X/) as rule(/\r\n|[\x00-\x7f]/); keep the action; Unicode graphemes need separate segmentation')
433
+ when "c", "C"
434
+ raise unsupported("\\#{char}",
435
+ 'for control-A, rewrite rule(/\cA/) or rule(/\C-A/) as rule(/\x01/); keep the action; other control characters need their own hexadecimal code')
436
+ when "M"
437
+ raise unsupported("\\M",
438
+ 'for binary input, rewrite rule(/\M-a/n) as rule(/\xE1/n); keep the action; other meta characters need their own byte value')
439
+ when "1".."9", "k"
440
+ read_until(">") if char == "k" && consume?("<")
441
+ feature = char == "k" ? "backreferences" : "\\#{char}"
442
+ help = if @class_depth.positive? && char != "k"
443
+ 'for a character code, use an explicit hexadecimal escape such as rule(/[\x01]/); keep the action'
444
+ else
445
+ 'for a capture limited to a or b, rewrite rule(/(a|b)\1/) as rule(/(?:aa|bb)/) and keep the action; otherwise split the rule into states and compare matched text in actions'
446
+ end
447
+ raise unsupported(feature, help)
448
+ when *LITERAL_ESCAPES
308
449
  @escaped_value = true
309
450
  @last_codepoint = char.ord
451
+ else
452
+ raise unsupported("unknown escape \\#{char}",
453
+ 'if q is literal, rewrite rule(/\q/) as rule(/q/); otherwise use a supported escape; keep the action')
310
454
  end
311
455
  end
312
456
 
313
457
  def shorthand_ranges(char)
314
458
  if @unicode && @encoding != Encoding::BINARY
315
459
  property = { "d" => "Nd", "w" => "Word", "s" => "Space" }.fetch(char.downcase, nil)
316
- return [[AST::Property, char == char.upcase, property]] if property
460
+ return [[AST::Property, char == char.upcase, property, ignorecase?]] if property
317
461
  end
318
462
 
319
463
  base = {
320
464
  "d" => [[48, 57]],
321
465
  "w" => [[48, 57], [65, 90], [95, 95], [97, 122]],
322
466
  "s" => [[9, 13], [32, 32]],
323
- "h" => [[9, 9], [32, 32]]
467
+ "h" => [[48, 57], [65, 70], [97, 102]]
324
468
  }.fetch(char.downcase)
325
469
  return base unless char == char.upcase
326
470
 
@@ -339,10 +483,11 @@ module Flexr
339
483
  end
340
484
 
341
485
  def parse_anchor
486
+ starting = @index
342
487
  char = advance
343
- return AST::Anchor.new(kind: :bol, loc: nil) if char == "^"
488
+ return AST::Anchor.new(kind: :bol, loc: span(starting)) if char == "^"
344
489
 
345
- AST::Anchor.new(kind: :eol, loc: nil)
490
+ AST::Anchor.new(kind: :eol, loc: span(starting))
346
491
  end
347
492
 
348
493
  def validate_anchor_positions(node)
@@ -375,7 +520,7 @@ module Flexr
375
520
 
376
521
  def anchor_nodes(node)
377
522
  return [node] if node.is_a?(AST::Anchor)
378
- return anchor_nodes(node.child) if node.is_a?(AST::Star)
523
+ return anchor_nodes(node.child) if node.is_a?(AST::Star) || node.is_a?(AST::Repeat)
379
524
  return [] unless node.respond_to?(:children)
380
525
 
381
526
  node.children.flat_map { |child| anchor_nodes(child) }
@@ -383,7 +528,7 @@ module Flexr
383
528
 
384
529
  def anchor_nested_in_alternative?(node)
385
530
  return node.children.any? { |child| anchor_nodes(child).any? } if node.is_a?(AST::Alt)
386
- return anchor_nested_in_alternative?(node.child) if node.is_a?(AST::Star)
531
+ return anchor_nested_in_alternative?(node.child) if node.is_a?(AST::Star) || node.is_a?(AST::Repeat)
387
532
  return false unless node.respond_to?(:children)
388
533
 
389
534
  node.children.any? { |child| anchor_nested_in_alternative?(child) }
@@ -397,7 +542,8 @@ module Flexr
397
542
  def reject_postfix_quantifier
398
543
  return unless ["?", "+"].include?(current)
399
544
 
400
- raise unsupported("lazy or possessive quantifier", "use a negated character class")
545
+ raise unsupported("lazy or possessive quantifier",
546
+ 'for unescaped single-line quoted text, use a negated character class: rule(/"[^"\n]*"/); keep the action; removing lazy or possessive modifiers can change matches')
401
547
  end
402
548
 
403
549
  def merge_ranges(ranges)
@@ -503,6 +649,37 @@ module Flexr
503
649
  @last_codepoint = codepoint
504
650
  end
505
651
 
652
+ def assign_hex_byte(digits)
653
+ byte = digits.to_i(16)
654
+ if @encoding == Encoding::BINARY
655
+ @escaped_byte = true
656
+ @last_byte = byte
657
+ return
658
+ end
659
+
660
+ length = case byte
661
+ when 0x00..0x7f then 1
662
+ when 0xc2..0xdf then 2
663
+ when 0xe0..0xef then 3
664
+ when 0xf0..0xf4 then 4
665
+ else 0
666
+ end
667
+ bytes = [byte]
668
+ (length - 1).times do
669
+ expect("\\x")
670
+ bytes << read_exact(2).to_i(16)
671
+ end
672
+ value = bytes.pack("C*").force_encoding(Encoding::UTF_8)
673
+ raise_syntax("invalid UTF-8 byte escape") unless length.positive? && value.valid_encoding?
674
+
675
+ @escaped_value = true
676
+ @last_codepoint = value.ord
677
+ end
678
+
679
+ def ignorecase?
680
+ @options.anybits?(::Regexp::IGNORECASE)
681
+ end
682
+
506
683
  def skip_extended_space
507
684
  loop do
508
685
  @index += 1 while @source[@index]&.match?(/\s/)
@@ -512,6 +689,10 @@ module Flexr
512
689
  @index += 1 while @source[@index] && @source[@index] != "\n"
513
690
  end
514
691
  end
692
+
693
+ def span(starting, ending = @index)
694
+ @byte_offsets.fetch(starting)...@byte_offsets.fetch(ending)
695
+ end
515
696
  end
516
697
  end
517
698
  end