flexr 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/CHANGELOG.md +116 -0
- data/CONTRIBUTING.md +46 -3
- data/README.md +5 -4
- data/Rakefile +241 -72
- data/Steepfile +7 -0
- data/benchmark/baselines/json.json +2 -2
- data/benchmark/golden/calculator_lexer.rb +276 -0
- data/benchmark/golden/graphql_lexer.rb +310 -0
- data/benchmark/golden/json_lexer.rb +292 -0
- data/benchmark/golden/regexp_tokenizer.rb +293 -0
- data/benchmark/golden/ruby_subset_lexer.rb +280 -0
- data/benchmark/golden/sql_lexer.rb +293 -0
- data/benchmark/golden/toml_lexer.rb +292 -0
- data/benchmark/golden/toy_lang_lexer.rb +273 -0
- data/benchmark/golden/with_lrama_lexer.rb +264 -0
- data/benchmark/golden/with_racc_lexer.rb +264 -0
- data/benchmark/lrama_lexer.rb +165 -0
- data/benchmark/matrix.rb +79 -0
- data/benchmark/profile.rb +109 -0
- data/benchmark/run.rb +4 -1
- data/benchmark/state_dispatch.rb +78 -0
- data/docs/RELEASING.md +3 -2
- data/docs/explanation/backends.md +30 -10
- data/docs/explanation/matching-semantics.md +4 -2
- data/docs/explanation/security-model.md +10 -3
- data/docs/explanation/unicode-and-encoding.md +3 -1
- data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
- data/docs/how-to/generate-a-lexer.md +3 -3
- data/docs/how-to/handle-errors.md +4 -1
- data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
- data/docs/how-to/track-token-locations.md +2 -1
- data/docs/how-to/tune-performance.md +15 -3
- data/docs/migration-v2.md +74 -0
- data/docs/perf-log.md +212 -0
- data/docs/reference/README.md +2 -0
- data/docs/reference/actions.md +5 -2
- data/docs/reference/cli.md +71 -5
- data/docs/reference/compatibility.md +10 -4
- data/docs/reference/diagnostics.md +346 -0
- data/docs/reference/dsl.md +30 -9
- data/docs/reference/errors.md +34 -3
- data/docs/reference/generated-artifacts.md +71 -6
- data/docs/reference/public-api.md +37 -7
- data/docs/reference/regexp.md +27 -13
- data/docs/reference/ruby-lsp.md +34 -0
- data/docs/reference/runtime.md +21 -4
- data/docs/reference/testing.md +57 -0
- data/docs/reference/tokens-and-locations.md +14 -4
- data/examples/graphql/README.md +14 -0
- data/examples/graphql/lexer.flexr.rb +29 -0
- data/examples/json/lexer.flexr.rb +6 -6
- data/examples/sql/README.md +16 -0
- data/examples/sql/lexer.flexr.rb +23 -0
- data/examples/toml/README.md +14 -0
- data/examples/toml/lexer.flexr.rb +23 -0
- data/gemfiles/ruby_lsp.gemfile +4 -0
- data/gemfiles/types.gemfile +7 -0
- data/lib/flexr/action_resolver.rb +15 -0
- data/lib/flexr/artifact_writer.rb +61 -0
- data/lib/flexr/automaton/accel.rb +78 -5
- data/lib/flexr/automaton/analysis.rb +202 -7
- data/lib/flexr/automaton/backend_cost_model.rb +40 -0
- data/lib/flexr/automaton/compiler.rb +229 -96
- data/lib/flexr/automaton/dfa.rb +106 -19
- data/lib/flexr/automaton/minimizer.rb +79 -31
- data/lib/flexr/automaton/nfa.rb +45 -13
- data/lib/flexr/automaton/types.rb +14 -0
- data/lib/flexr/cli.rb +128 -22
- data/lib/flexr/codegen/direct.rb +5 -41
- data/lib/flexr/codegen/firstmatch.rb +1 -7
- data/lib/flexr/codegen/table.rb +181 -69
- data/lib/flexr/codegen.rb +8 -0
- data/lib/flexr/configuration.rb +45 -0
- data/lib/flexr/diagnostics.rb +39 -3
- data/lib/flexr/dsl.rb +116 -47
- data/lib/flexr/errors.rb +7 -6
- data/lib/flexr/generated.rb +120 -46
- data/lib/flexr/generator.rb +102 -109
- data/lib/flexr/grammar.rb +41 -0
- data/lib/flexr/importer.rb +4 -40
- data/lib/flexr/ir.rb +2 -2
- data/lib/flexr/migration.rb +31 -0
- data/lib/flexr/minitest.rb +24 -0
- data/lib/flexr/options.rb +9 -5
- data/lib/flexr/rake_task.rb +6 -1
- data/lib/flexr/regexp/ast.rb +5 -2
- data/lib/flexr/regexp/matcher.rb +26 -0
- data/lib/flexr/regexp/normalizer.rb +63 -21
- data/lib/flexr/regexp/parser.rb +241 -60
- data/lib/flexr/regexp/tokenizer.rb +172 -47
- data/lib/flexr/rspec.rb +51 -0
- data/lib/flexr/runtime/buffer.rb +115 -9
- data/lib/flexr/runtime/core.rb +248 -51
- data/lib/flexr/runtime/errors.rb +66 -4
- data/lib/flexr/runtime/interpreter.rb +213 -97
- data/lib/flexr/runtime/location.rb +35 -12
- data/lib/flexr/runtime/record.rb +14 -0
- data/lib/flexr/runtime/token.rb +7 -1
- data/lib/flexr/runtime.rb +75 -0
- data/lib/flexr/source/passthrough.rb +70 -14
- data/lib/flexr/source/prism_reader.rb +217 -51
- data/lib/flexr/source/static_eval.rb +40 -4
- data/lib/flexr/source.rb +6 -0
- data/lib/flexr/testing.rb +47 -0
- data/lib/flexr/trace.rb +197 -0
- data/lib/flexr/unicode/data/properties.rb +1 -1
- data/lib/flexr/unicode/data.rb +11 -0
- data/lib/flexr/unicode/property.rb +4 -8
- data/lib/flexr/unicode/reference_regexp.rb +11 -9
- data/lib/flexr/unicode/version.rb +7 -0
- data/lib/flexr/version.rb +1 -1
- data/lib/flexr.rb +10 -76
- data/lib/ruby_lsp/flexr/addon.rb +114 -0
- data/sig/flexr.rbs +191 -0
- data/site/README.md +54 -3
- data/site/astro.config.mjs +3 -1
- data/site/package.json +12 -5
- data/site/playwright.config.mjs +16 -0
- data/site/pnpm-lock.yaml +218 -88
- data/site/ruby/playground.rb +57 -0
- data/site/scripts/prepare-playground.mjs +35 -0
- data/site/scripts/verify-site.mjs +5 -3
- data/site/src/content/docs/concepts/matching-semantics.md +4 -2
- data/site/src/content/docs/concepts/security-model.md +9 -1
- data/site/src/content/docs/learn/generation.md +1 -1
- data/site/src/content/docs/reference/dsl.md +2 -2
- data/site/src/content/docs/reference/public-api.md +4 -2
- data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
- data/site/src/layouts/SiteLayout.astro +11 -3
- data/site/src/pages/index.astro +69 -155
- data/site/src/pages/playground.astro +40 -38
- data/site/src/playground/client.js +211 -0
- data/site/src/playground/graph-worker.mjs +11 -0
- data/site/src/playground/presets.mjs +33 -0
- data/site/src/playground/runtime.mjs +34 -0
- data/site/src/playground/share.mjs +30 -0
- data/site/src/playground/worker.mjs +29 -0
- data/site/src/styles/custom.css +12 -287
- data/site/src/styles/landing.css +239 -0
- data/site/tests/browser/playground.spec.mjs +99 -0
- data/site/tests/playground-runtime.test.mjs +85 -0
- data/test/ruby_lsp_addon_spec.rb +135 -0
- data/tools/coverage.rb +6 -1
- data/tools/docs_verify.rb +3 -2
- data/tools/golden.rb +54 -0
- data/tools/verification_failure.rb +119 -0
- metadata +66 -12
- data/benchmark/golden/calculator_lexer.sha256 +0 -1
- data/benchmark/golden/json_lexer.sha256 +0 -1
- data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
- data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
- data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
- data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
- data/benchmark/golden/with_racc_lexer.sha256 +0 -1
- data/site/public/playground.js +0 -189
|
@@ -18,7 +18,7 @@ module Flexr
|
|
|
18
18
|
|
|
19
19
|
def normalize_node(node)
|
|
20
20
|
case node
|
|
21
|
-
when AST::Empty, AST::ByteRange, AST::CodepointRange, AST::Anchor
|
|
21
|
+
when AST::Empty, AST::Fail, AST::ByteRange, AST::CodepointRange, AST::Anchor
|
|
22
22
|
normalize_leaf(node)
|
|
23
23
|
when AST::CharClass
|
|
24
24
|
char_class(node)
|
|
@@ -27,7 +27,10 @@ module Flexr
|
|
|
27
27
|
when AST::Alt
|
|
28
28
|
alternatives(node.children.map { |child| normalize_node(child) })
|
|
29
29
|
when AST::Star
|
|
30
|
-
|
|
30
|
+
child = normalize_node(node.child)
|
|
31
|
+
child.is_a?(AST::Fail) ? AST::Empty.new(loc: node.loc) : AST::Star.new(child: child, loc: node.loc)
|
|
32
|
+
when AST::Repeat
|
|
33
|
+
repeat(node)
|
|
31
34
|
else
|
|
32
35
|
raise CompileError, "unknown regexp AST node: #{node.class}"
|
|
33
36
|
end
|
|
@@ -36,7 +39,7 @@ module Flexr
|
|
|
36
39
|
def normalize_leaf(node)
|
|
37
40
|
case node
|
|
38
41
|
when AST::CodepointRange
|
|
39
|
-
byte_sequences_for_ranges(
|
|
42
|
+
byte_sequences_for_ranges([[node.lo, node.hi]], loc: node.loc)
|
|
40
43
|
else
|
|
41
44
|
node
|
|
42
45
|
end
|
|
@@ -45,44 +48,76 @@ module Flexr
|
|
|
45
48
|
def char_class(node)
|
|
46
49
|
ranges = node.ranges.flat_map do |range|
|
|
47
50
|
if range.first == AST::Property
|
|
48
|
-
Unicode::Property.ranges(range
|
|
51
|
+
property_ranges = Unicode::Property.ranges(range[2])
|
|
52
|
+
property_ranges = property_ranges.flat_map { |lo, hi| Unicode::CaseFold.ranges(lo, hi) } if range[3]
|
|
53
|
+
range[1] ? complement(property_ranges) : property_ranges
|
|
49
54
|
else
|
|
50
|
-
|
|
55
|
+
[range]
|
|
51
56
|
end
|
|
52
57
|
end
|
|
53
58
|
ranges = complement(ranges) if node.negated
|
|
54
|
-
byte_sequences_for_ranges(ranges)
|
|
59
|
+
byte_sequences_for_ranges(ranges, loc: node.loc)
|
|
55
60
|
end
|
|
56
61
|
|
|
57
|
-
def byte_sequences_for_ranges(ranges)
|
|
62
|
+
def byte_sequences_for_ranges(ranges, loc:)
|
|
58
63
|
if @byte_mode
|
|
59
64
|
byte_ranges = ranges.filter_map do |lo, hi|
|
|
60
65
|
next if lo > 255
|
|
61
66
|
|
|
62
67
|
[lo, [hi, 255].min]
|
|
63
68
|
end
|
|
64
|
-
return AST::
|
|
69
|
+
return AST::Fail.new(loc: loc) if byte_ranges.empty?
|
|
65
70
|
|
|
66
|
-
return alternatives(byte_ranges.map { |lo, hi| AST::ByteRange.new(lo: lo, hi: hi, loc:
|
|
71
|
+
return alternatives(byte_ranges.map { |lo, hi| AST::ByteRange.new(lo: lo, hi: hi, loc: loc) })
|
|
67
72
|
end
|
|
68
73
|
|
|
69
74
|
sequences = ranges.flat_map { |lo, hi| Unicode::Utf8Splitter.split(lo, hi) }
|
|
75
|
+
@trie_cache = {}
|
|
70
76
|
trie(sequences, 0)
|
|
71
77
|
end
|
|
72
78
|
|
|
73
79
|
def trie(sequences, index)
|
|
74
|
-
return AST::
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
80
|
+
return AST::Fail.new(loc: nil) if sequences.empty?
|
|
81
|
+
|
|
82
|
+
finished, continuing = sequences.partition { |sequence| sequence.length == index }
|
|
83
|
+
nodes = []
|
|
84
|
+
nodes << AST::Empty.new(loc: nil) unless finished.empty?
|
|
85
|
+
return nodes.first if continuing.empty?
|
|
86
|
+
|
|
87
|
+
key = [index, continuing]
|
|
88
|
+
return alternatives(nodes + [@trie_cache[key]]) if @trie_cache.key?(key)
|
|
89
|
+
|
|
90
|
+
endpoints = continuing.flat_map do |sequence|
|
|
91
|
+
lo, hi = sequence.fetch(index)
|
|
92
|
+
[lo, hi + 1]
|
|
93
|
+
end.uniq.sort
|
|
94
|
+
partitions = endpoints.each_cons(2).filter_map do |lo, after_hi|
|
|
95
|
+
covered = continuing.select do |sequence|
|
|
96
|
+
range = sequence.fetch(index)
|
|
97
|
+
range.first <= lo && range.last >= after_hi - 1
|
|
98
|
+
end
|
|
99
|
+
[lo, after_hi - 1, covered] unless covered.empty?
|
|
100
|
+
end
|
|
101
|
+
branches = partitions.map { |lo, hi, covered| [lo, hi, trie(covered, index + 1)] }
|
|
102
|
+
merged = branches.each_with_object([]) do |(lo, hi, child), result|
|
|
103
|
+
if !result.empty? && result.last[1] + 1 == lo && result.last[2] == child
|
|
104
|
+
result.last[1] = hi
|
|
105
|
+
else
|
|
106
|
+
result << [lo, hi, child]
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
tree = alternatives(merged.map do |lo, hi, child|
|
|
110
|
+
sequence([AST::ByteRange.new(lo: lo, hi: hi, loc: nil), child])
|
|
81
111
|
end)
|
|
112
|
+
@trie_cache[key] = tree
|
|
113
|
+
alternatives(nodes + [tree])
|
|
82
114
|
end
|
|
83
115
|
|
|
84
116
|
def sequence(children)
|
|
85
117
|
flattened = children.flat_map { |child| child.is_a?(AST::Seq) ? child.children : [child] }
|
|
118
|
+
return AST::Fail.new(loc: nil) if flattened.any?(AST::Fail)
|
|
119
|
+
|
|
120
|
+
flattened.reject! { |child| child.is_a?(AST::Empty) }
|
|
86
121
|
return AST::Empty.new(loc: nil) if flattened.empty?
|
|
87
122
|
return flattened.first if flattened.length == 1
|
|
88
123
|
|
|
@@ -91,12 +126,23 @@ module Flexr
|
|
|
91
126
|
|
|
92
127
|
def alternatives(children)
|
|
93
128
|
flattened = children.flat_map { |child| child.is_a?(AST::Alt) ? child.children : [child] }
|
|
94
|
-
|
|
129
|
+
flattened.reject! { |child| child.is_a?(AST::Fail) }
|
|
130
|
+
flattened.uniq!
|
|
131
|
+
return AST::Fail.new(loc: nil) if flattened.empty?
|
|
95
132
|
return flattened.first if flattened.length == 1
|
|
96
133
|
|
|
97
134
|
AST::Alt.new(children: flattened, loc: nil)
|
|
98
135
|
end
|
|
99
136
|
|
|
137
|
+
def repeat(node)
|
|
138
|
+
child = normalize_node(node.child)
|
|
139
|
+
return AST::Empty.new(loc: node.loc) if node.maximum&.zero? || child.is_a?(AST::Empty)
|
|
140
|
+
return node.minimum.zero? ? AST::Empty.new(loc: node.loc) : AST::Fail.new(loc: node.loc) if
|
|
141
|
+
child.is_a?(AST::Fail)
|
|
142
|
+
|
|
143
|
+
AST::Repeat.new(child: child, minimum: node.minimum, maximum: node.maximum, loc: node.loc)
|
|
144
|
+
end
|
|
145
|
+
|
|
100
146
|
def complement(ranges)
|
|
101
147
|
result = []
|
|
102
148
|
cursor = 0
|
|
@@ -108,10 +154,6 @@ module Flexr
|
|
|
108
154
|
result
|
|
109
155
|
end
|
|
110
156
|
|
|
111
|
-
def casefold_ranges(lo, hi)
|
|
112
|
-
return [[lo, hi]] if @options.nobits?(::Regexp::IGNORECASE)
|
|
113
|
-
Unicode::CaseFold.ranges(lo, hi)
|
|
114
|
-
end
|
|
115
157
|
end
|
|
116
158
|
end
|
|
117
159
|
end
|
data/lib/flexr/regexp/parser.rb
CHANGED
|
@@ -12,6 +12,7 @@ module Flexr
|
|
|
12
12
|
"word" => "Word", "space" => "Space"
|
|
13
13
|
}.freeze
|
|
14
14
|
POSIX_CLASSES = %w[alnum alpha blank cntrl digit graph lower print punct space upper xdigit].freeze
|
|
15
|
+
LITERAL_ESCAPES = (%w[. [ ] { } ( ) * + ? | ^ $ \\ / - #] + [" "]).freeze
|
|
15
16
|
|
|
16
17
|
attr_reader :source
|
|
17
18
|
|
|
@@ -22,6 +23,12 @@ module Flexr
|
|
|
22
23
|
@unicode = unicode
|
|
23
24
|
@index = 0
|
|
24
25
|
@class_depth = 0
|
|
26
|
+
offset = 0
|
|
27
|
+
@byte_offsets = [0]
|
|
28
|
+
source.each_char do |character|
|
|
29
|
+
offset += character.bytesize
|
|
30
|
+
@byte_offsets << offset
|
|
31
|
+
end
|
|
25
32
|
end
|
|
26
33
|
|
|
27
34
|
def parse
|
|
@@ -34,43 +41,46 @@ module Flexr
|
|
|
34
41
|
private
|
|
35
42
|
|
|
36
43
|
def parse_expression
|
|
44
|
+
starting = @index
|
|
37
45
|
branches = [parse_sequence]
|
|
38
46
|
branches << parse_sequence while consume?("|")
|
|
39
47
|
return branches.first if branches.length == 1
|
|
40
48
|
|
|
41
|
-
AST::Alt.new(children: branches, loc:
|
|
49
|
+
AST::Alt.new(children: branches, loc: span(starting))
|
|
42
50
|
end
|
|
43
51
|
|
|
44
52
|
def parse_sequence
|
|
53
|
+
starting = @index
|
|
45
54
|
children = []
|
|
46
55
|
children << parse_quantified until eof? || [")", "|"].include?(current)
|
|
47
|
-
return AST::Empty.new(loc:
|
|
56
|
+
return AST::Empty.new(loc: span(starting)) if children.empty?
|
|
48
57
|
return children.first if children.length == 1
|
|
49
58
|
|
|
50
|
-
AST::Seq.new(children: children, loc:
|
|
59
|
+
AST::Seq.new(children: children, loc: span(starting))
|
|
51
60
|
end
|
|
52
61
|
|
|
53
62
|
def parse_quantified
|
|
63
|
+
starting = @index
|
|
54
64
|
atom = parse_atom
|
|
55
65
|
return atom unless ["*", "+", "?", "{"].include?(current)
|
|
56
66
|
|
|
57
67
|
if consume?("*")
|
|
58
68
|
reject_postfix_quantifier
|
|
59
|
-
return AST::
|
|
69
|
+
return AST::Repeat.new(child: atom, minimum: 0, maximum: nil, loc: span(starting))
|
|
60
70
|
end
|
|
61
71
|
if consume?("+")
|
|
62
72
|
reject_postfix_quantifier
|
|
63
|
-
return AST::
|
|
73
|
+
return AST::Repeat.new(child: atom, minimum: 1, maximum: nil, loc: span(starting))
|
|
64
74
|
end
|
|
65
75
|
if consume?("?")
|
|
66
76
|
reject_postfix_quantifier
|
|
67
|
-
return AST::
|
|
77
|
+
return AST::Repeat.new(child: atom, minimum: 0, maximum: 1, loc: span(starting))
|
|
68
78
|
end
|
|
69
79
|
|
|
70
|
-
parse_repetition(atom)
|
|
80
|
+
parse_repetition(atom, starting)
|
|
71
81
|
end
|
|
72
82
|
|
|
73
|
-
def parse_repetition(atom)
|
|
83
|
+
def parse_repetition(atom, starting)
|
|
74
84
|
consume?("{")
|
|
75
85
|
min = read_number
|
|
76
86
|
max = if consume?(",")
|
|
@@ -80,8 +90,7 @@ module Flexr
|
|
|
80
90
|
end
|
|
81
91
|
expect("}")
|
|
82
92
|
raise_syntax("invalid repetition") if min.nil? || (!max.nil? && max < min)
|
|
83
|
-
|
|
84
|
-
if max > 1000
|
|
93
|
+
if (max || min) > 1000
|
|
85
94
|
raise_diagnostic(
|
|
86
95
|
diagnostic("FLEXR-E007", "repetition limit exceeds 1000",
|
|
87
96
|
help: "split the rule or use a smaller bounded repetition")
|
|
@@ -89,19 +98,13 @@ module Flexr
|
|
|
89
98
|
end
|
|
90
99
|
reject_postfix_quantifier
|
|
91
100
|
|
|
92
|
-
|
|
93
|
-
optional = Array.new(max - min) do
|
|
94
|
-
AST::Alt.new(children: [atom, AST::Empty.new(loc: nil)], loc: nil)
|
|
95
|
-
end
|
|
96
|
-
children = required + optional
|
|
97
|
-
return AST::Empty.new(loc: nil) if children.empty?
|
|
98
|
-
return children.first if children.length == 1
|
|
99
|
-
|
|
100
|
-
AST::Seq.new(children: children, loc: nil)
|
|
101
|
+
AST::Repeat.new(child: atom, minimum: min, maximum: max, loc: span(starting))
|
|
101
102
|
end
|
|
102
103
|
|
|
103
104
|
def parse_atom
|
|
105
|
+
starting = @index
|
|
104
106
|
@escaped_value = false
|
|
107
|
+
@escaped_byte = false
|
|
105
108
|
@last_ranges = nil
|
|
106
109
|
return parse_group if consume?("(")
|
|
107
110
|
return parse_class if consume?("[")
|
|
@@ -109,25 +112,29 @@ module Flexr
|
|
|
109
112
|
|
|
110
113
|
if consume?(".")
|
|
111
114
|
upper = @options.nobits?(::Regexp::MULTILINE) ? 0x0a - 1 : 0x10ffff
|
|
112
|
-
return AST::CharClass.new(
|
|
115
|
+
return AST::CharClass.new(
|
|
116
|
+
ranges: [[0, upper], [0x0b, 0x10ffff]], negated: false, loc: span(starting)
|
|
117
|
+
)
|
|
113
118
|
end
|
|
114
119
|
|
|
115
120
|
if consume?("\\")
|
|
116
121
|
parse_escape
|
|
117
|
-
return AST::CharClass.new(ranges: @last_ranges, negated: false, loc:
|
|
118
|
-
return
|
|
122
|
+
return AST::CharClass.new(ranges: @last_ranges, negated: false, loc: span(starting)) if @last_ranges
|
|
123
|
+
return AST::ByteRange.new(lo: @last_byte, hi: @last_byte, loc: span(starting)) if @escaped_byte
|
|
124
|
+
return codepoint_node(read_codepoint, span(starting)) if @escaped_value
|
|
119
125
|
end
|
|
120
126
|
|
|
121
127
|
char = advance
|
|
122
128
|
raise_syntax("unexpected end of expression") unless char
|
|
123
|
-
codepoint_node(char.ord)
|
|
129
|
+
codepoint_node(char.ord, span(starting))
|
|
124
130
|
end
|
|
125
131
|
|
|
126
132
|
def parse_group
|
|
133
|
+
starting = @index - 1
|
|
127
134
|
saved_options = @options
|
|
128
135
|
if consume?("?")
|
|
129
136
|
prefix = parse_group_prefix
|
|
130
|
-
return AST::Empty.new(loc:
|
|
137
|
+
return AST::Empty.new(loc: span(starting)) if prefix == :global
|
|
131
138
|
else
|
|
132
139
|
warn_capture
|
|
133
140
|
end
|
|
@@ -164,8 +171,30 @@ module Flexr
|
|
|
164
171
|
end
|
|
165
172
|
raise_syntax("invalid inline option group")
|
|
166
173
|
end
|
|
167
|
-
|
|
168
|
-
|
|
174
|
+
if peek_prefix?("=")
|
|
175
|
+
raise unsupported("look-around",
|
|
176
|
+
"for a trailing positive lookahead, rewrite rule(/a(?=b)/) as rule(/a/, followed_by: /b/); keep the action")
|
|
177
|
+
end
|
|
178
|
+
if peek_prefix?("!") || peek_prefix?("<=") || peek_prefix?("<!")
|
|
179
|
+
raise unsupported("look-around",
|
|
180
|
+
"use separate rules or states; followed_by: cannot express negative lookahead or lookbehind")
|
|
181
|
+
end
|
|
182
|
+
if consume?(">")
|
|
183
|
+
raise unsupported("atomic groups",
|
|
184
|
+
'for unescaped single-line quoted text, a DFA-compatible rewrite is rule(/"[^"\n]*"/); keep the action; removing atomic groups can change matches')
|
|
185
|
+
end
|
|
186
|
+
if peek_prefix?("<") || peek_prefix?("'")
|
|
187
|
+
raise unsupported("named captures",
|
|
188
|
+
"when only grouping is needed, rewrite rule(/(?<word>ab)/) as rule(/(?:ab)/); keep the action; capture-dependent patterns need a separate rewrite")
|
|
189
|
+
end
|
|
190
|
+
if peek_prefix?("(")
|
|
191
|
+
raise unsupported("conditional groups",
|
|
192
|
+
"for this optional-prefix conditional, rewrite rule(/(a)?(?(1)b|c)/) as rule(/(?:ab|c)/); keep the action; other conditions need separate rules or state checks")
|
|
193
|
+
end
|
|
194
|
+
if peek_prefix?("~")
|
|
195
|
+
raise unsupported("absence groups",
|
|
196
|
+
"when excluding a single literal, rewrite rule(/x(?~a)/) as rule(/x[^a]*/); keep the action; larger absence expressions need a separate rewrite")
|
|
197
|
+
end
|
|
169
198
|
raise unsupported("unsupported group syntax", "use a non-capturing group (?:...)")
|
|
170
199
|
end
|
|
171
200
|
|
|
@@ -174,10 +203,11 @@ module Flexr
|
|
|
174
203
|
@options = enabled ? (@options | bit) : (@options & ~bit)
|
|
175
204
|
end
|
|
176
205
|
|
|
177
|
-
def codepoint_node(codepoint)
|
|
178
|
-
return AST::CodepointRange.new(lo: codepoint, hi: codepoint, loc:
|
|
206
|
+
def codepoint_node(codepoint, location)
|
|
207
|
+
return AST::CodepointRange.new(lo: codepoint, hi: codepoint, loc: location) if
|
|
208
|
+
@options.nobits?(::Regexp::IGNORECASE)
|
|
179
209
|
|
|
180
|
-
AST::CharClass.new(ranges: fold_ranges([[codepoint, codepoint]]), negated: false, loc:
|
|
210
|
+
AST::CharClass.new(ranges: fold_ranges([[codepoint, codepoint]]), negated: false, loc: location)
|
|
181
211
|
end
|
|
182
212
|
|
|
183
213
|
def fold_ranges(ranges)
|
|
@@ -190,29 +220,51 @@ module Flexr
|
|
|
190
220
|
end
|
|
191
221
|
|
|
192
222
|
def parse_class
|
|
223
|
+
starting = @index - 1
|
|
224
|
+
@class_depth += 1
|
|
193
225
|
negated = consume?("^")
|
|
226
|
+
ranges = parse_class_union
|
|
227
|
+
while peek_prefix?("&&")
|
|
228
|
+
@index += 2
|
|
229
|
+
ranges = materialize_class_ranges(ranges, negated: negated)
|
|
230
|
+
negated = false
|
|
231
|
+
right = if consume?("[")
|
|
232
|
+
nested = parse_class
|
|
233
|
+
materialize_class_ranges(nested.ranges, negated: nested.negated)
|
|
234
|
+
else
|
|
235
|
+
materialize_class_ranges(parse_class_union, negated: false)
|
|
236
|
+
end
|
|
237
|
+
ranges = intersect_ranges(ranges, right)
|
|
238
|
+
end
|
|
239
|
+
expect("]")
|
|
240
|
+
ranges = merge_ranges(ranges)
|
|
241
|
+
ranges = fold_ranges(ranges) if @options.anybits?(::Regexp::IGNORECASE)
|
|
242
|
+
AST::CharClass.new(ranges: ranges, negated: negated, loc: span(starting))
|
|
243
|
+
ensure
|
|
244
|
+
@class_depth -= 1
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
def parse_class_union
|
|
194
248
|
ranges = []
|
|
195
|
-
|
|
196
|
-
until eof? || current == "]"
|
|
249
|
+
until eof? || current == "]" || peek_prefix?("&&")
|
|
197
250
|
if current == "[" && @source[@index, 2] == "[:"
|
|
198
251
|
ranges.concat(parse_posix_class)
|
|
199
252
|
next
|
|
200
253
|
end
|
|
201
254
|
first = parse_class_atom
|
|
202
|
-
if consume?("-")
|
|
203
|
-
|
|
204
|
-
|
|
255
|
+
if consume?("-")
|
|
256
|
+
if current == "]"
|
|
257
|
+
ranges.concat(first)
|
|
258
|
+
ranges << [45, 45]
|
|
259
|
+
else
|
|
260
|
+
last = parse_class_atom
|
|
261
|
+
ranges.concat(expand_class_range(first, last))
|
|
262
|
+
end
|
|
205
263
|
else
|
|
206
264
|
ranges.concat(first)
|
|
207
265
|
end
|
|
208
266
|
end
|
|
209
|
-
|
|
210
|
-
@class_depth -= 1
|
|
211
|
-
ranges = merge_ranges(ranges)
|
|
212
|
-
ranges = fold_ranges(ranges) if @options.anybits?(::Regexp::IGNORECASE)
|
|
213
|
-
AST::CharClass.new(ranges: ranges, negated: negated, loc: nil)
|
|
214
|
-
ensure
|
|
215
|
-
@class_depth -= 1 if @class_depth.positive? && current != "]"
|
|
267
|
+
ranges
|
|
216
268
|
end
|
|
217
269
|
|
|
218
270
|
def parse_posix_class
|
|
@@ -220,7 +272,7 @@ module Flexr
|
|
|
220
272
|
raw_name = read_until(":]")
|
|
221
273
|
inner_negated = raw_name.start_with?("^")
|
|
222
274
|
name = raw_name.delete_prefix("^")
|
|
223
|
-
return [[AST::Property, inner_negated, "POSIX_#{name}"]] if
|
|
275
|
+
return [[AST::Property, inner_negated, "POSIX_#{name}", ignorecase?]] if
|
|
224
276
|
@encoding != Encoding::BINARY && POSIX_CLASSES.include?(name)
|
|
225
277
|
|
|
226
278
|
ranges = case name
|
|
@@ -245,6 +297,7 @@ module Flexr
|
|
|
245
297
|
def parse_class_atom
|
|
246
298
|
if consume?("\\")
|
|
247
299
|
parse_escape
|
|
300
|
+
return [[@last_byte, @last_byte]] if @escaped_byte
|
|
248
301
|
return [[@last_codepoint, @last_codepoint]] if @escaped_value
|
|
249
302
|
return @last_ranges
|
|
250
303
|
end
|
|
@@ -261,16 +314,59 @@ module Flexr
|
|
|
261
314
|
[[lo, hi]]
|
|
262
315
|
end
|
|
263
316
|
|
|
317
|
+
def materialize_class_ranges(ranges, negated:)
|
|
318
|
+
concrete = ranges.flat_map do |range|
|
|
319
|
+
if range.first == AST::Property
|
|
320
|
+
property = Unicode::Property.ranges(range[2])
|
|
321
|
+
property = fold_ranges(property) if range[3]
|
|
322
|
+
range[1] ? complement_ranges(property) : property
|
|
323
|
+
else
|
|
324
|
+
[range]
|
|
325
|
+
end
|
|
326
|
+
end
|
|
327
|
+
concrete = merge_ranges(concrete)
|
|
328
|
+
negated ? complement_ranges(concrete) : concrete
|
|
329
|
+
end
|
|
330
|
+
|
|
331
|
+
def intersect_ranges(left, right)
|
|
332
|
+
result = []
|
|
333
|
+
left_index = 0
|
|
334
|
+
right_index = 0
|
|
335
|
+
while left_index < left.length && right_index < right.length
|
|
336
|
+
left_range = left[left_index]
|
|
337
|
+
right_range = right[right_index]
|
|
338
|
+
lo = [left_range.first, right_range.first].max
|
|
339
|
+
hi = [left_range.last, right_range.last].min
|
|
340
|
+
result << [lo, hi] if lo <= hi
|
|
341
|
+
if left_range.last < right_range.last
|
|
342
|
+
left_index += 1
|
|
343
|
+
else
|
|
344
|
+
right_index += 1
|
|
345
|
+
end
|
|
346
|
+
end
|
|
347
|
+
result
|
|
348
|
+
end
|
|
349
|
+
|
|
264
350
|
def parse_escape
|
|
265
351
|
@escaped_value = false
|
|
352
|
+
@escaped_byte = false
|
|
266
353
|
@last_ranges = nil
|
|
267
354
|
char = advance_raw
|
|
268
355
|
raise_syntax("trailing backslash") unless char
|
|
269
356
|
if ESCAPES.key?(char)
|
|
357
|
+
if char == "0" && current&.match?(/[0-9]/)
|
|
358
|
+
raise unsupported("octal escapes",
|
|
359
|
+
'use an explicit hexadecimal escape: rewrite rule(/\011/) as rule(/\x09/) for a tab; keep the action')
|
|
360
|
+
end
|
|
361
|
+
|
|
270
362
|
@escaped_value = true
|
|
271
363
|
@last_codepoint = ESCAPES.fetch(char)
|
|
272
364
|
return
|
|
273
365
|
end
|
|
366
|
+
if @class_depth.positive? && %w[A B G z Z K g X].include?(char)
|
|
367
|
+
raise unsupported("\\#{char}",
|
|
368
|
+
"inside a character class this is a literal character: rewrite rule(/[\\#{char}]/) as rule(/[#{char}]/); keep the action")
|
|
369
|
+
end
|
|
274
370
|
case char
|
|
275
371
|
when "d", "D", "w", "W", "s", "S", "h", "H"
|
|
276
372
|
ranges = shorthand_ranges(char)
|
|
@@ -279,16 +375,15 @@ module Flexr
|
|
|
279
375
|
when "p", "P"
|
|
280
376
|
expect("{")
|
|
281
377
|
name = read_until("}")
|
|
282
|
-
ranges = [[AST::Property, char == "P", name]]
|
|
378
|
+
ranges = [[AST::Property, char == "P", name, ignorecase?]]
|
|
283
379
|
@last_ranges = ranges
|
|
284
380
|
nil
|
|
285
381
|
when "x"
|
|
286
|
-
|
|
287
|
-
read_until("}")
|
|
382
|
+
if consume?("{")
|
|
383
|
+
assign_codepoint(read_until("}"))
|
|
288
384
|
else
|
|
289
|
-
read_exact(2)
|
|
385
|
+
assign_hex_byte(read_exact(2))
|
|
290
386
|
end
|
|
291
|
-
assign_codepoint(digits)
|
|
292
387
|
nil
|
|
293
388
|
when "u"
|
|
294
389
|
digits = if consume?("{")
|
|
@@ -299,28 +394,77 @@ module Flexr
|
|
|
299
394
|
end
|
|
300
395
|
assign_codepoint(digits)
|
|
301
396
|
nil
|
|
302
|
-
when "
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
397
|
+
when "b"
|
|
398
|
+
unless @class_depth.positive?
|
|
399
|
+
raise unsupported("\\b",
|
|
400
|
+
'when surrounding literals already establish the boundary, rewrite rule(/foo\b:/) as rule(/foo:/); keep the action; token edges need separate rules or state checks')
|
|
401
|
+
end
|
|
402
|
+
|
|
403
|
+
@escaped_value = true
|
|
404
|
+
@last_codepoint = 0x08
|
|
405
|
+
when "B"
|
|
406
|
+
raise unsupported("\\B",
|
|
407
|
+
'when surrounding literals already establish the non-boundary, rewrite rule(/foo\Bbar/) as rule(/foobar/); keep the action; token edges need separate rules or state checks')
|
|
408
|
+
when "G"
|
|
409
|
+
raise unsupported("\\G",
|
|
410
|
+
'at the start of a rule, rewrite rule(/\Ga/) as rule(/a/); keep the action; rules already match at the current lexer position; interior \G needs a separate rewrite')
|
|
411
|
+
when "A"
|
|
412
|
+
raise unsupported("\\A",
|
|
413
|
+
'for input without line feeds, rewrite rule(/\Aa/) as rule(/^a/); keep the action; line boundaries differ from string boundaries')
|
|
414
|
+
when "z", "Z"
|
|
415
|
+
raise unsupported("\\#{char}",
|
|
416
|
+
"for input without line feeds, rewrite rule(/a\\#{char}/) as rule(/a$/); keep the action; line boundaries differ from string boundaries")
|
|
417
|
+
when "R"
|
|
418
|
+
example = if @encoding == Encoding::BINARY
|
|
419
|
+
'rule(/\r\n|[\n\v\f\r]/n)'
|
|
420
|
+
else
|
|
421
|
+
'rule(/\r\n|[\n\v\f\r\u0085\u2028\u2029]/)'
|
|
422
|
+
end
|
|
423
|
+
raise unsupported("\\R", "for a standalone line-break token, use #{example}; keep the action")
|
|
424
|
+
when "K"
|
|
425
|
+
raise unsupported("\\K",
|
|
426
|
+
'for /a\Kb/ in :initial, use rule(/a/, followed_by: /b/) { begin_state :suffix }; state(:suffix) { rule(/b/) { begin_state :initial; emit :B } }; keep the action on the suffix rule after returning to :initial; choose an unused state name')
|
|
427
|
+
when "g"
|
|
428
|
+
raise unsupported("\\g",
|
|
429
|
+
'for this non-recursive call, rewrite rule(/(a|b)\g<1>/) as rule(/[ab]{2}/); keep the action; recursive calls need a parser or state stack')
|
|
430
|
+
when "X"
|
|
431
|
+
raise unsupported("\\X",
|
|
432
|
+
'for a standalone grapheme token with ASCII-only input, rewrite rule(/\X/) as rule(/\r\n|[\x00-\x7f]/); keep the action; Unicode graphemes need separate segmentation')
|
|
433
|
+
when "c", "C"
|
|
434
|
+
raise unsupported("\\#{char}",
|
|
435
|
+
'for control-A, rewrite rule(/\cA/) or rule(/\C-A/) as rule(/\x01/); keep the action; other control characters need their own hexadecimal code')
|
|
436
|
+
when "M"
|
|
437
|
+
raise unsupported("\\M",
|
|
438
|
+
'for binary input, rewrite rule(/\M-a/n) as rule(/\xE1/n); keep the action; other meta characters need their own byte value')
|
|
439
|
+
when "1".."9", "k"
|
|
440
|
+
read_until(">") if char == "k" && consume?("<")
|
|
441
|
+
feature = char == "k" ? "backreferences" : "\\#{char}"
|
|
442
|
+
help = if @class_depth.positive? && char != "k"
|
|
443
|
+
'for a character code, use an explicit hexadecimal escape such as rule(/[\x01]/); keep the action'
|
|
444
|
+
else
|
|
445
|
+
'for a capture limited to a or b, rewrite rule(/(a|b)\1/) as rule(/(?:aa|bb)/) and keep the action; otherwise split the rule into states and compare matched text in actions'
|
|
446
|
+
end
|
|
447
|
+
raise unsupported(feature, help)
|
|
448
|
+
when *LITERAL_ESCAPES
|
|
308
449
|
@escaped_value = true
|
|
309
450
|
@last_codepoint = char.ord
|
|
451
|
+
else
|
|
452
|
+
raise unsupported("unknown escape \\#{char}",
|
|
453
|
+
'if q is literal, rewrite rule(/\q/) as rule(/q/); otherwise use a supported escape; keep the action')
|
|
310
454
|
end
|
|
311
455
|
end
|
|
312
456
|
|
|
313
457
|
def shorthand_ranges(char)
|
|
314
458
|
if @unicode && @encoding != Encoding::BINARY
|
|
315
459
|
property = { "d" => "Nd", "w" => "Word", "s" => "Space" }.fetch(char.downcase, nil)
|
|
316
|
-
return [[AST::Property, char == char.upcase, property]] if property
|
|
460
|
+
return [[AST::Property, char == char.upcase, property, ignorecase?]] if property
|
|
317
461
|
end
|
|
318
462
|
|
|
319
463
|
base = {
|
|
320
464
|
"d" => [[48, 57]],
|
|
321
465
|
"w" => [[48, 57], [65, 90], [95, 95], [97, 122]],
|
|
322
466
|
"s" => [[9, 13], [32, 32]],
|
|
323
|
-
"h" => [[
|
|
467
|
+
"h" => [[48, 57], [65, 70], [97, 102]]
|
|
324
468
|
}.fetch(char.downcase)
|
|
325
469
|
return base unless char == char.upcase
|
|
326
470
|
|
|
@@ -339,10 +483,11 @@ module Flexr
|
|
|
339
483
|
end
|
|
340
484
|
|
|
341
485
|
def parse_anchor
|
|
486
|
+
starting = @index
|
|
342
487
|
char = advance
|
|
343
|
-
return AST::Anchor.new(kind: :bol, loc:
|
|
488
|
+
return AST::Anchor.new(kind: :bol, loc: span(starting)) if char == "^"
|
|
344
489
|
|
|
345
|
-
AST::Anchor.new(kind: :eol, loc:
|
|
490
|
+
AST::Anchor.new(kind: :eol, loc: span(starting))
|
|
346
491
|
end
|
|
347
492
|
|
|
348
493
|
def validate_anchor_positions(node)
|
|
@@ -375,7 +520,7 @@ module Flexr
|
|
|
375
520
|
|
|
376
521
|
def anchor_nodes(node)
|
|
377
522
|
return [node] if node.is_a?(AST::Anchor)
|
|
378
|
-
return anchor_nodes(node.child) if node.is_a?(AST::Star)
|
|
523
|
+
return anchor_nodes(node.child) if node.is_a?(AST::Star) || node.is_a?(AST::Repeat)
|
|
379
524
|
return [] unless node.respond_to?(:children)
|
|
380
525
|
|
|
381
526
|
node.children.flat_map { |child| anchor_nodes(child) }
|
|
@@ -383,7 +528,7 @@ module Flexr
|
|
|
383
528
|
|
|
384
529
|
def anchor_nested_in_alternative?(node)
|
|
385
530
|
return node.children.any? { |child| anchor_nodes(child).any? } if node.is_a?(AST::Alt)
|
|
386
|
-
return anchor_nested_in_alternative?(node.child) if node.is_a?(AST::Star)
|
|
531
|
+
return anchor_nested_in_alternative?(node.child) if node.is_a?(AST::Star) || node.is_a?(AST::Repeat)
|
|
387
532
|
return false unless node.respond_to?(:children)
|
|
388
533
|
|
|
389
534
|
node.children.any? { |child| anchor_nested_in_alternative?(child) }
|
|
@@ -397,7 +542,8 @@ module Flexr
|
|
|
397
542
|
def reject_postfix_quantifier
|
|
398
543
|
return unless ["?", "+"].include?(current)
|
|
399
544
|
|
|
400
|
-
raise unsupported("lazy or possessive quantifier",
|
|
545
|
+
raise unsupported("lazy or possessive quantifier",
|
|
546
|
+
'for unescaped single-line quoted text, use a negated character class: rule(/"[^"\n]*"/); keep the action; removing lazy or possessive modifiers can change matches')
|
|
401
547
|
end
|
|
402
548
|
|
|
403
549
|
def merge_ranges(ranges)
|
|
@@ -503,6 +649,37 @@ module Flexr
|
|
|
503
649
|
@last_codepoint = codepoint
|
|
504
650
|
end
|
|
505
651
|
|
|
652
|
+
def assign_hex_byte(digits)
|
|
653
|
+
byte = digits.to_i(16)
|
|
654
|
+
if @encoding == Encoding::BINARY
|
|
655
|
+
@escaped_byte = true
|
|
656
|
+
@last_byte = byte
|
|
657
|
+
return
|
|
658
|
+
end
|
|
659
|
+
|
|
660
|
+
length = case byte
|
|
661
|
+
when 0x00..0x7f then 1
|
|
662
|
+
when 0xc2..0xdf then 2
|
|
663
|
+
when 0xe0..0xef then 3
|
|
664
|
+
when 0xf0..0xf4 then 4
|
|
665
|
+
else 0
|
|
666
|
+
end
|
|
667
|
+
bytes = [byte]
|
|
668
|
+
(length - 1).times do
|
|
669
|
+
expect("\\x")
|
|
670
|
+
bytes << read_exact(2).to_i(16)
|
|
671
|
+
end
|
|
672
|
+
value = bytes.pack("C*").force_encoding(Encoding::UTF_8)
|
|
673
|
+
raise_syntax("invalid UTF-8 byte escape") unless length.positive? && value.valid_encoding?
|
|
674
|
+
|
|
675
|
+
@escaped_value = true
|
|
676
|
+
@last_codepoint = value.ord
|
|
677
|
+
end
|
|
678
|
+
|
|
679
|
+
def ignorecase?
|
|
680
|
+
@options.anybits?(::Regexp::IGNORECASE)
|
|
681
|
+
end
|
|
682
|
+
|
|
506
683
|
def skip_extended_space
|
|
507
684
|
loop do
|
|
508
685
|
@index += 1 while @source[@index]&.match?(/\s/)
|
|
@@ -512,6 +689,10 @@ module Flexr
|
|
|
512
689
|
@index += 1 while @source[@index] && @source[@index] != "\n"
|
|
513
690
|
end
|
|
514
691
|
end
|
|
692
|
+
|
|
693
|
+
def span(starting, ending = @index)
|
|
694
|
+
@byte_offsets.fetch(starting)...@byte_offsets.fetch(ending)
|
|
695
|
+
end
|
|
515
696
|
end
|
|
516
697
|
end
|
|
517
698
|
end
|