flexr 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/CHANGELOG.md +116 -0
- data/CONTRIBUTING.md +46 -3
- data/README.md +5 -4
- data/Rakefile +241 -72
- data/Steepfile +7 -0
- data/benchmark/baselines/json.json +2 -2
- data/benchmark/golden/calculator_lexer.rb +276 -0
- data/benchmark/golden/graphql_lexer.rb +310 -0
- data/benchmark/golden/json_lexer.rb +292 -0
- data/benchmark/golden/regexp_tokenizer.rb +293 -0
- data/benchmark/golden/ruby_subset_lexer.rb +280 -0
- data/benchmark/golden/sql_lexer.rb +293 -0
- data/benchmark/golden/toml_lexer.rb +292 -0
- data/benchmark/golden/toy_lang_lexer.rb +273 -0
- data/benchmark/golden/with_lrama_lexer.rb +264 -0
- data/benchmark/golden/with_racc_lexer.rb +264 -0
- data/benchmark/lrama_lexer.rb +165 -0
- data/benchmark/matrix.rb +79 -0
- data/benchmark/profile.rb +109 -0
- data/benchmark/run.rb +4 -1
- data/benchmark/state_dispatch.rb +78 -0
- data/docs/RELEASING.md +3 -2
- data/docs/explanation/backends.md +30 -10
- data/docs/explanation/matching-semantics.md +4 -2
- data/docs/explanation/security-model.md +10 -3
- data/docs/explanation/unicode-and-encoding.md +3 -1
- data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
- data/docs/how-to/generate-a-lexer.md +3 -3
- data/docs/how-to/handle-errors.md +4 -1
- data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
- data/docs/how-to/track-token-locations.md +2 -1
- data/docs/how-to/tune-performance.md +15 -3
- data/docs/migration-v2.md +74 -0
- data/docs/perf-log.md +212 -0
- data/docs/reference/README.md +2 -0
- data/docs/reference/actions.md +5 -2
- data/docs/reference/cli.md +71 -5
- data/docs/reference/compatibility.md +10 -4
- data/docs/reference/diagnostics.md +346 -0
- data/docs/reference/dsl.md +30 -9
- data/docs/reference/errors.md +34 -3
- data/docs/reference/generated-artifacts.md +71 -6
- data/docs/reference/public-api.md +37 -7
- data/docs/reference/regexp.md +27 -13
- data/docs/reference/ruby-lsp.md +34 -0
- data/docs/reference/runtime.md +21 -4
- data/docs/reference/testing.md +57 -0
- data/docs/reference/tokens-and-locations.md +14 -4
- data/examples/graphql/README.md +14 -0
- data/examples/graphql/lexer.flexr.rb +29 -0
- data/examples/json/lexer.flexr.rb +6 -6
- data/examples/sql/README.md +16 -0
- data/examples/sql/lexer.flexr.rb +23 -0
- data/examples/toml/README.md +14 -0
- data/examples/toml/lexer.flexr.rb +23 -0
- data/gemfiles/ruby_lsp.gemfile +4 -0
- data/gemfiles/types.gemfile +7 -0
- data/lib/flexr/action_resolver.rb +15 -0
- data/lib/flexr/artifact_writer.rb +61 -0
- data/lib/flexr/automaton/accel.rb +78 -5
- data/lib/flexr/automaton/analysis.rb +202 -7
- data/lib/flexr/automaton/backend_cost_model.rb +40 -0
- data/lib/flexr/automaton/compiler.rb +229 -96
- data/lib/flexr/automaton/dfa.rb +106 -19
- data/lib/flexr/automaton/minimizer.rb +79 -31
- data/lib/flexr/automaton/nfa.rb +45 -13
- data/lib/flexr/automaton/types.rb +14 -0
- data/lib/flexr/cli.rb +128 -22
- data/lib/flexr/codegen/direct.rb +5 -41
- data/lib/flexr/codegen/firstmatch.rb +1 -7
- data/lib/flexr/codegen/table.rb +181 -69
- data/lib/flexr/codegen.rb +8 -0
- data/lib/flexr/configuration.rb +45 -0
- data/lib/flexr/diagnostics.rb +39 -3
- data/lib/flexr/dsl.rb +116 -47
- data/lib/flexr/errors.rb +7 -6
- data/lib/flexr/generated.rb +120 -46
- data/lib/flexr/generator.rb +102 -109
- data/lib/flexr/grammar.rb +41 -0
- data/lib/flexr/importer.rb +4 -40
- data/lib/flexr/ir.rb +2 -2
- data/lib/flexr/migration.rb +31 -0
- data/lib/flexr/minitest.rb +24 -0
- data/lib/flexr/options.rb +9 -5
- data/lib/flexr/rake_task.rb +6 -1
- data/lib/flexr/regexp/ast.rb +5 -2
- data/lib/flexr/regexp/matcher.rb +26 -0
- data/lib/flexr/regexp/normalizer.rb +63 -21
- data/lib/flexr/regexp/parser.rb +241 -60
- data/lib/flexr/regexp/tokenizer.rb +172 -47
- data/lib/flexr/rspec.rb +51 -0
- data/lib/flexr/runtime/buffer.rb +115 -9
- data/lib/flexr/runtime/core.rb +248 -51
- data/lib/flexr/runtime/errors.rb +66 -4
- data/lib/flexr/runtime/interpreter.rb +213 -97
- data/lib/flexr/runtime/location.rb +35 -12
- data/lib/flexr/runtime/record.rb +14 -0
- data/lib/flexr/runtime/token.rb +7 -1
- data/lib/flexr/runtime.rb +75 -0
- data/lib/flexr/source/passthrough.rb +70 -14
- data/lib/flexr/source/prism_reader.rb +217 -51
- data/lib/flexr/source/static_eval.rb +40 -4
- data/lib/flexr/source.rb +6 -0
- data/lib/flexr/testing.rb +47 -0
- data/lib/flexr/trace.rb +197 -0
- data/lib/flexr/unicode/data/properties.rb +1 -1
- data/lib/flexr/unicode/data.rb +11 -0
- data/lib/flexr/unicode/property.rb +4 -8
- data/lib/flexr/unicode/reference_regexp.rb +11 -9
- data/lib/flexr/unicode/version.rb +7 -0
- data/lib/flexr/version.rb +1 -1
- data/lib/flexr.rb +10 -76
- data/lib/ruby_lsp/flexr/addon.rb +114 -0
- data/sig/flexr.rbs +191 -0
- data/site/README.md +54 -3
- data/site/astro.config.mjs +3 -1
- data/site/package.json +12 -5
- data/site/playwright.config.mjs +16 -0
- data/site/pnpm-lock.yaml +218 -88
- data/site/ruby/playground.rb +57 -0
- data/site/scripts/prepare-playground.mjs +35 -0
- data/site/scripts/verify-site.mjs +5 -3
- data/site/src/content/docs/concepts/matching-semantics.md +4 -2
- data/site/src/content/docs/concepts/security-model.md +9 -1
- data/site/src/content/docs/learn/generation.md +1 -1
- data/site/src/content/docs/reference/dsl.md +2 -2
- data/site/src/content/docs/reference/public-api.md +4 -2
- data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
- data/site/src/layouts/SiteLayout.astro +11 -3
- data/site/src/pages/index.astro +69 -155
- data/site/src/pages/playground.astro +40 -38
- data/site/src/playground/client.js +211 -0
- data/site/src/playground/graph-worker.mjs +11 -0
- data/site/src/playground/presets.mjs +33 -0
- data/site/src/playground/runtime.mjs +34 -0
- data/site/src/playground/share.mjs +30 -0
- data/site/src/playground/worker.mjs +29 -0
- data/site/src/styles/custom.css +12 -287
- data/site/src/styles/landing.css +239 -0
- data/site/tests/browser/playground.spec.mjs +99 -0
- data/site/tests/playground-runtime.test.mjs +85 -0
- data/test/ruby_lsp_addon_spec.rb +135 -0
- data/tools/coverage.rb +6 -1
- data/tools/docs_verify.rb +3 -2
- data/tools/golden.rb +54 -0
- data/tools/verification_failure.rb +119 -0
- metadata +66 -12
- data/benchmark/golden/calculator_lexer.sha256 +0 -1
- data/benchmark/golden/json_lexer.sha256 +0 -1
- data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
- data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
- data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
- data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
- data/benchmark/golden/with_racc_lexer.sha256 +0 -1
- data/site/public/playground.js +0 -189
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
raise LoadError, "flexr 2.0 requires Ruby 3.4 or newer" if RUBY_VERSION < "3.4"
|
|
4
|
+
|
|
5
|
+
require "json"
|
|
6
|
+
require "monitor"
|
|
7
|
+
require_relative "version"
|
|
8
|
+
require_relative "errors"
|
|
9
|
+
require_relative "diagnostics"
|
|
10
|
+
require_relative "action_resolver"
|
|
11
|
+
require_relative "configuration"
|
|
12
|
+
require_relative "ir"
|
|
13
|
+
require_relative "unicode/version"
|
|
14
|
+
require_relative "regexp/ast"
|
|
15
|
+
require_relative "regexp/parser"
|
|
16
|
+
require_relative "regexp/normalizer"
|
|
17
|
+
require_relative "regexp/unsupported"
|
|
18
|
+
require_relative "regexp/char_class"
|
|
19
|
+
require_relative "unicode/utf8_splitter"
|
|
20
|
+
require_relative "unicode/data"
|
|
21
|
+
require_relative "unicode/property"
|
|
22
|
+
require_relative "unicode/reference_regexp"
|
|
23
|
+
require_relative "unicode/case_fold"
|
|
24
|
+
require_relative "regexp/matcher"
|
|
25
|
+
require_relative "automaton/types"
|
|
26
|
+
require_relative "automaton/byte_class_set"
|
|
27
|
+
require_relative "automaton/nfa"
|
|
28
|
+
require_relative "automaton/dfa"
|
|
29
|
+
require_relative "automaton/compiler"
|
|
30
|
+
require_relative "automaton/analysis"
|
|
31
|
+
require_relative "automaton/minimizer"
|
|
32
|
+
require_relative "automaton/backend_cost_model"
|
|
33
|
+
require_relative "automaton/accel"
|
|
34
|
+
require_relative "runtime/record"
|
|
35
|
+
require_relative "runtime/location"
|
|
36
|
+
require_relative "runtime/token"
|
|
37
|
+
require_relative "runtime/buffer"
|
|
38
|
+
require_relative "runtime/errors"
|
|
39
|
+
require_relative "runtime/interpreter"
|
|
40
|
+
require_relative "runtime/core"
|
|
41
|
+
require_relative "dsl"
|
|
42
|
+
require_relative "lexer"
|
|
43
|
+
require_relative "generated"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
module Flexr
|
|
47
|
+
class << self
|
|
48
|
+
def compile_pattern(pattern, options: {}, encoding: nil)
|
|
49
|
+
regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(pattern.to_s)
|
|
50
|
+
encoding ||= regexp.encoding == Encoding::BINARY ? Encoding::BINARY : Encoding::UTF_8
|
|
51
|
+
rule = IR::Rule.new(index: 0, patterns: [regexp], action: :skip, states: [:initial])
|
|
52
|
+
state = IR::State.new(name: :initial, inclusive: true, id: 0)
|
|
53
|
+
spec = IR::Spec.new(
|
|
54
|
+
class_name: "Pattern", backend: :table, token_kind: :array,
|
|
55
|
+
encoding: encoding, options: options, declared_tokens: [],
|
|
56
|
+
states: { initial: state }, rules: [rule], eof_rules: {}, verbatim: nil
|
|
57
|
+
)
|
|
58
|
+
Automaton::Compiler.new(spec).compile.machines.fetch(:initial).dfa
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def reference_pattern?(regexp, unicode: false)
|
|
62
|
+
return true if regexp.source.match?(/\\[pP]\{/) || regexp.source.match?(/\[:(?:\^)?[a-z]+:\]/)
|
|
63
|
+
|
|
64
|
+
unicode && regexp.encoding != Encoding::BINARY && regexp.source.match?(/\\[dDwWsS]/)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def parse_pattern(pattern, options: {})
|
|
68
|
+
regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(pattern.to_s)
|
|
69
|
+
encoding = regexp.encoding == Encoding::BINARY ? Encoding::BINARY : Encoding::UTF_8
|
|
70
|
+
ast = Regexp::Parser.new(regexp.source, options: regexp.options, encoding: encoding,
|
|
71
|
+
unicode: options[:unicode] == true).parse
|
|
72
|
+
Regexp::Normalizer.new(ast, encoding: encoding, options: regexp.options).normalize
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
@@ -3,28 +3,84 @@
|
|
|
3
3
|
module Flexr
|
|
4
4
|
module Source
|
|
5
5
|
module Passthrough
|
|
6
|
+
ENCODING_COMMENT = /\A[ \t]*#.*(?:coding|encoding)\s*[:=]\s*[-A-Za-z0-9_]+/i
|
|
7
|
+
FROZEN_COMMENT = /\A[ \t]*#\s*frozen_string_literal\s*:\s*(?:true|false)\b/i
|
|
8
|
+
|
|
6
9
|
module_function
|
|
7
10
|
|
|
8
11
|
def remove_spans(source, spans, insertion:, payload:)
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
12
|
+
rewrite(source, spans.map { |start_offset, end_offset| [start_offset, end_offset, ""] },
|
|
13
|
+
insertion: insertion, payload: payload)
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def rewrite(source, edits, insertion:, payload:)
|
|
17
|
+
result = source.b.dup
|
|
18
|
+
normalized_edits = edits.map do |start_offset, end_offset, replacement|
|
|
19
|
+
start_offset = line_start_offset(source, start_offset) if replacement.empty? && start_offset != insertion
|
|
20
|
+
[start_offset, end_offset, replacement]
|
|
17
21
|
end
|
|
18
|
-
|
|
19
|
-
|
|
22
|
+
validate_non_overlapping!(normalized_edits)
|
|
23
|
+
adjusted_insertion = insertion + normalized_edits.sum do |start_offset, end_offset, replacement|
|
|
24
|
+
next 0 if start_offset >= insertion
|
|
25
|
+
|
|
26
|
+
replaced_size = [end_offset, insertion].min - start_offset
|
|
27
|
+
replacement.bytesize - replaced_size
|
|
28
|
+
end
|
|
29
|
+
normalized_edits.sort_by(&:first).reverse_each do |start_offset, end_offset, replacement|
|
|
30
|
+
result[start_offset...end_offset] = replacement.b
|
|
20
31
|
end
|
|
21
|
-
result.insert(
|
|
22
|
-
result
|
|
32
|
+
result.insert(adjusted_insertion, payload.b)
|
|
33
|
+
result.force_encoding(source.encoding)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def insert_after_preamble(source, payload)
|
|
37
|
+
offset = 0
|
|
38
|
+
line, next_offset = next_line(source, offset)
|
|
39
|
+
if line&.start_with?("#!")
|
|
40
|
+
offset = next_offset
|
|
41
|
+
line, next_offset = next_line(source, offset)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
while line && magic_comment?(line)
|
|
45
|
+
offset = next_offset
|
|
46
|
+
line, next_offset = next_line(source, offset)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
source.b.dup.insert(offset, payload.b).force_encoding(source.encoding)
|
|
23
50
|
end
|
|
24
51
|
|
|
25
52
|
def indentation(source, offset)
|
|
26
|
-
|
|
27
|
-
|
|
53
|
+
binary = source.b
|
|
54
|
+
line_start = binary.rindex("\n", offset - 1)
|
|
55
|
+
binary.byteslice((line_start ? line_start + 1 : 0)...offset).to_s[/\A[ \t]*/].to_s
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def line_start_offset(source, offset)
|
|
59
|
+
binary = source.b
|
|
60
|
+
line_start = binary.rindex("\n", offset - 1) unless offset.zero?
|
|
61
|
+
line_start = line_start ? line_start + 1 : 0
|
|
62
|
+
prefix = binary.byteslice(line_start...offset)
|
|
63
|
+
prefix&.match?(/\A[ \t]*\z/) ? line_start : offset
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def next_line(source, offset)
|
|
67
|
+
return [nil, offset] if offset >= source.bytesize
|
|
68
|
+
|
|
69
|
+
binary = source.b
|
|
70
|
+
ending = binary.index("\n", offset)
|
|
71
|
+
ending ? [binary.byteslice(offset..ending), ending + 1] : [binary.byteslice(offset..), source.bytesize]
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def magic_comment?(line)
|
|
75
|
+
ENCODING_COMMENT.match?(line) || FROZEN_COMMENT.match?(line)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def validate_non_overlapping!(edits)
|
|
79
|
+
edits.sort_by(&:first).each_cons(2) do |left, right|
|
|
80
|
+
next if left[1] <= right[0]
|
|
81
|
+
|
|
82
|
+
raise ArgumentError, "overlapping source edits"
|
|
83
|
+
end
|
|
28
84
|
end
|
|
29
85
|
end
|
|
30
86
|
end
|
|
@@ -4,12 +4,12 @@ module Flexr
|
|
|
4
4
|
module Source
|
|
5
5
|
RuleDefinition = Struct.new(
|
|
6
6
|
:index, :patterns, :trailing, :action_source, :action, :states, :bol_only, :end_anchor,
|
|
7
|
-
:pattern_conditions,
|
|
7
|
+
:pattern_conditions, :bind_action, :reads_locals,
|
|
8
8
|
:span, keyword_init: true
|
|
9
9
|
)
|
|
10
10
|
SpecSource = Struct.new(
|
|
11
11
|
:source, :path, :class_name, :rules, :config, :dsl_spans, :first_dsl_offset, :constants,
|
|
12
|
-
:states, keyword_init: true
|
|
12
|
+
:states, :flexr_require_spans, :dsl_edits, keyword_init: true
|
|
13
13
|
)
|
|
14
14
|
|
|
15
15
|
class PrismReader
|
|
@@ -19,9 +19,11 @@ module Flexr
|
|
|
19
19
|
@source = source
|
|
20
20
|
@path = path
|
|
21
21
|
@constants = {}
|
|
22
|
-
@constant_definitions =
|
|
22
|
+
@constant_definitions = []
|
|
23
|
+
@resolved_constant_definitions = {}
|
|
23
24
|
@rules = []
|
|
24
25
|
@spans = []
|
|
26
|
+
@edits = []
|
|
25
27
|
@states = { initial: { inclusive: true } }
|
|
26
28
|
@eof_rules = {}
|
|
27
29
|
@config = { backend: :table, token_kind: :array, encoding: Encoding::UTF_8, declared_tokens: [] }
|
|
@@ -33,34 +35,42 @@ module Flexr
|
|
|
33
35
|
require "prism"
|
|
34
36
|
Prism.parse(@source)
|
|
35
37
|
rescue LoadError => e
|
|
36
|
-
diagnostic = Diagnostics.error("FLEXR-E019", "Prism is required for source generation",
|
|
38
|
+
diagnostic = Diagnostics.error("FLEXR-E019", "Prism is required for source generation",
|
|
39
|
+
location: [0, 0], help: "install the generator dependency with gem install prism",
|
|
40
|
+
note: e.message)
|
|
37
41
|
raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
|
|
38
42
|
end
|
|
39
43
|
unless prism.errors.empty?
|
|
40
44
|
error = prism.errors.first
|
|
41
|
-
diagnostic = Diagnostics.error("FLEXR-E010", error.message
|
|
45
|
+
diagnostic = Diagnostics.error("FLEXR-E010", error.message,
|
|
46
|
+
location: [error.location.start_offset, error.location.end_offset])
|
|
42
47
|
raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
|
|
43
48
|
end
|
|
44
49
|
|
|
45
50
|
klass, class_name = find_lexer_class(prism.value)
|
|
46
51
|
raise CompileError, "no class inheriting from Flexr::Lexer found" unless klass
|
|
47
52
|
|
|
53
|
+
@states[:initial][:location] = [klass.location.start_offset, klass.location.end_offset]
|
|
48
54
|
@lexer_scope = class_name.to_s.delete_prefix("::").split("::").reject(&:empty?)
|
|
49
55
|
collect_constants(prism.value)
|
|
50
|
-
resolve_constants
|
|
51
56
|
collect_body(klass.body, [:initial])
|
|
57
|
+
require_spans = flexr_require_spans(prism.value)
|
|
52
58
|
first_dsl_offset = @spans.map(&:first).min || class_body_offset(klass)
|
|
53
59
|
@config[:eof_rules] = @eof_rules
|
|
54
60
|
SpecSource.new(source: @source, path: @path, class_name: class_name, rules: @rules,
|
|
55
61
|
config: @config, dsl_spans: @spans.uniq(&:first),
|
|
56
|
-
first_dsl_offset: first_dsl_offset, constants: @constants, states: @states
|
|
62
|
+
first_dsl_offset: first_dsl_offset, constants: @constants, states: @states,
|
|
63
|
+
flexr_require_spans: require_spans, dsl_edits: @edits)
|
|
57
64
|
end
|
|
58
65
|
|
|
59
66
|
private
|
|
60
67
|
|
|
61
68
|
def find_lexer_class(node, namespace = [])
|
|
62
69
|
candidates = lexer_class_candidates(node, namespace)
|
|
63
|
-
candidates.
|
|
70
|
+
dsl_candidates = candidates.select { |candidate, _name| lexer_has_dsl?(candidate.body) }
|
|
71
|
+
unsupported_static!("multiple lexer classes require an explicit selection", dsl_candidates[1].first) if dsl_candidates.length > 1
|
|
72
|
+
|
|
73
|
+
dsl_candidates.first || candidates.first
|
|
64
74
|
end
|
|
65
75
|
|
|
66
76
|
def lexer_class_candidates(node, namespace = [])
|
|
@@ -99,6 +109,11 @@ module Flexr
|
|
|
99
109
|
return unless node
|
|
100
110
|
|
|
101
111
|
kind = node.class.name.split("::").last
|
|
112
|
+
if kind == "ProgramNode"
|
|
113
|
+
collect_constants(node.statements, scope: scope)
|
|
114
|
+
return
|
|
115
|
+
end
|
|
116
|
+
|
|
102
117
|
if %w[ClassNode ModuleNode].include?(kind)
|
|
103
118
|
nested_scope = scope + namespace_parts(node.constant_path)
|
|
104
119
|
collect_constants(node.body, scope: nested_scope)
|
|
@@ -107,28 +122,30 @@ module Flexr
|
|
|
107
122
|
|
|
108
123
|
if kind.end_with?("ConstantWriteNode")
|
|
109
124
|
name = qualify_constant(scope, node.name)
|
|
110
|
-
@constant_definitions[name
|
|
125
|
+
@constant_definitions << [name, node.value, scope.dup, node.location.start_offset]
|
|
126
|
+
return
|
|
111
127
|
end
|
|
112
128
|
|
|
113
|
-
|
|
129
|
+
if kind == "StatementsNode"
|
|
130
|
+
node.body.each { |child| collect_constants(child, scope: scope) }
|
|
131
|
+
return
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
return unless kind == "CallNode" && node.receiver.nil? && %i[state all_states].include?(node.name.to_sym)
|
|
135
|
+
|
|
136
|
+
collect_constants(node.block&.body, scope: scope)
|
|
114
137
|
end
|
|
115
138
|
|
|
116
|
-
def resolve_constants
|
|
117
|
-
|
|
139
|
+
def resolve_constants(before_offset:)
|
|
140
|
+
@constant_definitions.sort_by(&:last).each do |name, value_node, scope, offset|
|
|
141
|
+
next if offset >= before_offset || @resolved_constant_definitions[offset]
|
|
118
142
|
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
pending.delete(name)
|
|
126
|
-
resolved = true
|
|
127
|
-
rescue StaticResolutionError
|
|
128
|
-
# Keep unresolved definitions out of the table. If a DSL expression
|
|
129
|
-
# refers to one later, StaticEval emits the normal FLEXR-E017.
|
|
130
|
-
end
|
|
131
|
-
break unless resolved
|
|
143
|
+
@resolved_constant_definitions[offset] = true
|
|
144
|
+
value = StaticEval.new(@source, constants: @constants, scope: scope).call(value_node)
|
|
145
|
+
@constants[name] = value
|
|
146
|
+
rescue StaticResolutionError
|
|
147
|
+
# Preserve dynamic constants as ordinary Ruby. A later DSL reference
|
|
148
|
+
# still receives FLEXR-E017 instead of seeing a forward value.
|
|
132
149
|
end
|
|
133
150
|
end
|
|
134
151
|
|
|
@@ -143,6 +160,7 @@ module Flexr
|
|
|
143
160
|
def collect_body(node, states)
|
|
144
161
|
body = node.respond_to?(:body) ? node.body : []
|
|
145
162
|
Array(body).each do |child|
|
|
163
|
+
resolve_constants(before_offset: child.location.start_offset)
|
|
146
164
|
collect_call(child, states)
|
|
147
165
|
end
|
|
148
166
|
end
|
|
@@ -150,51 +168,87 @@ module Flexr
|
|
|
150
168
|
def collect_call(node, states)
|
|
151
169
|
return unless node
|
|
152
170
|
return collect_body(node, states) if node.class.name.end_with?("StatementsNode")
|
|
153
|
-
|
|
154
|
-
|
|
171
|
+
unless node.class.name.end_with?("CallNode") && node.receiver.nil?
|
|
172
|
+
reject_unhandled_dsl!(node)
|
|
173
|
+
return
|
|
174
|
+
end
|
|
155
175
|
|
|
156
176
|
name = node.name.to_sym
|
|
157
|
-
|
|
158
|
-
|
|
177
|
+
unless DSL_NAMES.include?(name)
|
|
178
|
+
reject_unhandled_dsl!(node)
|
|
179
|
+
return
|
|
180
|
+
end
|
|
181
|
+
record_deletion(node) unless %i[rule state all_states on_eof].include?(name)
|
|
159
182
|
case name
|
|
160
183
|
when :rule
|
|
161
184
|
collect_rule(node, states)
|
|
162
185
|
when :state
|
|
163
186
|
collect_state(node, states)
|
|
164
187
|
when :all_states
|
|
165
|
-
|
|
188
|
+
collect_all_states(node, states)
|
|
166
189
|
when :on_eof
|
|
167
190
|
action_source = node.block && source_slice(node.block)
|
|
168
191
|
action = if action_source
|
|
169
192
|
action_source.start_with?("do") ? "proc #{action_source}" : "proc#{action_source}"
|
|
170
193
|
end
|
|
171
|
-
|
|
194
|
+
if action
|
|
195
|
+
state = states.last.to_sym
|
|
196
|
+
@eof_rules[state] = "proc { instance_exec(&self.class.__flexr_generated_eof_actions.fetch(#{state.inspect})) }"
|
|
197
|
+
record_eof_edit(node, state, action)
|
|
198
|
+
else
|
|
199
|
+
record_deletion(node)
|
|
200
|
+
end
|
|
172
201
|
when :emits
|
|
173
202
|
values = positional(node).map { |item| static(item) }.compact.flatten
|
|
174
203
|
@config[:declared_tokens].concat(values.map(&:to_sym))
|
|
175
204
|
when :backend
|
|
176
|
-
@config[:backend] = static(positional(node).first)
|
|
205
|
+
@config[:backend] = Configuration.backend!(static(positional(node).first))
|
|
177
206
|
when :token_kind
|
|
178
|
-
@config[:token_kind] = static(positional(node).first)
|
|
207
|
+
@config[:token_kind] = Configuration.token_kind!(static(positional(node).first))
|
|
179
208
|
when :encoding
|
|
180
|
-
@config[:encoding] = static(positional(node).first)
|
|
209
|
+
@config[:encoding] = Configuration.encoding!(static(positional(node).first))
|
|
181
210
|
when :option
|
|
182
211
|
@config[:options] ||= {}
|
|
183
|
-
positional(node).each { |item| @config[:options][static(item)
|
|
212
|
+
positional(node).each { |item| @config[:options][Configuration.option!(static(item))] = true }
|
|
184
213
|
when :accel
|
|
185
214
|
@config[:options] ||= {}
|
|
186
|
-
@config[:options][:accel] = static(positional(node).first)
|
|
215
|
+
@config[:options][:accel] = Configuration.accelerator!(static(positional(node).first))
|
|
187
216
|
end
|
|
217
|
+
rescue CompileError => e
|
|
218
|
+
e.diagnostic.location ||= [node.location.start_offset, node.location.end_offset] if e.diagnostic
|
|
219
|
+
raise
|
|
188
220
|
end
|
|
189
221
|
|
|
190
222
|
def collect_state(node, parent_states)
|
|
223
|
+
block = node.block
|
|
224
|
+
raise ArgumentError, "state requires a block" unless block
|
|
225
|
+
|
|
191
226
|
args = positional(node)
|
|
192
227
|
names = args.map { |argument| static(argument, scope: @lexer_scope).to_sym }
|
|
193
228
|
names = parent_states if node.name.to_sym == :all_states
|
|
229
|
+
raise ArgumentError, "state requires a name" if names.empty?
|
|
230
|
+
|
|
231
|
+
validate_keywords!(node, %i[inclusive])
|
|
194
232
|
inclusive = keyword(node, :inclusive) ? static(keyword(node, :inclusive), scope: @lexer_scope) : false
|
|
195
|
-
names.each
|
|
233
|
+
names.each do |name|
|
|
234
|
+
existing = @states[name]
|
|
235
|
+
raise ArgumentError, "state #{name.inspect} already declared with inclusive: #{existing[:inclusive]}" if
|
|
236
|
+
existing && existing[:inclusive] != inclusive
|
|
237
|
+
@states[name] ||= { inclusive: inclusive, location: [node.location.start_offset, node.location.end_offset] }
|
|
238
|
+
end
|
|
239
|
+
record_state_wrapper(node, block)
|
|
240
|
+
active_states = parent_states == [:initial] ? names : (parent_states + names).uniq
|
|
241
|
+
collect_body(block.body, active_states)
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
def collect_all_states(node, parent_states)
|
|
196
245
|
block = node.block
|
|
197
|
-
|
|
246
|
+
raise ArgumentError, "all_states requires a block" unless block
|
|
247
|
+
|
|
248
|
+
validate_keywords!(node, [])
|
|
249
|
+
unsupported_static!("all_states does not accept arguments", node) unless positional(node).empty?
|
|
250
|
+
record_state_wrapper(node, block)
|
|
251
|
+
collect_body(block.body, (parent_states + @states.keys).uniq)
|
|
198
252
|
end
|
|
199
253
|
|
|
200
254
|
def collect_rule(node, states)
|
|
@@ -206,36 +260,45 @@ module Flexr
|
|
|
206
260
|
raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
|
|
207
261
|
end
|
|
208
262
|
options = keywords(node)
|
|
263
|
+
validate_keywords!(node, %i[skip emit followed_by])
|
|
209
264
|
action_source = node.block && source_slice(node.block)
|
|
210
|
-
if action_source
|
|
265
|
+
if action_source && reject_action?(node.block)
|
|
211
266
|
diagnostic = Diagnostics.error(
|
|
212
267
|
"FLEXR-E013", "reject is not supported by flexr",
|
|
213
268
|
help: "use a state transition and less(n) to express the fallback"
|
|
214
269
|
)
|
|
215
270
|
raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
|
|
216
271
|
end
|
|
217
|
-
|
|
272
|
+
block_action = if action_source
|
|
218
273
|
action_source.start_with?("do") ? "proc #{action_source}" : "proc#{action_source}"
|
|
219
274
|
end
|
|
275
|
+
skip = options.key?(:skip) ? static(options[:skip], scope: @lexer_scope) : false
|
|
276
|
+
emit = options.key?(:emit) ? static(options[:emit], scope: @lexer_scope) : nil
|
|
277
|
+
action = ActionResolver.resolve(
|
|
278
|
+
skip: skip, emit: emit, block: block_action, default: "proc { emit(nil, text) }"
|
|
279
|
+
)
|
|
280
|
+
bind_action = !block_action.nil? && action.equal?(block_action)
|
|
281
|
+
reads_locals = false
|
|
282
|
+
if node.block
|
|
283
|
+
each_node_including_self(node.block) do |child|
|
|
284
|
+
reads_locals = true if child.is_a?(Prism::LocalVariableReadNode)
|
|
285
|
+
end
|
|
286
|
+
end
|
|
287
|
+
rule_index = @rules.length
|
|
288
|
+
record_rule_edit(node, rule_index, bind_action ? block_action : nil)
|
|
220
289
|
@rules << RuleDefinition.new(
|
|
221
|
-
index:
|
|
290
|
+
index: rule_index, patterns: patterns,
|
|
222
291
|
trailing: options[:followed_by] && static(options[:followed_by], scope: @lexer_scope),
|
|
223
|
-
action_source: action_source, action: action
|
|
292
|
+
action_source: action_source, action: action, states: states.dup,
|
|
224
293
|
bol_only: false, end_anchor: nil,
|
|
225
|
-
span: [node.location.start_offset, node.location.end_offset]
|
|
294
|
+
span: [node.location.start_offset, node.location.end_offset], bind_action: bind_action, reads_locals: reads_locals
|
|
226
295
|
)
|
|
227
296
|
end
|
|
228
297
|
|
|
229
|
-
def action_for(options)
|
|
230
|
-
return :skip if options[:skip] && static(options[:skip])
|
|
231
|
-
return [:emit, static(options[:emit]).to_sym] if options[:emit]
|
|
232
|
-
|
|
233
|
-
"proc { emit(nil, text) }"
|
|
234
|
-
end
|
|
235
|
-
|
|
236
298
|
def static(node, scope: @lexer_scope)
|
|
237
299
|
StaticEval.new(@source, constants: @constants, scope: scope).call(node)
|
|
238
|
-
rescue StaticResolutionError
|
|
300
|
+
rescue StaticResolutionError => e
|
|
301
|
+
e.diagnostic.location ||= [node.location.start_offset, node.location.end_offset] if e.diagnostic && node
|
|
239
302
|
raise unless @allow_dynamic
|
|
240
303
|
|
|
241
304
|
nil
|
|
@@ -254,6 +317,85 @@ module Flexr
|
|
|
254
317
|
keywords(node)[key]
|
|
255
318
|
end
|
|
256
319
|
|
|
320
|
+
def validate_keywords!(node, allowed)
|
|
321
|
+
unknown = keywords(node).keys - allowed
|
|
322
|
+
unsupported_static!("unknown keyword #{unknown.first.inspect} for #{node.name}", node) unless unknown.empty?
|
|
323
|
+
end
|
|
324
|
+
|
|
325
|
+
def record_deletion(node)
|
|
326
|
+
span = [node.location.start_offset, node.location.end_offset]
|
|
327
|
+
@spans << span
|
|
328
|
+
@edits << [*span, ""]
|
|
329
|
+
end
|
|
330
|
+
|
|
331
|
+
def record_rule_edit(node, rule_index, block_action)
|
|
332
|
+
span = [node.location.start_offset, node.location.end_offset]
|
|
333
|
+
replacement = block_action ? "__flexr_bind_generated_action(#{rule_index}, #{block_action})" : ""
|
|
334
|
+
@spans << span
|
|
335
|
+
@edits << [*span, replacement]
|
|
336
|
+
end
|
|
337
|
+
|
|
338
|
+
def record_eof_edit(node, state, action)
|
|
339
|
+
span = [node.location.start_offset, node.location.end_offset]
|
|
340
|
+
@spans << span
|
|
341
|
+
@edits << [*span, "__flexr_bind_generated_eof(#{state.inspect}, #{action})"]
|
|
342
|
+
end
|
|
343
|
+
|
|
344
|
+
def reject_action?(block)
|
|
345
|
+
found = false
|
|
346
|
+
each_node_including_self(block.body) do |child|
|
|
347
|
+
next unless child.class.name.end_with?("CallNode") && child.name.to_sym == :reject
|
|
348
|
+
|
|
349
|
+
found ||= child.receiver.nil? || child.receiver.class.name.end_with?("SelfNode")
|
|
350
|
+
end
|
|
351
|
+
found
|
|
352
|
+
end
|
|
353
|
+
|
|
354
|
+
def record_state_wrapper(node, block)
|
|
355
|
+
body = block.body
|
|
356
|
+
return record_deletion(node) unless body
|
|
357
|
+
|
|
358
|
+
body_start = Source::Passthrough.line_start_offset(@source, body.location.start_offset)
|
|
359
|
+
opening = [node.location.start_offset, body_start]
|
|
360
|
+
closing = [body.location.end_offset, node.location.end_offset]
|
|
361
|
+
opening_source = @source.byteslice(block.location.start_offset...body_start)
|
|
362
|
+
closing_source = @source.byteslice(body.location.end_offset...block.location.end_offset)
|
|
363
|
+
@spans.push(opening, closing)
|
|
364
|
+
@edits << [*opening, "(::Kernel.proc #{opening_source}"]
|
|
365
|
+
@edits << [*closing, "#{closing_source}).call"]
|
|
366
|
+
end
|
|
367
|
+
|
|
368
|
+
def reject_unhandled_dsl!(node)
|
|
369
|
+
each_node_including_self(node) do |child|
|
|
370
|
+
next unless child.class.name.end_with?("CallNode")
|
|
371
|
+
if %i[send public_send].include?(child.name.to_sym)
|
|
372
|
+
dynamic_name = dynamically_dispatched_dsl(child)
|
|
373
|
+
unsupported_static!("dynamic dispatch of #{dynamic_name} is not statically supported", child) if dynamic_name
|
|
374
|
+
end
|
|
375
|
+
next unless DSL_NAMES.include?(child.name.to_sym)
|
|
376
|
+
next unless child.receiver.nil? || child.receiver.class.name.end_with?("SelfNode")
|
|
377
|
+
|
|
378
|
+
unsupported_static!("#{child.name} must be a direct receiverless statement in a lexer or state body", child)
|
|
379
|
+
end
|
|
380
|
+
end
|
|
381
|
+
|
|
382
|
+
def dynamically_dispatched_dsl(node)
|
|
383
|
+
first_argument = Array(node.arguments&.arguments).first
|
|
384
|
+
return unless first_argument && first_argument.class.name.end_with?("SymbolNode")
|
|
385
|
+
|
|
386
|
+
name = first_argument.unescaped.to_sym
|
|
387
|
+
name if DSL_NAMES.include?(name)
|
|
388
|
+
end
|
|
389
|
+
|
|
390
|
+
def unsupported_static!(reason, node)
|
|
391
|
+
diagnostic = Diagnostics.error(
|
|
392
|
+
"FLEXR-E017", "DSL structure is not statically supported",
|
|
393
|
+
location: [node.location.start_offset, node.location.end_offset],
|
|
394
|
+
help: "move the DSL call to the lexer or state body, or use --eval", note: reason
|
|
395
|
+
)
|
|
396
|
+
raise StaticResolutionError.new(diagnostic.message, diagnostic: diagnostic)
|
|
397
|
+
end
|
|
398
|
+
|
|
257
399
|
def each_node(node, &block)
|
|
258
400
|
return unless node
|
|
259
401
|
|
|
@@ -263,6 +405,30 @@ module Flexr
|
|
|
263
405
|
end
|
|
264
406
|
end
|
|
265
407
|
|
|
408
|
+
def flexr_require_spans(node)
|
|
409
|
+
spans = []
|
|
410
|
+
each_node_including_self(node) do |child|
|
|
411
|
+
next unless child.class.name.end_with?("CallNode")
|
|
412
|
+
next unless child.receiver.nil? && child.name.to_sym == :require
|
|
413
|
+
|
|
414
|
+
arguments = Array(child.arguments&.arguments)
|
|
415
|
+
next unless arguments.one?
|
|
416
|
+
|
|
417
|
+
argument = arguments.first
|
|
418
|
+
next unless argument.class.name.end_with?("StringNode") && argument.unescaped == "flexr"
|
|
419
|
+
|
|
420
|
+
spans << [child.location.start_offset, child.location.end_offset]
|
|
421
|
+
end
|
|
422
|
+
spans
|
|
423
|
+
end
|
|
424
|
+
|
|
425
|
+
def each_node_including_self(node, &block)
|
|
426
|
+
return unless node
|
|
427
|
+
|
|
428
|
+
block.call(node)
|
|
429
|
+
node.child_nodes.compact.each { |child| each_node_including_self(child, &block) }
|
|
430
|
+
end
|
|
431
|
+
|
|
266
432
|
def source_slice(node)
|
|
267
433
|
return "" unless node
|
|
268
434
|
|
|
@@ -45,7 +45,15 @@ module Flexr
|
|
|
45
45
|
when "NilNode"
|
|
46
46
|
nil
|
|
47
47
|
when "ArrayNode"
|
|
48
|
-
node.elements.
|
|
48
|
+
node.elements.flat_map do |element|
|
|
49
|
+
if element.class.name.end_with?("SplatNode")
|
|
50
|
+
Array(evaluate(element.expression))
|
|
51
|
+
else
|
|
52
|
+
[evaluate(element)]
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
when "HashNode"
|
|
56
|
+
evaluate_hash(node)
|
|
49
57
|
when "ConstantReadNode"
|
|
50
58
|
evaluate_constant([node.name.to_s], absolute: false)
|
|
51
59
|
when "ConstantPathNode"
|
|
@@ -66,8 +74,10 @@ module Flexr
|
|
|
66
74
|
kind = part.class.name.split("::").last
|
|
67
75
|
if kind == "EmbeddedStatementsNode"
|
|
68
76
|
body = part.statements&.body || []
|
|
69
|
-
|
|
70
|
-
|
|
77
|
+
raise_resolution("interpolation must contain exactly one expression") unless body.one?
|
|
78
|
+
|
|
79
|
+
value = evaluate(body.first)
|
|
80
|
+
value.is_a?(::Regexp) ? regexp_fragment(value) : value.to_s
|
|
71
81
|
else
|
|
72
82
|
evaluate(part).to_s
|
|
73
83
|
end
|
|
@@ -95,7 +105,7 @@ module Flexr
|
|
|
95
105
|
return value unless value.equal?(MISSING)
|
|
96
106
|
end
|
|
97
107
|
|
|
98
|
-
return ::Encoding.const_get(parts.last) if
|
|
108
|
+
return ::Encoding.const_get(parts.last) if parts.length == 2 && parts.first == "Encoding"
|
|
99
109
|
|
|
100
110
|
raise_resolution("constant #{parts.join('::')} is not statically known")
|
|
101
111
|
rescue NameError
|
|
@@ -122,6 +132,32 @@ module Flexr
|
|
|
122
132
|
raise_resolution("method call #{node.name} is not statically supported")
|
|
123
133
|
end
|
|
124
134
|
|
|
135
|
+
def evaluate_hash(node)
|
|
136
|
+
node.elements.each_with_object({}) do |element, result|
|
|
137
|
+
kind = element.class.name.split("::").last
|
|
138
|
+
if kind == "AssocNode"
|
|
139
|
+
result[evaluate(element.key)] = evaluate(element.value)
|
|
140
|
+
elsif kind == "AssocSplatNode"
|
|
141
|
+
value = evaluate(element.value)
|
|
142
|
+
raise_resolution("hash splat value must be a Hash") unless value.is_a?(Hash)
|
|
143
|
+
|
|
144
|
+
result.merge!(value)
|
|
145
|
+
else
|
|
146
|
+
raise_resolution("hash element #{kind} is not statically supported")
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def regexp_fragment(regexp)
|
|
152
|
+
flags = {
|
|
153
|
+
"i" => ::Regexp::IGNORECASE,
|
|
154
|
+
"m" => ::Regexp::MULTILINE,
|
|
155
|
+
"x" => ::Regexp::EXTENDED
|
|
156
|
+
}
|
|
157
|
+
enabled, disabled = flags.partition { |_name, bit| regexp.options.anybits?(bit) }
|
|
158
|
+
"(?#{enabled.to_h.keys.join}-#{disabled.to_h.keys.join}:#{regexp.source})"
|
|
159
|
+
end
|
|
160
|
+
|
|
125
161
|
def positional_arguments(node)
|
|
126
162
|
Array(node.arguments&.arguments).reject { |argument| argument.class.name.end_with?("KeywordHashNode") }
|
|
127
163
|
end
|