flexr 1.0.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +241 -72
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/RELEASING.md +3 -2
  25. data/docs/explanation/backends.md +30 -10
  26. data/docs/explanation/matching-semantics.md +4 -2
  27. data/docs/explanation/security-model.md +10 -3
  28. data/docs/explanation/unicode-and-encoding.md +3 -1
  29. data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
  30. data/docs/how-to/generate-a-lexer.md +3 -3
  31. data/docs/how-to/handle-errors.md +4 -1
  32. data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
  33. data/docs/how-to/track-token-locations.md +2 -1
  34. data/docs/how-to/tune-performance.md +15 -3
  35. data/docs/migration-v2.md +74 -0
  36. data/docs/perf-log.md +212 -0
  37. data/docs/reference/README.md +2 -0
  38. data/docs/reference/actions.md +5 -2
  39. data/docs/reference/cli.md +71 -5
  40. data/docs/reference/compatibility.md +10 -4
  41. data/docs/reference/diagnostics.md +346 -0
  42. data/docs/reference/dsl.md +30 -9
  43. data/docs/reference/errors.md +34 -3
  44. data/docs/reference/generated-artifacts.md +71 -6
  45. data/docs/reference/public-api.md +37 -7
  46. data/docs/reference/regexp.md +27 -13
  47. data/docs/reference/ruby-lsp.md +34 -0
  48. data/docs/reference/runtime.md +21 -4
  49. data/docs/reference/testing.md +57 -0
  50. data/docs/reference/tokens-and-locations.md +14 -4
  51. data/examples/graphql/README.md +14 -0
  52. data/examples/graphql/lexer.flexr.rb +29 -0
  53. data/examples/json/lexer.flexr.rb +6 -6
  54. data/examples/sql/README.md +16 -0
  55. data/examples/sql/lexer.flexr.rb +23 -0
  56. data/examples/toml/README.md +14 -0
  57. data/examples/toml/lexer.flexr.rb +23 -0
  58. data/gemfiles/ruby_lsp.gemfile +4 -0
  59. data/gemfiles/types.gemfile +7 -0
  60. data/lib/flexr/action_resolver.rb +15 -0
  61. data/lib/flexr/artifact_writer.rb +61 -0
  62. data/lib/flexr/automaton/accel.rb +78 -5
  63. data/lib/flexr/automaton/analysis.rb +202 -7
  64. data/lib/flexr/automaton/backend_cost_model.rb +40 -0
  65. data/lib/flexr/automaton/compiler.rb +229 -96
  66. data/lib/flexr/automaton/dfa.rb +106 -19
  67. data/lib/flexr/automaton/minimizer.rb +79 -31
  68. data/lib/flexr/automaton/nfa.rb +45 -13
  69. data/lib/flexr/automaton/types.rb +14 -0
  70. data/lib/flexr/cli.rb +128 -22
  71. data/lib/flexr/codegen/direct.rb +5 -41
  72. data/lib/flexr/codegen/firstmatch.rb +1 -7
  73. data/lib/flexr/codegen/table.rb +181 -69
  74. data/lib/flexr/codegen.rb +8 -0
  75. data/lib/flexr/configuration.rb +45 -0
  76. data/lib/flexr/diagnostics.rb +39 -3
  77. data/lib/flexr/dsl.rb +116 -47
  78. data/lib/flexr/errors.rb +7 -6
  79. data/lib/flexr/generated.rb +120 -46
  80. data/lib/flexr/generator.rb +102 -109
  81. data/lib/flexr/grammar.rb +41 -0
  82. data/lib/flexr/importer.rb +4 -40
  83. data/lib/flexr/ir.rb +2 -2
  84. data/lib/flexr/migration.rb +31 -0
  85. data/lib/flexr/minitest.rb +24 -0
  86. data/lib/flexr/options.rb +9 -5
  87. data/lib/flexr/rake_task.rb +6 -1
  88. data/lib/flexr/regexp/ast.rb +5 -2
  89. data/lib/flexr/regexp/matcher.rb +26 -0
  90. data/lib/flexr/regexp/normalizer.rb +63 -21
  91. data/lib/flexr/regexp/parser.rb +241 -60
  92. data/lib/flexr/regexp/tokenizer.rb +172 -47
  93. data/lib/flexr/rspec.rb +51 -0
  94. data/lib/flexr/runtime/buffer.rb +115 -9
  95. data/lib/flexr/runtime/core.rb +248 -51
  96. data/lib/flexr/runtime/errors.rb +66 -4
  97. data/lib/flexr/runtime/interpreter.rb +213 -97
  98. data/lib/flexr/runtime/location.rb +35 -12
  99. data/lib/flexr/runtime/record.rb +14 -0
  100. data/lib/flexr/runtime/token.rb +7 -1
  101. data/lib/flexr/runtime.rb +75 -0
  102. data/lib/flexr/source/passthrough.rb +70 -14
  103. data/lib/flexr/source/prism_reader.rb +217 -51
  104. data/lib/flexr/source/static_eval.rb +40 -4
  105. data/lib/flexr/source.rb +6 -0
  106. data/lib/flexr/testing.rb +47 -0
  107. data/lib/flexr/trace.rb +197 -0
  108. data/lib/flexr/unicode/data/properties.rb +1 -1
  109. data/lib/flexr/unicode/data.rb +11 -0
  110. data/lib/flexr/unicode/property.rb +4 -8
  111. data/lib/flexr/unicode/reference_regexp.rb +11 -9
  112. data/lib/flexr/unicode/version.rb +7 -0
  113. data/lib/flexr/version.rb +1 -1
  114. data/lib/flexr.rb +10 -76
  115. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  116. data/sig/flexr.rbs +191 -0
  117. data/site/README.md +54 -3
  118. data/site/astro.config.mjs +3 -1
  119. data/site/package.json +12 -5
  120. data/site/playwright.config.mjs +16 -0
  121. data/site/pnpm-lock.yaml +218 -88
  122. data/site/ruby/playground.rb +57 -0
  123. data/site/scripts/prepare-playground.mjs +35 -0
  124. data/site/scripts/verify-site.mjs +5 -3
  125. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  126. data/site/src/content/docs/concepts/security-model.md +9 -1
  127. data/site/src/content/docs/learn/generation.md +1 -1
  128. data/site/src/content/docs/reference/dsl.md +2 -2
  129. data/site/src/content/docs/reference/public-api.md +4 -2
  130. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  131. data/site/src/layouts/SiteLayout.astro +11 -3
  132. data/site/src/pages/index.astro +69 -155
  133. data/site/src/pages/playground.astro +40 -38
  134. data/site/src/playground/client.js +211 -0
  135. data/site/src/playground/graph-worker.mjs +11 -0
  136. data/site/src/playground/presets.mjs +33 -0
  137. data/site/src/playground/runtime.mjs +34 -0
  138. data/site/src/playground/share.mjs +30 -0
  139. data/site/src/playground/worker.mjs +29 -0
  140. data/site/src/styles/custom.css +12 -287
  141. data/site/src/styles/landing.css +239 -0
  142. data/site/tests/browser/playground.spec.mjs +99 -0
  143. data/site/tests/playground-runtime.test.mjs +85 -0
  144. data/test/ruby_lsp_addon_spec.rb +135 -0
  145. data/tools/coverage.rb +6 -1
  146. data/tools/docs_verify.rb +3 -2
  147. data/tools/golden.rb +54 -0
  148. data/tools/verification_failure.rb +119 -0
  149. metadata +66 -12
  150. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  151. data/benchmark/golden/json_lexer.sha256 +0 -1
  152. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  153. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  154. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  155. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  156. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  157. data/site/public/playground.js +0 -189
@@ -0,0 +1,75 @@
1
+ # frozen_string_literal: true
2
+
3
+ raise LoadError, "flexr 2.0 requires Ruby 3.4 or newer" if RUBY_VERSION < "3.4"
4
+
5
+ require "json"
6
+ require "monitor"
7
+ require_relative "version"
8
+ require_relative "errors"
9
+ require_relative "diagnostics"
10
+ require_relative "action_resolver"
11
+ require_relative "configuration"
12
+ require_relative "ir"
13
+ require_relative "unicode/version"
14
+ require_relative "regexp/ast"
15
+ require_relative "regexp/parser"
16
+ require_relative "regexp/normalizer"
17
+ require_relative "regexp/unsupported"
18
+ require_relative "regexp/char_class"
19
+ require_relative "unicode/utf8_splitter"
20
+ require_relative "unicode/data"
21
+ require_relative "unicode/property"
22
+ require_relative "unicode/reference_regexp"
23
+ require_relative "unicode/case_fold"
24
+ require_relative "regexp/matcher"
25
+ require_relative "automaton/types"
26
+ require_relative "automaton/byte_class_set"
27
+ require_relative "automaton/nfa"
28
+ require_relative "automaton/dfa"
29
+ require_relative "automaton/compiler"
30
+ require_relative "automaton/analysis"
31
+ require_relative "automaton/minimizer"
32
+ require_relative "automaton/backend_cost_model"
33
+ require_relative "automaton/accel"
34
+ require_relative "runtime/record"
35
+ require_relative "runtime/location"
36
+ require_relative "runtime/token"
37
+ require_relative "runtime/buffer"
38
+ require_relative "runtime/errors"
39
+ require_relative "runtime/interpreter"
40
+ require_relative "runtime/core"
41
+ require_relative "dsl"
42
+ require_relative "lexer"
43
+ require_relative "generated"
44
+
45
+
46
+ module Flexr
47
+ class << self
48
+ def compile_pattern(pattern, options: {}, encoding: nil)
49
+ regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(pattern.to_s)
50
+ encoding ||= regexp.encoding == Encoding::BINARY ? Encoding::BINARY : Encoding::UTF_8
51
+ rule = IR::Rule.new(index: 0, patterns: [regexp], action: :skip, states: [:initial])
52
+ state = IR::State.new(name: :initial, inclusive: true, id: 0)
53
+ spec = IR::Spec.new(
54
+ class_name: "Pattern", backend: :table, token_kind: :array,
55
+ encoding: encoding, options: options, declared_tokens: [],
56
+ states: { initial: state }, rules: [rule], eof_rules: {}, verbatim: nil
57
+ )
58
+ Automaton::Compiler.new(spec).compile.machines.fetch(:initial).dfa
59
+ end
60
+
61
+ def reference_pattern?(regexp, unicode: false)
62
+ return true if regexp.source.match?(/\\[pP]\{/) || regexp.source.match?(/\[:(?:\^)?[a-z]+:\]/)
63
+
64
+ unicode && regexp.encoding != Encoding::BINARY && regexp.source.match?(/\\[dDwWsS]/)
65
+ end
66
+
67
+ def parse_pattern(pattern, options: {})
68
+ regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(pattern.to_s)
69
+ encoding = regexp.encoding == Encoding::BINARY ? Encoding::BINARY : Encoding::UTF_8
70
+ ast = Regexp::Parser.new(regexp.source, options: regexp.options, encoding: encoding,
71
+ unicode: options[:unicode] == true).parse
72
+ Regexp::Normalizer.new(ast, encoding: encoding, options: regexp.options).normalize
73
+ end
74
+ end
75
+ end
@@ -3,28 +3,84 @@
3
3
  module Flexr
4
4
  module Source
5
5
  module Passthrough
6
+ ENCODING_COMMENT = /\A[ \t]*#.*(?:coding|encoding)\s*[:=]\s*[-A-Za-z0-9_]+/i
7
+ FROZEN_COMMENT = /\A[ \t]*#\s*frozen_string_literal\s*:\s*(?:true|false)\b/i
8
+
6
9
  module_function
7
10
 
8
11
  def remove_spans(source, spans, insertion:, payload:)
9
- result = source.dup
10
- top_level = spans.reject do |span|
11
- spans.any? do |outer|
12
- next false if outer.equal?(span)
13
-
14
- outer.first <= span.first && span.last <= outer.last &&
15
- (outer.first < span.first || span.last < outer.last)
16
- end
12
+ rewrite(source, spans.map { |start_offset, end_offset| [start_offset, end_offset, ""] },
13
+ insertion: insertion, payload: payload)
14
+ end
15
+
16
+ def rewrite(source, edits, insertion:, payload:)
17
+ result = source.b.dup
18
+ normalized_edits = edits.map do |start_offset, end_offset, replacement|
19
+ start_offset = line_start_offset(source, start_offset) if replacement.empty? && start_offset != insertion
20
+ [start_offset, end_offset, replacement]
17
21
  end
18
- top_level.sort_by(&:first).reverse_each do |start_offset, end_offset|
19
- result.slice!(start_offset...end_offset)
22
+ validate_non_overlapping!(normalized_edits)
23
+ adjusted_insertion = insertion + normalized_edits.sum do |start_offset, end_offset, replacement|
24
+ next 0 if start_offset >= insertion
25
+
26
+ replaced_size = [end_offset, insertion].min - start_offset
27
+ replacement.bytesize - replaced_size
28
+ end
29
+ normalized_edits.sort_by(&:first).reverse_each do |start_offset, end_offset, replacement|
30
+ result[start_offset...end_offset] = replacement.b
20
31
  end
21
- result.insert(insertion, payload)
22
- result
32
+ result.insert(adjusted_insertion, payload.b)
33
+ result.force_encoding(source.encoding)
34
+ end
35
+
36
+ def insert_after_preamble(source, payload)
37
+ offset = 0
38
+ line, next_offset = next_line(source, offset)
39
+ if line&.start_with?("#!")
40
+ offset = next_offset
41
+ line, next_offset = next_line(source, offset)
42
+ end
43
+
44
+ while line && magic_comment?(line)
45
+ offset = next_offset
46
+ line, next_offset = next_line(source, offset)
47
+ end
48
+
49
+ source.b.dup.insert(offset, payload.b).force_encoding(source.encoding)
23
50
  end
24
51
 
25
52
  def indentation(source, offset)
26
- line_start = source.rindex("\n", offset - 1)
27
- source[(line_start ? line_start + 1 : 0)...offset].to_s[/\A[ \t]*/].to_s
53
+ binary = source.b
54
+ line_start = binary.rindex("\n", offset - 1)
55
+ binary.byteslice((line_start ? line_start + 1 : 0)...offset).to_s[/\A[ \t]*/].to_s
56
+ end
57
+
58
+ def line_start_offset(source, offset)
59
+ binary = source.b
60
+ line_start = binary.rindex("\n", offset - 1) unless offset.zero?
61
+ line_start = line_start ? line_start + 1 : 0
62
+ prefix = binary.byteslice(line_start...offset)
63
+ prefix&.match?(/\A[ \t]*\z/) ? line_start : offset
64
+ end
65
+
66
+ def next_line(source, offset)
67
+ return [nil, offset] if offset >= source.bytesize
68
+
69
+ binary = source.b
70
+ ending = binary.index("\n", offset)
71
+ ending ? [binary.byteslice(offset..ending), ending + 1] : [binary.byteslice(offset..), source.bytesize]
72
+ end
73
+
74
+ def magic_comment?(line)
75
+ ENCODING_COMMENT.match?(line) || FROZEN_COMMENT.match?(line)
76
+ end
77
+
78
+ def validate_non_overlapping!(edits)
79
+ edits.sort_by(&:first).each_cons(2) do |left, right|
80
+ next if left[1] <= right[0]
81
+
82
+ raise ArgumentError, "overlapping source edits"
83
+ end
28
84
  end
29
85
  end
30
86
  end
@@ -4,12 +4,12 @@ module Flexr
4
4
  module Source
5
5
  RuleDefinition = Struct.new(
6
6
  :index, :patterns, :trailing, :action_source, :action, :states, :bol_only, :end_anchor,
7
- :pattern_conditions,
7
+ :pattern_conditions, :bind_action, :reads_locals,
8
8
  :span, keyword_init: true
9
9
  )
10
10
  SpecSource = Struct.new(
11
11
  :source, :path, :class_name, :rules, :config, :dsl_spans, :first_dsl_offset, :constants,
12
- :states, keyword_init: true
12
+ :states, :flexr_require_spans, :dsl_edits, keyword_init: true
13
13
  )
14
14
 
15
15
  class PrismReader
@@ -19,9 +19,11 @@ module Flexr
19
19
  @source = source
20
20
  @path = path
21
21
  @constants = {}
22
- @constant_definitions = {}
22
+ @constant_definitions = []
23
+ @resolved_constant_definitions = {}
23
24
  @rules = []
24
25
  @spans = []
26
+ @edits = []
25
27
  @states = { initial: { inclusive: true } }
26
28
  @eof_rules = {}
27
29
  @config = { backend: :table, token_kind: :array, encoding: Encoding::UTF_8, declared_tokens: [] }
@@ -33,34 +35,42 @@ module Flexr
33
35
  require "prism"
34
36
  Prism.parse(@source)
35
37
  rescue LoadError => e
36
- diagnostic = Diagnostics.error("FLEXR-E019", "Prism is required for source generation", note: e.message)
38
+ diagnostic = Diagnostics.error("FLEXR-E019", "Prism is required for source generation",
39
+ location: [0, 0], help: "install the generator dependency with gem install prism",
40
+ note: e.message)
37
41
  raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
38
42
  end
39
43
  unless prism.errors.empty?
40
44
  error = prism.errors.first
41
- diagnostic = Diagnostics.error("FLEXR-E010", error.message)
45
+ diagnostic = Diagnostics.error("FLEXR-E010", error.message,
46
+ location: [error.location.start_offset, error.location.end_offset])
42
47
  raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
43
48
  end
44
49
 
45
50
  klass, class_name = find_lexer_class(prism.value)
46
51
  raise CompileError, "no class inheriting from Flexr::Lexer found" unless klass
47
52
 
53
+ @states[:initial][:location] = [klass.location.start_offset, klass.location.end_offset]
48
54
  @lexer_scope = class_name.to_s.delete_prefix("::").split("::").reject(&:empty?)
49
55
  collect_constants(prism.value)
50
- resolve_constants
51
56
  collect_body(klass.body, [:initial])
57
+ require_spans = flexr_require_spans(prism.value)
52
58
  first_dsl_offset = @spans.map(&:first).min || class_body_offset(klass)
53
59
  @config[:eof_rules] = @eof_rules
54
60
  SpecSource.new(source: @source, path: @path, class_name: class_name, rules: @rules,
55
61
  config: @config, dsl_spans: @spans.uniq(&:first),
56
- first_dsl_offset: first_dsl_offset, constants: @constants, states: @states)
62
+ first_dsl_offset: first_dsl_offset, constants: @constants, states: @states,
63
+ flexr_require_spans: require_spans, dsl_edits: @edits)
57
64
  end
58
65
 
59
66
  private
60
67
 
61
68
  def find_lexer_class(node, namespace = [])
62
69
  candidates = lexer_class_candidates(node, namespace)
63
- candidates.find { |candidate, _name| lexer_has_dsl?(candidate.body) } || candidates.first
70
+ dsl_candidates = candidates.select { |candidate, _name| lexer_has_dsl?(candidate.body) }
71
+ unsupported_static!("multiple lexer classes require an explicit selection", dsl_candidates[1].first) if dsl_candidates.length > 1
72
+
73
+ dsl_candidates.first || candidates.first
64
74
  end
65
75
 
66
76
  def lexer_class_candidates(node, namespace = [])
@@ -99,6 +109,11 @@ module Flexr
99
109
  return unless node
100
110
 
101
111
  kind = node.class.name.split("::").last
112
+ if kind == "ProgramNode"
113
+ collect_constants(node.statements, scope: scope)
114
+ return
115
+ end
116
+
102
117
  if %w[ClassNode ModuleNode].include?(kind)
103
118
  nested_scope = scope + namespace_parts(node.constant_path)
104
119
  collect_constants(node.body, scope: nested_scope)
@@ -107,28 +122,30 @@ module Flexr
107
122
 
108
123
  if kind.end_with?("ConstantWriteNode")
109
124
  name = qualify_constant(scope, node.name)
110
- @constant_definitions[name] = [node.value, scope.dup]
125
+ @constant_definitions << [name, node.value, scope.dup, node.location.start_offset]
126
+ return
111
127
  end
112
128
 
113
- node.child_nodes.compact.each { |child| collect_constants(child, scope: scope) }
129
+ if kind == "StatementsNode"
130
+ node.body.each { |child| collect_constants(child, scope: scope) }
131
+ return
132
+ end
133
+
134
+ return unless kind == "CallNode" && node.receiver.nil? && %i[state all_states].include?(node.name.to_sym)
135
+
136
+ collect_constants(node.block&.body, scope: scope)
114
137
  end
115
138
 
116
- def resolve_constants
117
- pending = @constant_definitions.dup
139
+ def resolve_constants(before_offset:)
140
+ @constant_definitions.sort_by(&:last).each do |name, value_node, scope, offset|
141
+ next if offset >= before_offset || @resolved_constant_definitions[offset]
118
142
 
119
- loop do
120
- resolved = false
121
- pending.each_key do |name|
122
- value_node, scope = pending.fetch(name)
123
- value = StaticEval.new(@source, constants: @constants, scope: scope).call(value_node)
124
- @constants[name] = value
125
- pending.delete(name)
126
- resolved = true
127
- rescue StaticResolutionError
128
- # Keep unresolved definitions out of the table. If a DSL expression
129
- # refers to one later, StaticEval emits the normal FLEXR-E017.
130
- end
131
- break unless resolved
143
+ @resolved_constant_definitions[offset] = true
144
+ value = StaticEval.new(@source, constants: @constants, scope: scope).call(value_node)
145
+ @constants[name] = value
146
+ rescue StaticResolutionError
147
+ # Preserve dynamic constants as ordinary Ruby. A later DSL reference
148
+ # still receives FLEXR-E017 instead of seeing a forward value.
132
149
  end
133
150
  end
134
151
 
@@ -143,6 +160,7 @@ module Flexr
143
160
  def collect_body(node, states)
144
161
  body = node.respond_to?(:body) ? node.body : []
145
162
  Array(body).each do |child|
163
+ resolve_constants(before_offset: child.location.start_offset)
146
164
  collect_call(child, states)
147
165
  end
148
166
  end
@@ -150,51 +168,87 @@ module Flexr
150
168
  def collect_call(node, states)
151
169
  return unless node
152
170
  return collect_body(node, states) if node.class.name.end_with?("StatementsNode")
153
- return unless node.class.name.end_with?("CallNode")
154
- return if node.receiver
171
+ unless node.class.name.end_with?("CallNode") && node.receiver.nil?
172
+ reject_unhandled_dsl!(node)
173
+ return
174
+ end
155
175
 
156
176
  name = node.name.to_sym
157
- return unless DSL_NAMES.include?(name)
158
- @spans << [node.location.start_offset, node.location.end_offset]
177
+ unless DSL_NAMES.include?(name)
178
+ reject_unhandled_dsl!(node)
179
+ return
180
+ end
181
+ record_deletion(node) unless %i[rule state all_states on_eof].include?(name)
159
182
  case name
160
183
  when :rule
161
184
  collect_rule(node, states)
162
185
  when :state
163
186
  collect_state(node, states)
164
187
  when :all_states
165
- collect_state(node, @states.keys)
188
+ collect_all_states(node, states)
166
189
  when :on_eof
167
190
  action_source = node.block && source_slice(node.block)
168
191
  action = if action_source
169
192
  action_source.start_with?("do") ? "proc #{action_source}" : "proc#{action_source}"
170
193
  end
171
- @eof_rules[states.last.to_sym] = action if action
194
+ if action
195
+ state = states.last.to_sym
196
+ @eof_rules[state] = "proc { instance_exec(&self.class.__flexr_generated_eof_actions.fetch(#{state.inspect})) }"
197
+ record_eof_edit(node, state, action)
198
+ else
199
+ record_deletion(node)
200
+ end
172
201
  when :emits
173
202
  values = positional(node).map { |item| static(item) }.compact.flatten
174
203
  @config[:declared_tokens].concat(values.map(&:to_sym))
175
204
  when :backend
176
- @config[:backend] = static(positional(node).first).to_sym
205
+ @config[:backend] = Configuration.backend!(static(positional(node).first))
177
206
  when :token_kind
178
- @config[:token_kind] = static(positional(node).first).to_sym
207
+ @config[:token_kind] = Configuration.token_kind!(static(positional(node).first))
179
208
  when :encoding
180
- @config[:encoding] = static(positional(node).first)
209
+ @config[:encoding] = Configuration.encoding!(static(positional(node).first))
181
210
  when :option
182
211
  @config[:options] ||= {}
183
- positional(node).each { |item| @config[:options][static(item).to_sym] = true }
212
+ positional(node).each { |item| @config[:options][Configuration.option!(static(item))] = true }
184
213
  when :accel
185
214
  @config[:options] ||= {}
186
- @config[:options][:accel] = static(positional(node).first).to_sym
215
+ @config[:options][:accel] = Configuration.accelerator!(static(positional(node).first))
187
216
  end
217
+ rescue CompileError => e
218
+ e.diagnostic.location ||= [node.location.start_offset, node.location.end_offset] if e.diagnostic
219
+ raise
188
220
  end
189
221
 
190
222
  def collect_state(node, parent_states)
223
+ block = node.block
224
+ raise ArgumentError, "state requires a block" unless block
225
+
191
226
  args = positional(node)
192
227
  names = args.map { |argument| static(argument, scope: @lexer_scope).to_sym }
193
228
  names = parent_states if node.name.to_sym == :all_states
229
+ raise ArgumentError, "state requires a name" if names.empty?
230
+
231
+ validate_keywords!(node, %i[inclusive])
194
232
  inclusive = keyword(node, :inclusive) ? static(keyword(node, :inclusive), scope: @lexer_scope) : false
195
- names.each { |name| @states[name] = { inclusive: inclusive } }
233
+ names.each do |name|
234
+ existing = @states[name]
235
+ raise ArgumentError, "state #{name.inspect} already declared with inclusive: #{existing[:inclusive]}" if
236
+ existing && existing[:inclusive] != inclusive
237
+ @states[name] ||= { inclusive: inclusive, location: [node.location.start_offset, node.location.end_offset] }
238
+ end
239
+ record_state_wrapper(node, block)
240
+ active_states = parent_states == [:initial] ? names : (parent_states + names).uniq
241
+ collect_body(block.body, active_states)
242
+ end
243
+
244
+ def collect_all_states(node, parent_states)
196
245
  block = node.block
197
- collect_body(block&.body, names)
246
+ raise ArgumentError, "all_states requires a block" unless block
247
+
248
+ validate_keywords!(node, [])
249
+ unsupported_static!("all_states does not accept arguments", node) unless positional(node).empty?
250
+ record_state_wrapper(node, block)
251
+ collect_body(block.body, (parent_states + @states.keys).uniq)
198
252
  end
199
253
 
200
254
  def collect_rule(node, states)
@@ -206,36 +260,45 @@ module Flexr
206
260
  raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
207
261
  end
208
262
  options = keywords(node)
263
+ validate_keywords!(node, %i[skip emit followed_by])
209
264
  action_source = node.block && source_slice(node.block)
210
- if action_source&.match?(/\breject\b/)
265
+ if action_source && reject_action?(node.block)
211
266
  diagnostic = Diagnostics.error(
212
267
  "FLEXR-E013", "reject is not supported by flexr",
213
268
  help: "use a state transition and less(n) to express the fallback"
214
269
  )
215
270
  raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
216
271
  end
217
- action = if action_source
272
+ block_action = if action_source
218
273
  action_source.start_with?("do") ? "proc #{action_source}" : "proc#{action_source}"
219
274
  end
275
+ skip = options.key?(:skip) ? static(options[:skip], scope: @lexer_scope) : false
276
+ emit = options.key?(:emit) ? static(options[:emit], scope: @lexer_scope) : nil
277
+ action = ActionResolver.resolve(
278
+ skip: skip, emit: emit, block: block_action, default: "proc { emit(nil, text) }"
279
+ )
280
+ bind_action = !block_action.nil? && action.equal?(block_action)
281
+ reads_locals = false
282
+ if node.block
283
+ each_node_including_self(node.block) do |child|
284
+ reads_locals = true if child.is_a?(Prism::LocalVariableReadNode)
285
+ end
286
+ end
287
+ rule_index = @rules.length
288
+ record_rule_edit(node, rule_index, bind_action ? block_action : nil)
220
289
  @rules << RuleDefinition.new(
221
- index: @rules.length, patterns: patterns,
290
+ index: rule_index, patterns: patterns,
222
291
  trailing: options[:followed_by] && static(options[:followed_by], scope: @lexer_scope),
223
- action_source: action_source, action: action || action_for(options), states: states.dup,
292
+ action_source: action_source, action: action, states: states.dup,
224
293
  bol_only: false, end_anchor: nil,
225
- span: [node.location.start_offset, node.location.end_offset]
294
+ span: [node.location.start_offset, node.location.end_offset], bind_action: bind_action, reads_locals: reads_locals
226
295
  )
227
296
  end
228
297
 
229
- def action_for(options)
230
- return :skip if options[:skip] && static(options[:skip])
231
- return [:emit, static(options[:emit]).to_sym] if options[:emit]
232
-
233
- "proc { emit(nil, text) }"
234
- end
235
-
236
298
  def static(node, scope: @lexer_scope)
237
299
  StaticEval.new(@source, constants: @constants, scope: scope).call(node)
238
- rescue StaticResolutionError
300
+ rescue StaticResolutionError => e
301
+ e.diagnostic.location ||= [node.location.start_offset, node.location.end_offset] if e.diagnostic && node
239
302
  raise unless @allow_dynamic
240
303
 
241
304
  nil
@@ -254,6 +317,85 @@ module Flexr
254
317
  keywords(node)[key]
255
318
  end
256
319
 
320
+ def validate_keywords!(node, allowed)
321
+ unknown = keywords(node).keys - allowed
322
+ unsupported_static!("unknown keyword #{unknown.first.inspect} for #{node.name}", node) unless unknown.empty?
323
+ end
324
+
325
+ def record_deletion(node)
326
+ span = [node.location.start_offset, node.location.end_offset]
327
+ @spans << span
328
+ @edits << [*span, ""]
329
+ end
330
+
331
+ def record_rule_edit(node, rule_index, block_action)
332
+ span = [node.location.start_offset, node.location.end_offset]
333
+ replacement = block_action ? "__flexr_bind_generated_action(#{rule_index}, #{block_action})" : ""
334
+ @spans << span
335
+ @edits << [*span, replacement]
336
+ end
337
+
338
+ def record_eof_edit(node, state, action)
339
+ span = [node.location.start_offset, node.location.end_offset]
340
+ @spans << span
341
+ @edits << [*span, "__flexr_bind_generated_eof(#{state.inspect}, #{action})"]
342
+ end
343
+
344
+ def reject_action?(block)
345
+ found = false
346
+ each_node_including_self(block.body) do |child|
347
+ next unless child.class.name.end_with?("CallNode") && child.name.to_sym == :reject
348
+
349
+ found ||= child.receiver.nil? || child.receiver.class.name.end_with?("SelfNode")
350
+ end
351
+ found
352
+ end
353
+
354
+ def record_state_wrapper(node, block)
355
+ body = block.body
356
+ return record_deletion(node) unless body
357
+
358
+ body_start = Source::Passthrough.line_start_offset(@source, body.location.start_offset)
359
+ opening = [node.location.start_offset, body_start]
360
+ closing = [body.location.end_offset, node.location.end_offset]
361
+ opening_source = @source.byteslice(block.location.start_offset...body_start)
362
+ closing_source = @source.byteslice(body.location.end_offset...block.location.end_offset)
363
+ @spans.push(opening, closing)
364
+ @edits << [*opening, "(::Kernel.proc #{opening_source}"]
365
+ @edits << [*closing, "#{closing_source}).call"]
366
+ end
367
+
368
+ def reject_unhandled_dsl!(node)
369
+ each_node_including_self(node) do |child|
370
+ next unless child.class.name.end_with?("CallNode")
371
+ if %i[send public_send].include?(child.name.to_sym)
372
+ dynamic_name = dynamically_dispatched_dsl(child)
373
+ unsupported_static!("dynamic dispatch of #{dynamic_name} is not statically supported", child) if dynamic_name
374
+ end
375
+ next unless DSL_NAMES.include?(child.name.to_sym)
376
+ next unless child.receiver.nil? || child.receiver.class.name.end_with?("SelfNode")
377
+
378
+ unsupported_static!("#{child.name} must be a direct receiverless statement in a lexer or state body", child)
379
+ end
380
+ end
381
+
382
+ def dynamically_dispatched_dsl(node)
383
+ first_argument = Array(node.arguments&.arguments).first
384
+ return unless first_argument && first_argument.class.name.end_with?("SymbolNode")
385
+
386
+ name = first_argument.unescaped.to_sym
387
+ name if DSL_NAMES.include?(name)
388
+ end
389
+
390
+ def unsupported_static!(reason, node)
391
+ diagnostic = Diagnostics.error(
392
+ "FLEXR-E017", "DSL structure is not statically supported",
393
+ location: [node.location.start_offset, node.location.end_offset],
394
+ help: "move the DSL call to the lexer or state body, or use --eval", note: reason
395
+ )
396
+ raise StaticResolutionError.new(diagnostic.message, diagnostic: diagnostic)
397
+ end
398
+
257
399
  def each_node(node, &block)
258
400
  return unless node
259
401
 
@@ -263,6 +405,30 @@ module Flexr
263
405
  end
264
406
  end
265
407
 
408
+ def flexr_require_spans(node)
409
+ spans = []
410
+ each_node_including_self(node) do |child|
411
+ next unless child.class.name.end_with?("CallNode")
412
+ next unless child.receiver.nil? && child.name.to_sym == :require
413
+
414
+ arguments = Array(child.arguments&.arguments)
415
+ next unless arguments.one?
416
+
417
+ argument = arguments.first
418
+ next unless argument.class.name.end_with?("StringNode") && argument.unescaped == "flexr"
419
+
420
+ spans << [child.location.start_offset, child.location.end_offset]
421
+ end
422
+ spans
423
+ end
424
+
425
+ def each_node_including_self(node, &block)
426
+ return unless node
427
+
428
+ block.call(node)
429
+ node.child_nodes.compact.each { |child| each_node_including_self(child, &block) }
430
+ end
431
+
266
432
  def source_slice(node)
267
433
  return "" unless node
268
434
 
@@ -45,7 +45,15 @@ module Flexr
45
45
  when "NilNode"
46
46
  nil
47
47
  when "ArrayNode"
48
- node.elements.map { |element| evaluate(element) }
48
+ node.elements.flat_map do |element|
49
+ if element.class.name.end_with?("SplatNode")
50
+ Array(evaluate(element.expression))
51
+ else
52
+ [evaluate(element)]
53
+ end
54
+ end
55
+ when "HashNode"
56
+ evaluate_hash(node)
49
57
  when "ConstantReadNode"
50
58
  evaluate_constant([node.name.to_s], absolute: false)
51
59
  when "ConstantPathNode"
@@ -66,8 +74,10 @@ module Flexr
66
74
  kind = part.class.name.split("::").last
67
75
  if kind == "EmbeddedStatementsNode"
68
76
  body = part.statements&.body || []
69
- value = evaluate(body.last)
70
- value.is_a?(::Regexp) ? "(?:#{value.source})" : value.to_s
77
+ raise_resolution("interpolation must contain exactly one expression") unless body.one?
78
+
79
+ value = evaluate(body.first)
80
+ value.is_a?(::Regexp) ? regexp_fragment(value) : value.to_s
71
81
  else
72
82
  evaluate(part).to_s
73
83
  end
@@ -95,7 +105,7 @@ module Flexr
95
105
  return value unless value.equal?(MISSING)
96
106
  end
97
107
 
98
- return ::Encoding.const_get(parts.last) if !absolute && parts.length == 2 && parts.first == "Encoding"
108
+ return ::Encoding.const_get(parts.last) if parts.length == 2 && parts.first == "Encoding"
99
109
 
100
110
  raise_resolution("constant #{parts.join('::')} is not statically known")
101
111
  rescue NameError
@@ -122,6 +132,32 @@ module Flexr
122
132
  raise_resolution("method call #{node.name} is not statically supported")
123
133
  end
124
134
 
135
+ def evaluate_hash(node)
136
+ node.elements.each_with_object({}) do |element, result|
137
+ kind = element.class.name.split("::").last
138
+ if kind == "AssocNode"
139
+ result[evaluate(element.key)] = evaluate(element.value)
140
+ elsif kind == "AssocSplatNode"
141
+ value = evaluate(element.value)
142
+ raise_resolution("hash splat value must be a Hash") unless value.is_a?(Hash)
143
+
144
+ result.merge!(value)
145
+ else
146
+ raise_resolution("hash element #{kind} is not statically supported")
147
+ end
148
+ end
149
+ end
150
+
151
+ def regexp_fragment(regexp)
152
+ flags = {
153
+ "i" => ::Regexp::IGNORECASE,
154
+ "m" => ::Regexp::MULTILINE,
155
+ "x" => ::Regexp::EXTENDED
156
+ }
157
+ enabled, disabled = flags.partition { |_name, bit| regexp.options.anybits?(bit) }
158
+ "(?#{enabled.to_h.keys.join}-#{disabled.to_h.keys.join}:#{regexp.source})"
159
+ end
160
+
125
161
  def positional_arguments(node)
126
162
  Array(node.arguments&.arguments).reject { |argument| argument.class.name.end_with?("KeywordHashNode") }
127
163
  end
@@ -0,0 +1,6 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "runtime" unless defined?(Flexr::Lexer)
4
+ require_relative "source/static_eval"
5
+ require_relative "source/prism_reader"
6
+ require_relative "source/passthrough"