flexr 1.1.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +191 -68
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/README.md +1 -2
  25. data/docs/RELEASING.md +3 -2
  26. data/docs/explanation/backends.md +17 -0
  27. data/docs/how-to/deploy-a-standalone-lexer.md +1 -1
  28. data/docs/how-to/tune-performance.md +12 -0
  29. data/docs/migration-v2.md +74 -0
  30. data/docs/perf-log.md +196 -0
  31. data/docs/reference/README.md +2 -0
  32. data/docs/reference/cli.md +71 -5
  33. data/docs/reference/compatibility.md +10 -4
  34. data/docs/reference/diagnostics.md +338 -0
  35. data/docs/reference/dsl.md +12 -5
  36. data/docs/reference/errors.md +28 -0
  37. data/docs/reference/generated-artifacts.md +42 -4
  38. data/docs/reference/public-api.md +33 -8
  39. data/docs/reference/regexp.md +19 -7
  40. data/docs/reference/ruby-lsp.md +34 -0
  41. data/docs/reference/runtime.md +2 -2
  42. data/docs/reference/testing.md +57 -0
  43. data/docs/reference/tokens-and-locations.md +8 -0
  44. data/examples/graphql/README.md +14 -0
  45. data/examples/graphql/lexer.flexr.rb +29 -0
  46. data/examples/json/lexer.flexr.rb +6 -6
  47. data/examples/sql/README.md +16 -0
  48. data/examples/sql/lexer.flexr.rb +23 -0
  49. data/examples/toml/README.md +14 -0
  50. data/examples/toml/lexer.flexr.rb +23 -0
  51. data/gemfiles/ruby_lsp.gemfile +4 -0
  52. data/gemfiles/types.gemfile +7 -0
  53. data/lib/flexr/automaton/accel.rb +69 -1
  54. data/lib/flexr/automaton/analysis.rb +161 -7
  55. data/lib/flexr/automaton/compiler.rb +179 -25
  56. data/lib/flexr/automaton/types.rb +1 -1
  57. data/lib/flexr/cli.rb +117 -17
  58. data/lib/flexr/codegen/firstmatch.rb +1 -7
  59. data/lib/flexr/codegen/table.rb +8 -2
  60. data/lib/flexr/configuration.rb +8 -0
  61. data/lib/flexr/diagnostics.rb +39 -3
  62. data/lib/flexr/dsl.rb +12 -9
  63. data/lib/flexr/errors.rb +6 -6
  64. data/lib/flexr/generated.rb +65 -17
  65. data/lib/flexr/generator.rb +62 -34
  66. data/lib/flexr/grammar.rb +41 -0
  67. data/lib/flexr/ir.rb +2 -2
  68. data/lib/flexr/migration.rb +31 -0
  69. data/lib/flexr/minitest.rb +24 -0
  70. data/lib/flexr/options.rb +4 -2
  71. data/lib/flexr/regexp/matcher.rb +26 -0
  72. data/lib/flexr/regexp/normalizer.rb +4 -8
  73. data/lib/flexr/regexp/parser.rb +127 -21
  74. data/lib/flexr/regexp/tokenizer.rb +26 -26
  75. data/lib/flexr/rspec.rb +51 -0
  76. data/lib/flexr/runtime/core.rb +47 -17
  77. data/lib/flexr/runtime/errors.rb +6 -4
  78. data/lib/flexr/runtime/interpreter.rb +55 -46
  79. data/lib/flexr/runtime/location.rb +35 -12
  80. data/lib/flexr/runtime/record.rb +14 -0
  81. data/lib/flexr/runtime/token.rb +7 -1
  82. data/lib/flexr/runtime.rb +7 -2
  83. data/lib/flexr/source/passthrough.rb +14 -11
  84. data/lib/flexr/source/prism_reader.rb +53 -16
  85. data/lib/flexr/testing.rb +47 -0
  86. data/lib/flexr/trace.rb +197 -0
  87. data/lib/flexr/unicode/property.rb +1 -3
  88. data/lib/flexr/unicode/reference_regexp.rb +8 -10
  89. data/lib/flexr/version.rb +1 -1
  90. data/lib/flexr.rb +1 -0
  91. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  92. data/sig/flexr.rbs +191 -0
  93. data/site/README.md +54 -3
  94. data/site/astro.config.mjs +3 -1
  95. data/site/package.json +12 -5
  96. data/site/playwright.config.mjs +16 -0
  97. data/site/pnpm-lock.yaml +218 -88
  98. data/site/ruby/playground.rb +57 -0
  99. data/site/scripts/prepare-playground.mjs +35 -0
  100. data/site/scripts/verify-site.mjs +5 -3
  101. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  102. data/site/src/content/docs/concepts/security-model.md +9 -1
  103. data/site/src/content/docs/learn/generation.md +1 -1
  104. data/site/src/content/docs/reference/dsl.md +2 -2
  105. data/site/src/content/docs/reference/public-api.md +4 -2
  106. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  107. data/site/src/layouts/SiteLayout.astro +11 -3
  108. data/site/src/pages/index.astro +69 -155
  109. data/site/src/pages/playground.astro +40 -38
  110. data/site/src/playground/client.js +211 -0
  111. data/site/src/playground/graph-worker.mjs +11 -0
  112. data/site/src/playground/presets.mjs +33 -0
  113. data/site/src/playground/runtime.mjs +34 -0
  114. data/site/src/playground/share.mjs +30 -0
  115. data/site/src/playground/worker.mjs +29 -0
  116. data/site/src/styles/custom.css +12 -287
  117. data/site/src/styles/landing.css +239 -0
  118. data/site/tests/browser/playground.spec.mjs +99 -0
  119. data/site/tests/playground-runtime.test.mjs +85 -0
  120. data/test/ruby_lsp_addon_spec.rb +135 -0
  121. data/tools/docs_verify.rb +3 -2
  122. data/tools/golden.rb +54 -0
  123. data/tools/verification_failure.rb +119 -0
  124. metadata +56 -13
  125. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  126. data/benchmark/golden/json_lexer.sha256 +0 -1
  127. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  128. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  129. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  130. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  131. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  132. data/docs/releases/v1.1.0.md +0 -44
  133. data/site/public/playground.js +0 -189
@@ -0,0 +1,197 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "runtime" unless defined?(Flexr::Lexer)
4
+
5
+ module Flexr
6
+ class Trace
7
+ module Actions
8
+ private
9
+
10
+ def execute(rule)
11
+ @flexr_trace.record(rule) { super }
12
+ end
13
+
14
+ public
15
+
16
+ def instance_exec(*args, &block)
17
+ if block.equal?(self.class.__flexr_spec.eof_rules[state])
18
+ @flexr_trace.record_eof { super }
19
+ else
20
+ super
21
+ end
22
+ end
23
+
24
+ private
25
+
26
+ def handle_unmatched_byte
27
+ @flexr_trace.record_unmatched { super }
28
+ end
29
+ end
30
+
31
+ attr_reader :steps
32
+
33
+ def initialize(lexer)
34
+ @lexer = lexer
35
+ @steps = []
36
+ @config = lexer.class.__flexr_config
37
+ @compiled = lexer.class.__flexr_compiled
38
+ @rules = lexer.class.__flexr_rules
39
+ @lexer.singleton_class.prepend(Actions)
40
+ @lexer.instance_variable_set(:@flexr_trace, self)
41
+ end
42
+
43
+ def run
44
+ @lexer.tokens
45
+ end
46
+
47
+ def record(rule)
48
+ start = @lexer.instance_variable_get(:@match_start)
49
+ ending = @lexer.instance_variable_get(:@match_end)
50
+ state = @lexer.state
51
+ candidates = candidates_at(start)
52
+ selected = candidates.find { |candidate| candidate[:rule] == rule.index }
53
+ reason = if @config.backend == :firstmatch
54
+ "first_match"
55
+ elsif candidates.length == 1
56
+ "only_match"
57
+ elsif selected && candidates.count { |candidate| candidate[:total_length] == selected[:total_length] } > 1
58
+ "rule_order"
59
+ else
60
+ "longest"
61
+ end
62
+ yield
63
+ pending = @lexer.instance_variable_get(:@pending)
64
+ bytes = @lexer.buffer.byteslice(start...ending)
65
+ @steps << { byte_begin: start, byte_end: ending, next_byte: @lexer.byte_pos, state: state, next_state: @lexer.state,
66
+ text: display_text(bytes), bytes: bytes.bytes, candidates: candidates,
67
+ rule: rule.index, patterns: rule.patterns.map(&:inspect), reason: reason,
68
+ action: pending ? "emit" : "skip", token: pending&.inspect }
69
+ end
70
+
71
+ def record_eof
72
+ position = @lexer.byte_pos
73
+ state = @lexer.state
74
+ yield
75
+ pending = @lexer.instance_variable_get(:@pending)
76
+ @steps << { byte_begin: position, byte_end: position, next_byte: @lexer.byte_pos,
77
+ state: state, next_state: @lexer.state, text: "", bytes: [], candidates: [],
78
+ rule: nil, patterns: [], reason: "eof", action: pending ? "emit" : "skip", token: pending&.inspect }
79
+ end
80
+
81
+ def record_unmatched
82
+ position = @lexer.byte_pos
83
+ state = @lexer.state
84
+ bytes = @lexer.buffer.byteslice(position, 1)
85
+ token = yield
86
+ @steps << { byte_begin: position, byte_end: position + 1, next_byte: @lexer.byte_pos,
87
+ state: state, next_state: @lexer.state, text: display_text(bytes), bytes: bytes.bytes,
88
+ candidates: [], rule: nil, patterns: [], reason: "unmatched",
89
+ action: token ? "emit" : "skip", token: token&.inspect }
90
+ token
91
+ end
92
+
93
+ def render
94
+ steps.map do |step|
95
+ candidates = step[:candidates].map do |candidate|
96
+ "rule #{candidate[:rule]}(#{candidate[:length]}/#{candidate[:total_length]})"
97
+ end.join(" ")
98
+ target = step[:rule] ? "rule #{step[:rule]}" : step[:reason]
99
+ "@#{step[:byte_begin]} #{step[:text].inspect} state=#{step[:state]} " \
100
+ "candidates: #{candidates} -> #{target} #{step[:action]} (#{step[:reason]}) " \
101
+ "state=#{step[:next_state]}#{step[:token] && " #{step[:token]}"}"
102
+ end.join("\n")
103
+ end
104
+
105
+ private
106
+
107
+ def candidates_at(start)
108
+ @config.backend == :firstmatch ? firstmatch_candidates(start) : dfa_candidates(start)
109
+ end
110
+
111
+ def dfa_candidates(start)
112
+ dfa = @compiled.machines.fetch(@lexer.state).dfa
113
+ best = {}
114
+ walk(dfa, start) do |acceptance, ending|
115
+ next if acceptance.bol_only && !@lexer.beginning_of_line?
116
+
117
+ rule = @rules.fetch(acceptance.rule_index)
118
+ candidate = candidate_for(rule, start, ending)
119
+ next unless candidate
120
+ previous = best[rule.index]
121
+ best[rule.index] = candidate if !previous || candidate[:total_length] >= previous[:total_length]
122
+ end
123
+ best.values.sort_by { |candidate| candidate[:rule] }
124
+ end
125
+
126
+ def firstmatch_candidates(start)
127
+ state = @config.states.fetch(@lexer.state)
128
+ @lexer.buffer.read_to_end
129
+ ending = @lexer.buffer.bytesize
130
+ if @config.encoding == Encoding::UTF_8
131
+ ending = start
132
+ ending += @lexer.buffer.utf8_character_length(ending) while
133
+ ending < @lexer.buffer.bytesize && @lexer.buffer.valid_utf8_at?(ending)
134
+ end
135
+ subject = @lexer.buffer.byteslice(start...ending).to_s
136
+ subject.force_encoding(@config.encoding)
137
+ @rules.filter_map do |rule|
138
+ next unless rule.states.include?(@lexer.state) || (state.inclusive && rule.states.include?(:initial))
139
+ matches = rule.patterns.each_with_index.filter_map do |pattern, index|
140
+ condition = rule.pattern_conditions.fetch(index)
141
+ next if condition.bol_only && !@lexer.beginning_of_line?
142
+ regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(::Regexp.escape(pattern))
143
+ reference = Regexp::Matcher.reference?(regexp, encoding: @config.encoding, unicode: @config.options[:unicode] == true)
144
+ match = Regexp::Matcher.match(regexp, subject, encoding: @config.encoding,
145
+ unicode: @config.options[:unicode] == true, reference: reference)
146
+ [match[0].bytesize, condition] if match&.begin(0)&.zero?
147
+ end
148
+ length, condition = matches.max_by(&:first)
149
+ next unless length
150
+ next if condition.end_anchor && !end_anchor?(start + length)
151
+
152
+ candidate_for(rule, start, start + length)
153
+ end
154
+ end
155
+
156
+ def candidate_for(rule, start, ending)
157
+ trailing = if rule.trailing
158
+ dfa = rule.trailing_dfa || (@trailing_dfas ||= {})[rule.index] ||= Flexr.compile_pattern(
159
+ rule.trailing, encoding: @config.encoding, options: { unicode: @config.options[:unicode] == true }
160
+ )
161
+ longest = nil
162
+ walk(dfa, ending) { |_acceptance, cursor| longest = cursor - ending }
163
+ longest
164
+ else
165
+ 0
166
+ end
167
+ return if trailing.nil?
168
+
169
+ { rule: rule.index, patterns: rule.patterns.map(&:inspect), length: ending - start,
170
+ trailing_length: trailing, total_length: ending - start + trailing }
171
+ end
172
+
173
+ def walk(dfa, start)
174
+ state = dfa.start
175
+ cursor = start
176
+ loop do
177
+ if @lexer.utf8_boundary?(cursor)
178
+ dfa.accepts[state].each do |acceptance|
179
+ yield acceptance, cursor unless acceptance.end_anchor && !end_anchor?(cursor)
180
+ end
181
+ end
182
+ break unless @lexer.buffer.ensure_available?(cursor + 1)
183
+ break unless (state = dfa.transition(state, @lexer.buffer.getbyte(cursor)))
184
+
185
+ cursor += 1
186
+ end
187
+ end
188
+
189
+ def end_anchor?(position)
190
+ @lexer.buffer.eof?(position) || @lexer.buffer.getbyte(position) == 0x0a
191
+ end
192
+
193
+ def display_text(bytes)
194
+ bytes.dup.force_encoding(Encoding::UTF_8).scrub
195
+ end
196
+ end
197
+ end
@@ -13,9 +13,7 @@ module Flexr
13
13
  "Lower" => "Lowercase", "Lowercase" => "Lowercase",
14
14
  "Upper" => "Uppercase", "Uppercase" => "Uppercase"
15
15
  }.freeze
16
- # rubocop:disable Style/MutableConstant
17
- CACHE = {}
18
- # rubocop:enable Style/MutableConstant
16
+ CACHE = {} # rubocop:disable Style/MutableConstant
19
17
  module_function
20
18
 
21
19
  def ranges(name, negate: false)
@@ -3,9 +3,7 @@
3
3
  module Flexr
4
4
  module Unicode
5
5
  module ReferenceRegexp
6
- # rubocop:disable Style/MutableConstant
7
- CACHE = {}
8
- # rubocop:enable Style/MutableConstant
6
+ CACHE = {} # rubocop:disable Style/MutableConstant
9
7
  module_function
10
8
 
11
9
  def match(pattern, subject, encoding:, options: 0, unicode: false)
@@ -22,12 +20,12 @@ module Flexr
22
20
  return CACHE[key] if CACHE.key?(key)
23
21
 
24
22
  parser = Regexp::Parser.new(pattern.source, options: effective_options, encoding: encoding, unicode: unicode)
25
- source = source_for(parser.parse, ignorecase: effective_options.anybits?(::Regexp::IGNORECASE))
23
+ source = source_for(parser.parse)
26
24
  regexp_options = effective_options & ~::Regexp::IGNORECASE
27
25
  CACHE[key] = ::Regexp.new(source, regexp_options).freeze
28
26
  end
29
27
 
30
- def source_for(node, ignorecase: false)
28
+ def source_for(node)
31
29
  case node
32
30
  when Regexp::AST::Empty, Regexp::AST::Anchor then ""
33
31
  when Regexp::AST::Fail then "(?!)"
@@ -37,7 +35,7 @@ module Flexr
37
35
  ranges = node.ranges.flat_map do |range|
38
36
  if range.first == Regexp::AST::Property
39
37
  property_ranges = Unicode::Property.ranges(range[2])
40
- property_ranges = casefold_ranges(property_ranges) if ignorecase
38
+ property_ranges = casefold_ranges(property_ranges) if range[3]
41
39
  range[1] ? complement(property_ranges) : property_ranges
42
40
  else
43
41
  [range]
@@ -45,13 +43,13 @@ module Flexr
45
43
  end
46
44
  ranges = complement(ranges) if node.negated
47
45
  codepoint_class(ranges)
48
- when Regexp::AST::Seq then node.children.map { |child| source_for(child, ignorecase: ignorecase) }.join
46
+ when Regexp::AST::Seq then node.children.map { |child| source_for(child) }.join
49
47
  when Regexp::AST::Alt
50
- "(?:#{node.children.map { |child| source_for(child, ignorecase: ignorecase) }.join('|')})"
51
- when Regexp::AST::Star then "(?:#{source_for(node.child, ignorecase: ignorecase)})*"
48
+ "(?:#{node.children.map { |child| source_for(child) }.join('|')})"
49
+ when Regexp::AST::Star then "(?:#{source_for(node.child)})*"
52
50
  when Regexp::AST::Repeat
53
51
  maximum = node.maximum.nil? ? "" : node.maximum
54
- "(?:#{source_for(node.child, ignorecase: ignorecase)}){#{node.minimum},#{maximum}}"
52
+ "(?:#{source_for(node.child)}){#{node.minimum},#{maximum}}"
55
53
  else
56
54
  raise CompileError, "unsupported reference AST node: #{node.class}"
57
55
  end
data/lib/flexr/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Flexr
4
- VERSION = "1.1.0"
4
+ VERSION = "2.0.0"
5
5
  end
data/lib/flexr.rb CHANGED
@@ -11,4 +11,5 @@ module Flexr
11
11
  autoload :Options, File.expand_path("flexr/options", __dir__)
12
12
  autoload :RakeTask, File.expand_path("flexr/rake_task", __dir__)
13
13
  autoload :CLI, File.expand_path("flexr/cli", __dir__)
14
+ autoload :Trace, File.expand_path("flexr/trace", __dir__)
14
15
  end
@@ -0,0 +1,114 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "ruby_lsp/addon"
4
+ require "flexr/cli"
5
+
6
+ RubyLsp::Addon.depend_on_ruby_lsp!("~> 0.26.11")
7
+
8
+ module RubyLsp
9
+ module Flexr
10
+ class Analysis
11
+ def diagnostics(source)
12
+ return @diagnostics if @source == source
13
+
14
+ diagnostics = begin
15
+ parsed = ::Flexr::Source::PrismReader.new(source).read
16
+ compiled = ::Flexr::CLI.compile(parsed, overrides: { warn_level: :all })
17
+ compiled.diagnostics.reject { |diagnostic| diagnostic.code == "FLEXR-W016" }
18
+ rescue ::Flexr::Error => e
19
+ e.diagnostic ? [e.diagnostic] : []
20
+ rescue ArgumentError => e
21
+ [::Flexr::Diagnostics.error("FLEXR-E017", e.message, location: [0, 0])]
22
+ end
23
+ @source = source.dup.freeze
24
+ @diagnostics = diagnostics
25
+ end
26
+
27
+ def message(diagnostic)
28
+ [diagnostic.message, diagnostic.note,
29
+ ("help: #{diagnostic.help}" if diagnostic.help)].compact.join("\n\n")
30
+ end
31
+ end
32
+
33
+ class Diagnostics
34
+ def initialize(analysis, encoding)
35
+ @analysis = analysis
36
+ @encoding = encoding
37
+ end
38
+
39
+ def run_diagnostic(uri, document)
40
+ return [] unless uri.path&.end_with?(".flexr.rb")
41
+
42
+ source = document.source
43
+ @analysis.diagnostics(source).map do |diagnostic|
44
+ first, last = diagnostic.location || [0, 0]
45
+ severity = { error: Constant::DiagnosticSeverity::ERROR, warning: Constant::DiagnosticSeverity::WARNING,
46
+ info: Constant::DiagnosticSeverity::INFORMATION }.fetch(diagnostic.severity)
47
+ Interface::Diagnostic.new(
48
+ range: Interface::Range.new(start: position(source, first), end: position(source, last)),
49
+ message: @analysis.message(diagnostic), code: diagnostic.code, severity: severity, source: "flexr"
50
+ )
51
+ end
52
+ end
53
+
54
+ private
55
+
56
+ def position(source, offset)
57
+ prefix = source.byteslice(0, [offset, source.bytesize].min)
58
+ line = prefix.count("\n")
59
+ column = prefix.byteslice(((prefix.b.rindex("\n") || -1) + 1)..).delete_suffix("\r")
60
+ character = case @encoding
61
+ when Encoding::UTF_8 then column.bytesize
62
+ when Encoding::UTF_16LE then column.encode(Encoding::UTF_16LE).bytesize / 2
63
+ else column.length
64
+ end
65
+ Interface::Position.new(line: line, character: character)
66
+ end
67
+ end
68
+
69
+ class Hover
70
+ def initialize(builder, analysis, dispatcher)
71
+ @builder = builder
72
+ @analysis = analysis
73
+ dispatcher.register(self, :on_call_node_enter)
74
+ end
75
+
76
+ def on_call_node_enter(node)
77
+ return unless node.name == :rule && node.receiver.nil?
78
+
79
+ source = node.location.source_lines.join
80
+ diagnostics = @analysis.diagnostics(source).select do |diagnostic|
81
+ diagnostic.location &&
82
+ (node.location.start_offset...node.location.end_offset).cover?(diagnostic.location.first)
83
+ end
84
+ return if diagnostics.empty?
85
+
86
+ text = diagnostics.map { |diagnostic| "#{diagnostic.code}: #{@analysis.message(diagnostic)}" }.join("\n\n")
87
+ @builder.push("```text\n#{text}\n```", category: :documentation)
88
+ end
89
+ end
90
+
91
+ class Addon < ::RubyLsp::Addon
92
+ def activate(global_state, _message_queue)
93
+ @analysis = Analysis.new
94
+ global_state.register_formatter("flexr", Diagnostics.new(@analysis, global_state.encoding))
95
+ end
96
+
97
+ def deactivate
98
+ @analysis = nil
99
+ end
100
+
101
+ def name
102
+ "flexr"
103
+ end
104
+
105
+ def version
106
+ ::Flexr::VERSION
107
+ end
108
+
109
+ def create_hover_listener(response_builder, _node_context, dispatcher)
110
+ Hover.new(response_builder, @analysis, dispatcher)
111
+ end
112
+ end
113
+ end
114
+ end
data/sig/flexr.rbs ADDED
@@ -0,0 +1,191 @@
1
+ module Flexr
2
+ VERSION: String
3
+
4
+ type pattern = String | ::Regexp
5
+ type token = [untyped, untyped] | Runtime::Token
6
+ type source_location = [Integer, Integer]
7
+ type cancellation = (^() -> untyped) | (^(Lexer) -> untyped) | _Cancellation | _ContextCancellation
8
+ type recovery_action = :skip | :raise | :token | :halt
9
+ type error_handler = (^(LexError) -> recovery_action) | _ErrorHandler
10
+
11
+ interface _Cancellation
12
+ def call: () -> untyped
13
+ end
14
+
15
+ interface _ContextCancellation
16
+ def call: (Lexer) -> untyped
17
+ end
18
+
19
+ interface _ErrorHandler
20
+ def call: (LexError) -> recovery_action
21
+ end
22
+
23
+ interface _Input
24
+ def read: (Integer) -> String?
25
+ end
26
+
27
+ interface _CompiledPattern
28
+ def accept?: (String) -> bool
29
+ end
30
+
31
+ def self.compile_pattern: (pattern, ?options: Hash[Symbol, untyped], ?encoding: Encoding?) -> _CompiledPattern
32
+ def self.parse_pattern: (pattern, ?options: Hash[Symbol, untyped]) -> untyped
33
+
34
+ module DSL
35
+ def rule: (pattern | Array[pattern], ?skip: bool, ?emit: untyped, ?followed_by: pattern?) ?{ () [self: Lexer] -> untyped } -> nil
36
+ def state: (*(Symbol | String), ?inclusive: bool) { () [self: singleton(Lexer)] -> untyped } -> void
37
+ def all_states: () { () [self: singleton(Lexer)] -> untyped } -> void
38
+ def on_eof: () { () [self: Lexer] -> untyped } -> void
39
+ def emits: (*untyped) -> void
40
+ def backend: (:table | :direct | :auto | :firstmatch | String) -> void
41
+ def token_kind: (:array | :struct | :yield | String) -> void
42
+ def encoding: (Encoding | String) -> void
43
+ def option: (*(Symbol | String)) -> void
44
+ def accel: (:auto | :strscan | :regexp | :none | String) -> void
45
+ end
46
+
47
+ class Lexer
48
+ extend DSL
49
+
50
+ def initialize: (String | _Input, ?filename: String?, ?error_mode: Symbol, ?max_token_size: Integer, ?max_lookahead_size: Integer?, ?max_buffer_size: Integer, ?max_state_stack: Integer, ?max_steps: Integer?, ?cancellation: cancellation?, ?retain_input: bool, ?chunk_size: Integer) -> void
51
+ def next_token: () -> token?
52
+ def tokens: () -> Array[token]
53
+ def each_token: () -> Enumerator[token, self]
54
+ | () { (untyped, ?untyped) -> untyped } -> self
55
+ def racc_next_token: () -> [untyped, untyped]
56
+
57
+ attr_reader filename: String?
58
+ attr_reader error_mode: Symbol
59
+ attr_reader max_token_size: Integer
60
+ attr_reader max_lookahead_size: Integer
61
+ attr_reader steps: Integer
62
+ attr_writer on_error: error_handler?
63
+ def input: () -> String
64
+ def text: () -> String
65
+ def text_bytesize: () -> Integer
66
+ def last_location: () -> Runtime::Location
67
+ def byte_pos: () -> Integer
68
+ def lineno: () -> Integer
69
+ alias line lineno
70
+ def beginning_of_line?: () -> bool
71
+ def state: () -> Symbol
72
+ def binary_input: () -> String
73
+ def emit: (untyped, ?untyped) -> token
74
+ def skip: () -> nil
75
+ def echo: () -> token
76
+ def error!: (String) -> token?
77
+ def push: (Symbol | String) -> Symbol
78
+ alias push_state push
79
+ def pop: () -> Symbol
80
+ alias pop_state pop
81
+ def begin_state: (Symbol | String) -> Symbol
82
+ alias state= begin_state
83
+ def less: (Integer) -> nil
84
+ def more: () -> true
85
+ end
86
+
87
+ module Runtime
88
+ class Token < ::Data
89
+ def self.new: (?type: untyped, ?value: untyped, ?location: Location?) -> instance
90
+ attr_reader type: untyped
91
+ attr_reader value: untyped
92
+ attr_reader location: Location?
93
+ def []: (:location | "location") -> Location?
94
+ | (Integer | Symbol | String) -> untyped
95
+ def with: (?type: untyped, ?value: untyped, ?location: Location?) -> self
96
+ end
97
+
98
+ class Location < ::Data
99
+ def self.new: (?filename: String?, ?byte_begin: Integer?, ?byte_end: Integer?, ?line_begin: Integer?, ?column_begin: Integer?, ?line_end: Integer?, ?column_end: Integer?) -> instance
100
+ attr_reader filename: String?
101
+ attr_reader byte_begin: Integer?
102
+ attr_reader byte_end: Integer?
103
+ attr_reader line_begin: Integer?
104
+ attr_reader line_end: Integer?
105
+ attr_reader column_begin: Integer?
106
+ attr_reader column_end: Integer?
107
+ def []: (:filename | "filename") -> String?
108
+ | (Integer | Symbol | String) -> (Integer | String)?
109
+ def with: (?filename: String?, ?byte_begin: Integer?, ?byte_end: Integer?, ?line_begin: Integer?, ?column_begin: Integer?, ?line_end: Integer?, ?column_end: Integer?) -> self
110
+ end
111
+
112
+ class TokenTooLargeError < LexError
113
+ def initialize: (?String, ?filename: String?, ?byte_pos: Integer?, ?line: Integer?, ?text: String?, ?rule: Integer?) -> void
114
+ attr_reader code: String
115
+ attr_reader rule: Integer?
116
+ end
117
+ end
118
+
119
+ class Error < StandardError
120
+ def initialize: (?String?, ?diagnostic: Diagnostic?, ?filename: String?, ?byte_pos: Integer?) -> void
121
+ attr_reader diagnostic: Diagnostic?
122
+ attr_reader filename: String?
123
+ attr_reader byte_pos: Integer?
124
+ end
125
+
126
+ class LexError < Error
127
+ def initialize: (String, ?filename: String?, ?byte_pos: Integer?, ?line: Integer?, ?text: String?, ?diagnostic: Diagnostic?) -> void
128
+ attr_reader line: Integer?
129
+ attr_reader text: String?
130
+ end
131
+
132
+ class CompileError < Error
133
+ end
134
+
135
+ class UnsupportedRegexpError < CompileError
136
+ end
137
+
138
+ class Diagnostic < ::Struct[untyped]
139
+ def self.new: (?code: String?, ?severity: Symbol?, ?message: String?, ?location: source_location?, ?help: String?, ?note: String?) -> instance
140
+ attr_accessor code: String?
141
+ attr_accessor severity: Symbol?
142
+ attr_accessor message: String?
143
+ attr_accessor location: source_location?
144
+ attr_accessor help: String?
145
+ attr_accessor note: String?
146
+ def error?: () -> bool
147
+ def warning?: () -> bool
148
+ def to_h: () -> Hash[Symbol, untyped]
149
+ end
150
+
151
+ class DiagnosticSet
152
+ include Enumerable[Diagnostic]
153
+ def initialize: () -> void
154
+ def each: () -> Enumerator[Diagnostic, Array[Diagnostic]]
155
+ | () { (Diagnostic) -> untyped } -> Array[Diagnostic]
156
+ def <<: (Diagnostic) -> self
157
+ def any_error?: () -> bool
158
+ def empty?: () -> bool
159
+ def to_a: () -> Array[Diagnostic]
160
+ def render: (?format: Symbol, ?color: Symbol | bool, ?source: String?, ?path: String?) -> String
161
+ end
162
+
163
+ module Diagnostics
164
+ def self.error: (String, String, ?location: source_location?, ?help: String?, ?note: String?) -> Diagnostic
165
+ def self.warning: (String, String, ?location: source_location?, ?help: String?, ?note: String?) -> Diagnostic
166
+ def self.info: (String, String, ?location: source_location?, ?help: String?, ?note: String?) -> Diagnostic
167
+ def self.raise!: (Diagnostic) -> bot
168
+ end
169
+
170
+ class Generator
171
+ def initialize: (String, ?output: String?, ?eval_mode: bool, ?options: Hash[Symbol, untyped]) -> void
172
+ def generate: () -> String
173
+ attr_reader diagnostics: Array[Diagnostic]
174
+ end
175
+
176
+ class RakeTask
177
+ def initialize: (?Symbol | String) ?{ (RakeTask) -> untyped } -> void
178
+ attr_accessor spec: String?
179
+ attr_accessor output: String?
180
+ attr_accessor warn_as_error: bool
181
+ end
182
+
183
+ module CLI
184
+ def self.run: (Array[String], ?out: IO | _Output, ?err: IO | _Output) -> Integer
185
+ end
186
+
187
+ interface _Output
188
+ def puts: (*untyped) -> nil
189
+ def write: (String) -> Integer
190
+ end
191
+ end
data/site/README.md CHANGED
@@ -1,6 +1,7 @@
1
1
  # flexr product site
2
2
 
3
- This directory contains the Astro/Starlight product site for flexr. It is a static GitHub Pages site with a custom landing page, documentation routes, and a fixture-backed playground preview.
3
+ This directory contains the Astro/Starlight product site for flexr. The Playground
4
+ runs real Ruby and flexr locally through a pinned ruby.wasm runtime.
4
5
 
5
6
  ## Local development
6
7
 
@@ -14,9 +15,59 @@ Run the same checks used by the Pages workflow:
14
15
 
15
16
  ```sh
16
17
  pnpm verify
18
+ pnpm test
17
19
  pnpm build
20
+ pnpm exec playwright install chromium
21
+ pnpm test:browser
18
22
  ```
19
23
 
20
- The current playground does not execute arbitrary Ruby. It demonstrates the matching decision model from fixed fixtures and keeps `--eval` out of the browser. A future Ruby WASM worker must preserve that security boundary and add runtime/generated equivalence tests before the UI claims full execution.
24
+ `pnpm dev`, `pnpm build`, and `pnpm test` package the current repository's Ruby
25
+ files and copy the pinned runtime into ignored `public/playground-runtime` assets.
26
+ The WASM binary and source bundle are not checked into Git. The deployment build
27
+ produces them, including the Ruby distribution's license and notice.
21
28
 
22
- The repository examples, CLI definitions, and Ruby implementation remain the behavioral source of truth. Site copy links to those artifacts where a summary would otherwise drift.
29
+ The first Run lazily creates a worker and downloads the runtime. Every run starts
30
+ a new Ruby VM, loads the specification, compiles with flexr, executes its actions,
31
+ and returns tokens, real diagnostics, DOT, trace steps, and generated Ruby.
32
+ Prism 1.7.0 is available in the pinned Ruby 4.0.0 WASM build, so both runtime and
33
+ generated execution are tested. The static DSL restrictions of `flexr check` and
34
+ the generator apply. Specifications must be self-contained or use the bundled
35
+ Ruby libraries; additional gems and local companion files are unavailable.
36
+ The [ruby.wasm WASI limitations](https://ruby.github.io/ruby.wasm/README_md.html),
37
+ including unsupported thread creation, also apply.
38
+ The original three presets remain editable starting points. TOML, GraphQL, and
39
+ SQL subset presets load the actual repository example sources, packaged at build
40
+ time, so their lexer rules are maintained in one place.
41
+
42
+ The main page stops execution after five seconds; runtime download/startup has a
43
+ 30-second limit. Source and input are limited to 20,000 characters each, DFA
44
+ construction to 1024 states, and output to 4 MB. Graphviz loads in a separate
45
+ worker only when Draw DFA is selected and has its own five-second limit.
46
+
47
+ Source/input are not uploaded by the application. Share links store them in the
48
+ URL fragment, and opening a link never executes it. Review shared Ruby before
49
+ running it. Workers isolate the UI and have a virtual filesystem; they are not a
50
+ complete hostile-code or memory sandbox. The page applies a CSP, disables common
51
+ network APIs in the Ruby worker after loading assets, and sanitizes rendered SVG.
52
+ Ruby actions can still modify their VM and interact with worker JavaScript.
53
+
54
+ ## Size and startup measurements
55
+
56
+ Measured on 2026-11-21, macOS arm64, Headless Chrome 157, using the local static
57
+ preview server without network throttling:
58
+
59
+ | Asset | Uncompressed bytes | gzip bytes |
60
+ |---|---:|---:|
61
+ | Ruby 4.0.0 and standard library | 32,555,374 | 10,114,484 |
62
+ | flexr Ruby files and bridge | 533,463 | 119,954 |
63
+ | TOML, GraphQL, and SQL presets | 2,750 | 838 |
64
+
65
+ One browser run measured 43 ms for downloading/compiling the runtime and 203 ms
66
+ for starting the VM and loading flexr. These local numbers do not predict a
67
+ remote cold download. The UI reports measured sizes and timings on every run.
68
+ Initial page loading fetches neither Ruby WASM nor Graphviz; the browser test
69
+ checks that no WASM request occurs before Run or when opening a share link.
70
+
71
+ For local Chrome instead of a Playwright download, set
72
+ `PLAYWRIGHT_CHROMIUM_EXECUTABLE` to its executable path. Screenshots from the
73
+ browser tests are written to ignored `test-results/` directories.
@@ -1,9 +1,11 @@
1
- import { defineConfig } from 'astro/config';
1
+ import { defineConfig, passthroughImageService } from 'astro/config';
2
2
  import starlight from '@astrojs/starlight';
3
3
 
4
4
  export default defineConfig({
5
5
  site: 'https://ydah.github.io',
6
6
  base: '/flexr',
7
+ image: { service: passthroughImageService() },
8
+ vite: { worker: { format: 'es' } },
7
9
  integrations: [
8
10
  starlight({
9
11
  title: 'flexr',