flexr 1.0.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +241 -72
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/RELEASING.md +3 -2
  25. data/docs/explanation/backends.md +30 -10
  26. data/docs/explanation/matching-semantics.md +4 -2
  27. data/docs/explanation/security-model.md +10 -3
  28. data/docs/explanation/unicode-and-encoding.md +3 -1
  29. data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
  30. data/docs/how-to/generate-a-lexer.md +3 -3
  31. data/docs/how-to/handle-errors.md +4 -1
  32. data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
  33. data/docs/how-to/track-token-locations.md +2 -1
  34. data/docs/how-to/tune-performance.md +15 -3
  35. data/docs/migration-v2.md +74 -0
  36. data/docs/perf-log.md +212 -0
  37. data/docs/reference/README.md +2 -0
  38. data/docs/reference/actions.md +5 -2
  39. data/docs/reference/cli.md +71 -5
  40. data/docs/reference/compatibility.md +10 -4
  41. data/docs/reference/diagnostics.md +346 -0
  42. data/docs/reference/dsl.md +30 -9
  43. data/docs/reference/errors.md +34 -3
  44. data/docs/reference/generated-artifacts.md +71 -6
  45. data/docs/reference/public-api.md +37 -7
  46. data/docs/reference/regexp.md +27 -13
  47. data/docs/reference/ruby-lsp.md +34 -0
  48. data/docs/reference/runtime.md +21 -4
  49. data/docs/reference/testing.md +57 -0
  50. data/docs/reference/tokens-and-locations.md +14 -4
  51. data/examples/graphql/README.md +14 -0
  52. data/examples/graphql/lexer.flexr.rb +29 -0
  53. data/examples/json/lexer.flexr.rb +6 -6
  54. data/examples/sql/README.md +16 -0
  55. data/examples/sql/lexer.flexr.rb +23 -0
  56. data/examples/toml/README.md +14 -0
  57. data/examples/toml/lexer.flexr.rb +23 -0
  58. data/gemfiles/ruby_lsp.gemfile +4 -0
  59. data/gemfiles/types.gemfile +7 -0
  60. data/lib/flexr/action_resolver.rb +15 -0
  61. data/lib/flexr/artifact_writer.rb +61 -0
  62. data/lib/flexr/automaton/accel.rb +78 -5
  63. data/lib/flexr/automaton/analysis.rb +202 -7
  64. data/lib/flexr/automaton/backend_cost_model.rb +40 -0
  65. data/lib/flexr/automaton/compiler.rb +229 -96
  66. data/lib/flexr/automaton/dfa.rb +106 -19
  67. data/lib/flexr/automaton/minimizer.rb +79 -31
  68. data/lib/flexr/automaton/nfa.rb +45 -13
  69. data/lib/flexr/automaton/types.rb +14 -0
  70. data/lib/flexr/cli.rb +128 -22
  71. data/lib/flexr/codegen/direct.rb +5 -41
  72. data/lib/flexr/codegen/firstmatch.rb +1 -7
  73. data/lib/flexr/codegen/table.rb +181 -69
  74. data/lib/flexr/codegen.rb +8 -0
  75. data/lib/flexr/configuration.rb +45 -0
  76. data/lib/flexr/diagnostics.rb +39 -3
  77. data/lib/flexr/dsl.rb +116 -47
  78. data/lib/flexr/errors.rb +7 -6
  79. data/lib/flexr/generated.rb +120 -46
  80. data/lib/flexr/generator.rb +102 -109
  81. data/lib/flexr/grammar.rb +41 -0
  82. data/lib/flexr/importer.rb +4 -40
  83. data/lib/flexr/ir.rb +2 -2
  84. data/lib/flexr/migration.rb +31 -0
  85. data/lib/flexr/minitest.rb +24 -0
  86. data/lib/flexr/options.rb +9 -5
  87. data/lib/flexr/rake_task.rb +6 -1
  88. data/lib/flexr/regexp/ast.rb +5 -2
  89. data/lib/flexr/regexp/matcher.rb +26 -0
  90. data/lib/flexr/regexp/normalizer.rb +63 -21
  91. data/lib/flexr/regexp/parser.rb +241 -60
  92. data/lib/flexr/regexp/tokenizer.rb +172 -47
  93. data/lib/flexr/rspec.rb +51 -0
  94. data/lib/flexr/runtime/buffer.rb +115 -9
  95. data/lib/flexr/runtime/core.rb +248 -51
  96. data/lib/flexr/runtime/errors.rb +66 -4
  97. data/lib/flexr/runtime/interpreter.rb +213 -97
  98. data/lib/flexr/runtime/location.rb +35 -12
  99. data/lib/flexr/runtime/record.rb +14 -0
  100. data/lib/flexr/runtime/token.rb +7 -1
  101. data/lib/flexr/runtime.rb +75 -0
  102. data/lib/flexr/source/passthrough.rb +70 -14
  103. data/lib/flexr/source/prism_reader.rb +217 -51
  104. data/lib/flexr/source/static_eval.rb +40 -4
  105. data/lib/flexr/source.rb +6 -0
  106. data/lib/flexr/testing.rb +47 -0
  107. data/lib/flexr/trace.rb +197 -0
  108. data/lib/flexr/unicode/data/properties.rb +1 -1
  109. data/lib/flexr/unicode/data.rb +11 -0
  110. data/lib/flexr/unicode/property.rb +4 -8
  111. data/lib/flexr/unicode/reference_regexp.rb +11 -9
  112. data/lib/flexr/unicode/version.rb +7 -0
  113. data/lib/flexr/version.rb +1 -1
  114. data/lib/flexr.rb +10 -76
  115. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  116. data/sig/flexr.rbs +191 -0
  117. data/site/README.md +54 -3
  118. data/site/astro.config.mjs +3 -1
  119. data/site/package.json +12 -5
  120. data/site/playwright.config.mjs +16 -0
  121. data/site/pnpm-lock.yaml +218 -88
  122. data/site/ruby/playground.rb +57 -0
  123. data/site/scripts/prepare-playground.mjs +35 -0
  124. data/site/scripts/verify-site.mjs +5 -3
  125. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  126. data/site/src/content/docs/concepts/security-model.md +9 -1
  127. data/site/src/content/docs/learn/generation.md +1 -1
  128. data/site/src/content/docs/reference/dsl.md +2 -2
  129. data/site/src/content/docs/reference/public-api.md +4 -2
  130. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  131. data/site/src/layouts/SiteLayout.astro +11 -3
  132. data/site/src/pages/index.astro +69 -155
  133. data/site/src/pages/playground.astro +40 -38
  134. data/site/src/playground/client.js +211 -0
  135. data/site/src/playground/graph-worker.mjs +11 -0
  136. data/site/src/playground/presets.mjs +33 -0
  137. data/site/src/playground/runtime.mjs +34 -0
  138. data/site/src/playground/share.mjs +30 -0
  139. data/site/src/playground/worker.mjs +29 -0
  140. data/site/src/styles/custom.css +12 -287
  141. data/site/src/styles/landing.css +239 -0
  142. data/site/tests/browser/playground.spec.mjs +99 -0
  143. data/site/tests/playground-runtime.test.mjs +85 -0
  144. data/test/ruby_lsp_addon_spec.rb +135 -0
  145. data/tools/coverage.rb +6 -1
  146. data/tools/docs_verify.rb +3 -2
  147. data/tools/golden.rb +54 -0
  148. data/tools/verification_failure.rb +119 -0
  149. metadata +66 -12
  150. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  151. data/benchmark/golden/json_lexer.sha256 +0 -1
  152. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  153. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  154. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  155. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  156. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  157. data/site/public/playground.js +0 -189
@@ -0,0 +1,165 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "tempfile"
4
+ require "json"
5
+ require_relative "../lib/flexr/cli"
6
+
7
+ module Flexr
8
+ module Benchmarking
9
+ module LramaLexer
10
+ LINE_BREAK = /\n/
11
+ module_function
12
+
13
+ def recognizer(patterns)
14
+ source = +"class GrammarRecognizer < Flexr::Lexer\nbackend :firstmatch\noption :experimental, :standalone\n"
15
+ patterns.each { |pattern, kind| source << "rule(#{pattern.inspect}, emit: #{kind.inspect})\n" }
16
+ source << "end\n"
17
+ Tempfile.create(["lrama-recognizer-", ".flexr.rb"]) do |file|
18
+ file.write(source)
19
+ file.flush
20
+ scope = Module.new
21
+ scope.module_eval(Generator.new(file.path).generate, file.path)
22
+ scope.const_get(:GrammarRecognizer, false)
23
+ end
24
+ end
25
+
26
+ def patterns(native, path)
27
+ require "prism"
28
+ method = nil
29
+ visit = lambda do |node|
30
+ method = node if node.is_a?(Prism::DefNode) && node.name == :lex_token
31
+ node.compact_child_nodes.each { |child| visit.call(child) }
32
+ end
33
+ visit.call(Prism.parse_file(path).value)
34
+ raise "upstream lex_token was not found" unless method
35
+
36
+ scans = []
37
+ visit = lambda do |node|
38
+ if node.is_a?(Prism::CallNode) && node.name == :scan &&
39
+ node.receiver.is_a?(Prism::InstanceVariableReadNode) && node.receiver.name == :@scanner
40
+ argument = node.arguments.arguments.first
41
+ scans << case argument
42
+ when Prism::RegularExpressionNode then ::Regexp.new(argument.unescaped, argument.options)
43
+ when Prism::ConstantReadNode then native.const_get(argument.name)
44
+ else raise "unsupported upstream scan argument"
45
+ end
46
+ end
47
+ node.compact_child_nodes.each { |child| visit.call(child) }
48
+ end
49
+ visit.call(method)
50
+ raise "upstream lex_token scan layout changed" unless scans.length == 12
51
+
52
+ scans = scans.drop(4)
53
+ scans[0] = ::Regexp.union(native::SYMBOLS.map { |symbol| symbol.gsub(/\\(.)/, '\1') })
54
+ scans[1] = ::Regexp.union(native::PERCENT_TOKENS)
55
+ scans.zip(%i[SYMBOL DIRECTIVE SYMBOL TAG CHARACTER STRING INTEGER IDENTIFIER])
56
+ end
57
+
58
+ def adapter(native, recognizer)
59
+ Class.new(native) do
60
+ define_method(:initialize) do |file|
61
+ super(file)
62
+ @recognizer = recognizer.new(file.text)
63
+ end
64
+
65
+ define_method(:lex_token) do
66
+ loop do
67
+ if @scanner.scan(LINE_BREAK)
68
+ newline
69
+ elsif @scanner.scan(/[^\S\n]+/)
70
+ next
71
+ elsif @scanner.scan(%r{/\*})
72
+ lex_comment
73
+ elsif @scanner.scan(%r{//[^\n]*(?:\n|$)})
74
+ newline if @scanner.matched.end_with?("\n")
75
+ else
76
+ break
77
+ end
78
+ end
79
+ reset_first_position
80
+ return if @scanner.eos?
81
+
82
+ @recognizer.instance_variable_set(:@position, @scanner.pos)
83
+ kind, text = @recognizer.next_token
84
+ raise ::ParseError, location.generate_error_message("Unexpected token") if
85
+ kind == :DIRECTIVE && @scanner.string.byteslice(@recognizer.byte_pos, 1)&.match?(/[-A-Za-z0-9_.]/)
86
+ @scanner.pos = @recognizer.byte_pos
87
+ types = { TAG: :Tag, CHARACTER: :Char, STRING: :Str, INTEGER: :Int, IDENTIFIER: :Ident }
88
+ token_class = native::Token.const_get(types.fetch(kind, :Token))
89
+ value = kind == :INTEGER ? Integer(text) : text
90
+ type = case kind
91
+ when :SYMBOL, :DIRECTIVE then text
92
+ when :IDENTIFIER then identifier_colon? ? :IDENT_COLON : :IDENTIFIER
93
+ else kind
94
+ end
95
+ [type, token_class.new(s_value: value, location: location)]
96
+ rescue StandardError => e
97
+ raise unless e.class.name&.end_with?("LexError")
98
+
99
+ raise ::ParseError, location.generate_error_message("Unexpected token")
100
+ end
101
+ end
102
+ end
103
+
104
+ def observe(klass, source, path)
105
+ original = Lrama::Lexer
106
+ events = []
107
+ recording = Class.new(klass) do
108
+ define_method(:next_token) do
109
+ super().tap do |token|
110
+ if token
111
+ type, value = token
112
+ loc = value.location
113
+ events << [type, value.s_value, loc.first_line, loc.first_column, loc.last_line, loc.last_column]
114
+ end
115
+ end
116
+ end
117
+ end
118
+ Lrama.send(:remove_const, :Lexer)
119
+ Lrama.const_set(:Lexer, recording)
120
+ error = nil
121
+ begin
122
+ Lrama::Parser.new(source, path).parse
123
+ rescue StandardError => e
124
+ error = [e.class.name, e.message]
125
+ end
126
+ [events, error]
127
+ ensure
128
+ Lrama.send(:remove_const, :Lexer)
129
+ Lrama.const_set(:Lexer, original)
130
+ end
131
+
132
+ def run(root, iterations: 3, out: $stdout)
133
+ $LOAD_PATH.unshift(File.join(root, "lib"))
134
+ require "lrama"
135
+ native = Lrama::Lexer
136
+ candidate = adapter(native, recognizer(patterns(native, File.join(root, "lib/lrama/lexer.rb"))))
137
+ paths = Dir[File.join(root, "spec/fixtures/**/*.y")]
138
+ raise "no upstream fixture corpus" if paths.empty?
139
+
140
+ inputs = paths.map { |path| [path, File.read(path)] }
141
+ successful_parses = 0
142
+ tokens = 0
143
+ inputs.each do |path, source|
144
+ expected = observe(native, source, path)
145
+ actual = observe(candidate, source, path)
146
+ raise "lexer mismatch: #{path}" unless actual == expected
147
+ successful_parses += 1 unless expected.last
148
+ tokens += expected.first.length
149
+ end
150
+ timings = { native: native, standalone: candidate }.transform_values do |klass|
151
+ started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
152
+ iterations.times { inputs.each { |path, source| observe(klass, source, path) } }
153
+ Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
154
+ end
155
+ result = { ruby: RUBY_DESCRIPTION, fixtures: paths.length, successful_parses: successful_parses,
156
+ tokens_compared: tokens, iterations: iterations, seconds: timings,
157
+ scope: "normal grammar tokens; existing trivia, C-code scanning, locations, and parser retained" }
158
+ out.puts JSON.pretty_generate(result)
159
+ result
160
+ end
161
+ end
162
+ end
163
+ end
164
+
165
+ Flexr::Benchmarking::LramaLexer.run(ARGV.fetch(0)) if $PROGRAM_NAME == __FILE__
@@ -0,0 +1,79 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "open3"
4
+ require "rbconfig"
5
+ require_relative "run"
6
+
7
+ module Flexr
8
+ module Benchmarking
9
+ class RubyMatrix
10
+ ROOT = File.expand_path("..", __dir__)
11
+ FLAGS = { "interpreter" => ["--disable=yjit,zjit"], "yjit" => ["--yjit"], "zjit" => ["--zjit"] }.freeze
12
+ PROBE = <<~RUBY.lines.map(&:strip).join(";").freeze
13
+ require "json"
14
+ engine = ARGV.fetch(0)
15
+ enabled = case engine
16
+ when "interpreter"
17
+ !(defined?(RubyVM::YJIT) && RubyVM::YJIT.enabled?) && !(defined?(RubyVM::ZJIT) && RubyVM::ZJIT.enabled?)
18
+ when "yjit" then defined?(RubyVM::YJIT) && RubyVM::YJIT.enabled?
19
+ when "zjit" then defined?(RubyVM::ZJIT) && RubyVM::ZJIT.enabled?
20
+ end
21
+ puts JSON.generate(available: !!enabled, ruby: RUBY_DESCRIPTION, reason: enabled ? nil : "engine is not enabled by this Ruby build")
22
+ RUBY
23
+
24
+ def initialize(arguments: [], probe: nil, benchmark: nil)
25
+ @arguments = arguments
26
+ @probe = probe || method(:probe_engine)
27
+ @benchmark = benchmark || method(:benchmark_engine)
28
+ end
29
+
30
+ def run(engines: FLAGS.keys)
31
+ results = engines.to_h do |engine|
32
+ availability = @probe.call(engine)
33
+ result = if availability.fetch("available")
34
+ { "status" => "measured", "ruby" => availability["ruby"], "result" => @benchmark.call(engine) }
35
+ else
36
+ { "status" => "unavailable", "reason" => availability.fetch("reason") }
37
+ end
38
+ [engine, result]
39
+ end
40
+ { "schema" => 1, "engines" => results }
41
+ end
42
+
43
+ def self.command(argv, out: $stdout, err: $stderr)
44
+ options = Benchmarking.parse(argv.dup)
45
+ return Benchmarking.usage(out) if options[:help]
46
+ raise ArgumentError, "matrix results cannot share a baseline output file" if options[:write_baseline]
47
+
48
+ out.puts JSON.pretty_generate(new(arguments: argv).run)
49
+ 0
50
+ rescue ArgumentError, Errno::ENOENT => e
51
+ err.puts "matrix error: #{e.message}"
52
+ 2
53
+ rescue RuntimeError => e
54
+ err.puts "matrix error: #{e.message}"
55
+ 1
56
+ end
57
+
58
+ private
59
+
60
+ def probe_engine(engine)
61
+ stdout, stderr, status = Open3.capture3({ "RUBYOPT" => nil }, RbConfig.ruby, *FLAGS.fetch(engine),
62
+ "-e", PROBE, engine, chdir: ROOT)
63
+ return { "available" => false, "reason" => stderr.strip } unless status.success?
64
+
65
+ JSON.parse(stdout)
66
+ end
67
+
68
+ def benchmark_engine(engine)
69
+ stdout, stderr, status = Open3.capture3({ "RUBYOPT" => nil }, RbConfig.ruby, *FLAGS.fetch(engine),
70
+ "-Ilib", "benchmark/run.rb", *@arguments, "--json", chdir: ROOT)
71
+ raise "#{engine} benchmark failed (#{status.exitstatus}): #{stderr}#{stdout}" unless status.success?
72
+
73
+ JSON.parse(stdout)
74
+ end
75
+ end
76
+ end
77
+ end
78
+
79
+ exit Flexr::Benchmarking::RubyMatrix.command(ARGV) if $PROGRAM_NAME == __FILE__
@@ -0,0 +1,109 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "run"
4
+
5
+ module Flexr
6
+ module Benchmarking
7
+ class PhaseProfiler
8
+ attr_reader :seconds, :elapsed
9
+
10
+ def initialize
11
+ @seconds = %w[scanning actions tokens locations other].to_h { |phase| [phase, 0.0] }
12
+ end
13
+
14
+ def measure(&)
15
+ stack = ["other"]
16
+ started = previous = clock
17
+ @trace = TracePoint.new(:call, :return, :c_call, :c_return) do |event|
18
+ now = clock
19
+ @seconds[stack.last] += now - previous
20
+ previous = now
21
+ if %i[call c_call].include?(event.event)
22
+ stack << (phase(event) || stack.last)
23
+ elsif stack.length > 1
24
+ stack.pop
25
+ end
26
+ end
27
+ @trace.enable(target_thread: Thread.current, &)
28
+ ensure
29
+ @trace&.disable
30
+ finished = clock
31
+ @seconds[stack.last] += finished - previous
32
+ @elapsed = finished - started
33
+ end
34
+
35
+ def enabled?
36
+ @trace&.enabled? || false
37
+ end
38
+
39
+ private
40
+
41
+ def clock
42
+ Process.clock_gettime(Process::CLOCK_MONOTONIC)
43
+ end
44
+
45
+ def phase(event)
46
+ if event.self.is_a?(Runtime::Interpreter)
47
+ return "scanning" if event.method_id == :scan
48
+ elsif event.self.is_a?(Lexer)
49
+ case event.method_id
50
+ when :scan_one then return "scanning"
51
+ when :execute then return "actions"
52
+ when :emit, :text, :next_token, :tokens then return "tokens"
53
+ when :last_location, :update_position, :advance_location! then return "locations"
54
+ end
55
+ end
56
+ nil
57
+ end
58
+ end
59
+
60
+ class ProfileRunner < Runner
61
+ def run
62
+ path = File.expand_path(@options.fetch(:spec))
63
+ input = input_for(path)
64
+ runtime = profile_class(load_runtime(path), input)
65
+ generated = profile_class(load_generated(path), input)
66
+ {
67
+ "schema" => 1, "ruby" => RUBY_DESCRIPTION,
68
+ "spec" => relative_path(path), "input_bytes" => input.bytesize,
69
+ "input_sha256" => Digest::SHA256.hexdigest(input),
70
+ "iterations" => @options.fetch(:iterations),
71
+ "note" => "TracePoint changes execution costs and disables JIT optimizations; use phase timings only to locate work, not as throughput measurements.",
72
+ "modes" => { "runtime" => runtime, "generated" => generated }
73
+ }
74
+ end
75
+
76
+ private
77
+
78
+ def profile_class(klass, input)
79
+ iterations = @options.fetch(:iterations)
80
+ klass.new(input).tokens
81
+ profiler = PhaseProfiler.new
82
+ tokens = profiler.measure do
83
+ result = nil
84
+ iterations.times { result = klass.new(input).tokens }
85
+ result
86
+ end
87
+ {
88
+ "tokens" => tokens.length,
89
+ "instrumented_elapsed_seconds" => profiler.elapsed,
90
+ "phase_seconds" => profiler.seconds
91
+ }
92
+ end
93
+ end
94
+
95
+ def self.profile(argv, out: $stdout, err: $stderr)
96
+ options = parse(argv.dup)
97
+ return usage(out) if options[:help]
98
+ raise ArgumentError, "profiling does not support baseline checks or writes" if options[:baseline] || options[:write_baseline]
99
+
100
+ out.puts JSON.pretty_generate(ProfileRunner.new(options).run)
101
+ 0
102
+ rescue ArgumentError, Errno::ENOENT => e
103
+ err.puts "profile error: #{e.message}"
104
+ 2
105
+ end
106
+ end
107
+ end
108
+
109
+ exit Flexr::Benchmarking.profile(ARGV) if $PROGRAM_NAME == __FILE__
data/benchmark/run.rb CHANGED
@@ -13,7 +13,10 @@ module Flexr
13
13
  DEFAULT_SPEC = File.expand_path("../examples/json/lexer.flexr.rb", __dir__)
14
14
  DEFAULT_INPUTS = {
15
15
  %r{/examples/json/} => ('{"answer": 42}' * 10_000).freeze,
16
- %r{/examples/toy_lang/} => ("answer + 12" * 10_000).freeze
16
+ %r{/examples/toy_lang/} => ("answer + 12" * 10_000).freeze,
17
+ %r{/examples/toml/} => ('title = "flexr" ' * 10_000).freeze,
18
+ %r{/examples/graphql/} => ('query { user(id: 12) { name } } ' * 10_000).freeze,
19
+ %r{/examples/sql/} => ('select name from users where id >= 12; ' * 10_000).freeze
17
20
  }.freeze
18
21
 
19
22
  module_function
@@ -0,0 +1,78 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "run"
4
+ require_relative "../lib/flexr/cli"
5
+
6
+ module Flexr
7
+ module Benchmarking
8
+ module StateDispatch
9
+ module_function
10
+
11
+ def lexers(path)
12
+ parsed = CLI.read_spec(path)
13
+ compiled = CLI.compile(parsed, overrides: { backend: :table })
14
+ indent = Source::Passthrough.indentation(parsed.source, parsed.first_dsl_offset)
15
+ scanner = Codegen::Table.new(compiled).source(indent: indent)
16
+ replacement = scanner.gsub("dfa.transition(current, byte)", "__flexr_state_transition(current, ec[byte])")
17
+ raise "generated scanner has no table dispatch" if replacement == scanner
18
+
19
+ replacement += dispatch_source(compiled, indent)
20
+ generated = Generator.new(path, options: { backend: :table }).generate
21
+ raise "generated scanner template changed" unless generated.include?(scanner)
22
+
23
+ [generated, Generator.new(path, options: { backend: :direct }).generate,
24
+ generated.sub(scanner, replacement)].map do |source|
25
+ scope = Module.new
26
+ scope.module_eval(source, path)
27
+ parsed.class_name.split("::").reject(&:empty?).inject(scope) { |parent, name| parent.const_get(name, false) }
28
+ end
29
+ end
30
+
31
+ def dispatch_source(compiled, indent)
32
+ lines = ["def __flexr_state_transition(state, class_id)", "case @state"]
33
+ compiled.machines.each do |name, machine|
34
+ lines << "when #{name.inspect}" << "case state"
35
+ machine.dfa.transitions.each_with_index do |row, state|
36
+ if row.compact.empty?
37
+ lines << "when #{state} then nil"
38
+ next
39
+ end
40
+ lines << "when #{state}" << "case class_id"
41
+ row.each_with_index.group_by(&:first).each do |destination, pairs|
42
+ next unless destination
43
+
44
+ lines << "when #{pairs.map(&:last).join(', ')} then #{destination}"
45
+ end
46
+ lines << "else nil" << "end"
47
+ end
48
+ lines << "else nil" << "end"
49
+ end
50
+ (lines + ["else nil", "end", "end"]).map { |line| "#{indent}#{line}\n" }.join
51
+ end
52
+
53
+ def run(argv, out: $stdout)
54
+ options = Benchmarking.parse(argv.dup)
55
+ return Benchmarking.usage(out) if options[:help]
56
+ raise ArgumentError, "--baseline and --write-baseline do not apply to this experiment" if options[:baseline] || options[:write_baseline]
57
+
58
+ path = File.expand_path(options.fetch(:spec))
59
+ input = options[:input_file] ? File.binread(options[:input_file]) : Benchmarking::DEFAULT_INPUTS.find { |pattern, _| path.match?(pattern) }&.last
60
+ raise ArgumentError, "--input-file is required for this spec" unless input
61
+
62
+ table, direct, switched = lexers(path)
63
+ expected = table.new(input).tokens
64
+ raise "state dispatch changed tokens" unless [direct, switched].all? { |lexer| lexer.new(input).tokens == expected }
65
+
66
+ runner = Runner.new(options)
67
+ result = { ruby: RUBY_DESCRIPTION, input_bytes: input.bytesize, iterations: options[:iterations],
68
+ modes: { table: runner.send(:measure_class, table, input),
69
+ direct: runner.send(:measure_class, direct, input),
70
+ state_cases: runner.send(:measure_class, switched, input) } }
71
+ out.puts JSON.pretty_generate(result)
72
+ result
73
+ end
74
+ end
75
+ end
76
+ end
77
+
78
+ Flexr::Benchmarking::StateDispatch.run(ARGV) if $PROGRAM_NAME == __FILE__
data/docs/RELEASING.md CHANGED
@@ -18,7 +18,8 @@ output as SVG and should be run before a release that changes CLI visualization.
18
18
 
19
19
  Then:
20
20
 
21
- 1. Review the commits since the previous release and update `lib/flexr/version.rb`.
21
+ 1. Review the commits since the previous release, add a dated version section
22
+ to [the changelog](../CHANGELOG.md), and update `lib/flexr/version.rb`.
22
23
  2. Verify the gemspec metadata URLs.
23
24
  3. Build and inspect the package with `gem build flexr.gemspec`.
24
25
  4. Confirm generated golden files and the Unicode snapshot are intentional.
@@ -26,5 +27,5 @@ Then:
26
27
 
27
28
  Unicode snapshot changes are minor-version compatibility changes because they
28
29
  can change generated output. Stable runtime and generated-file contracts are
29
- preserved within a major version unless this document and the release commits
30
+ preserved within a major version unless this document and the changelog
30
31
  state a migration.
@@ -8,26 +8,46 @@ target for `firstmatch`.
8
8
 
9
9
  ## `direct`
10
10
 
11
- The stable dense-dispatch backend. It emits a flattened dispatch structure for
12
- large machines and can reduce lookup overhead at the cost of artifact size.
13
- It preserves the table backend's longest-match behavior.
11
+ The stable dense-dispatch backend. It emits one flattened transition structure
12
+ and avoids retaining a second packed or row table. It can reduce lookup
13
+ overhead at the cost of artifact size and preserves the table backend's
14
+ leftmost-longest behavior.
14
15
 
15
16
  ## `auto`
16
17
 
17
- The stable selection policy. At compile time it chooses direct dispatch when the
18
- largest state-by-byte-class table exceeds the configured threshold, otherwise
19
- it uses table dispatch. The effective backend is recorded in generated headers.
18
+ The stable selection policy. At compile time it compares estimated dense bytes,
19
+ packed bytes, and lookup cost for each lexical-state machine. The effective
20
+ backend is recorded in generated headers.
20
21
 
21
22
  ## `firstmatch`
22
23
 
23
24
  Experimental compatibility behavior. It requires `option :experimental` and
24
- can return a shorter match merely because its rule appears earlier. Generation
25
- compares a deterministic input corpus with the table matcher and aborts if the
26
- results differ.
25
+ can return a shorter match merely because its rule appears earlier. Runtime and
26
+ generated artifacts preserve that same behavior. Potential differences from
27
+ the stable longest-match backends are reported with `FLEXR-W010`.
27
28
 
28
29
  ## Acceleration and packing
29
30
 
30
31
  Region acceleration and packed tables are representations, not semantic
31
32
  backends. `accel` can be disabled for debugging or equivalence tests. Table
32
33
  compression and packed Base64 output trade source size against loading cost;
33
- they do not change token streams.
34
+ packed transitions are queried without creating a second dense table, and a
35
+ dense row is materialized lazily only for inspection. These choices do not
36
+ change token streams.
37
+
38
+ Explicit `accel :strscan` also derives complete-token expressions from the DFA.
39
+ For each starting-byte branch, all accepting states must select the same rule.
40
+ The reachable graph may have self-loops but no other cycles, and every edge
41
+ must consume ASCII bytes. Outgoing byte sets are disjoint because they come
42
+ from deterministic transitions. A greedy self-loop cannot consume an exit
43
+ byte; greedy optional exits therefore find the longest accepted prefix without
44
+ adopting the original Ruby regexp's alternation priority. Unproven branches
45
+ fall back to DFA traversal.
46
+
47
+ Expression expansion is capped at 32 states per path and 4 KiB per expression.
48
+ Plans are shared by lexer instances through their compiled machine. Native
49
+ scans use `StringScanner#skip`, retaining byte lengths without materializing
50
+ matched text. The fast path is disabled for mutable or streaming input,
51
+ non-retained buffers, token/step/cancellation guards, anchors, trailing context,
52
+ and `firstmatch`. These restrictions preserve the DFA's boundary and resource
53
+ checks.
@@ -11,8 +11,10 @@ extends the selection end without extending the consumed token end.
11
11
 
12
12
  The matcher is byte-oriented so binary input and invalid UTF-8 have deterministic
13
13
  behavior. UTF-8 patterns still require valid codepoint boundaries before a
14
- match is accepted. The runtime and generated scanner share the compiled model;
15
- acceleration may skip self-loop bytes but cannot change the winner.
14
+ match is accepted. Unicode properties and POSIX classes are expanded from the
15
+ vendored UCD into this same model rather than matched by a separate host-regexp
16
+ path. The runtime and generated scanner share the compiled model; acceleration
17
+ may skip self-loop bytes but cannot change the winner.
16
18
 
17
19
  `firstmatch` is intentionally separate. It picks the first rule that has a
18
20
  matching alternative and therefore is useful only for compatibility migrations
@@ -13,6 +13,13 @@ to discover dynamic patterns. Use it only with trusted files and a trusted
13
13
  environment. The same rule applies to generated output: review or sign the
14
14
  artifact before loading it in a process with sensitive access.
15
15
 
16
- The runtime's `max_token_size` and `max_state_stack` limits provide resource
17
- guards for input processing. They are operational controls, not a security
18
- sandbox and do not protect against arbitrary Ruby actions.
16
+ The runtime's `max_token_size`, `max_lookahead_size`, `max_buffer_size`,
17
+ `max_state_stack`, and optional `max_steps` limits provide resource guards for
18
+ input processing. A `cancellation` callback can stop concurrent work
19
+ cooperatively. Limits are checked while scanning, before an oversized token's
20
+ action runs. They are operational controls, not a security sandbox and do not
21
+ protect against arbitrary Ruby actions.
22
+
23
+ Use `retain_input: false` for long-lived streaming lexers that do not need the
24
+ complete input after consumption. This makes the buffer a sliding window;
25
+ `input` and `binary_input` no longer contain discarded prefixes.
@@ -8,7 +8,9 @@ allowing Unicode-aware patterns.
8
8
  The vendored Unicode Character Database snapshot, currently 15.1.0, is the
9
9
  compatibility oracle for `\p{...}`, POSIX properties where a Unicode property
10
10
  is defined, Unicode shorthand behavior with `option :unicode`, and simple case
11
- folding. Host Ruby Unicode tables are not used as a cross-version oracle.
11
+ folding. Property ranges are compiled into the same DFA as ordinary rules, so
12
+ they preserve leftmost-longest matching and streaming behavior. Host Ruby
13
+ Unicode tables are not used as a cross-version oracle.
12
14
 
13
15
  Use `encoding Encoding::BINARY` for protocol or byte-oriented formats. Use the
14
16
  default UTF-8 encoding for text. A pattern with invalid UTF-8 or an input that
@@ -6,15 +6,17 @@ Use `--standalone` to embed the flexr runtime into the generated file:
6
6
  flexr lexer.flexr.rb --standalone -o lexer_standalone.rb
7
7
  ```
8
8
 
9
- The generated file removes `require "flexr"` and includes the runtime sources
10
- needed to execute the compiled tables. It can be loaded by a Ruby process that
11
- does not have the flexr gem installed:
9
+ The generated file removes `require "flexr"` and includes only the runtime
10
+ sources needed to execute its compiled tables. Compiler, generator, and
11
+ minimizer sources are omitted. The regexp reference matcher and Unicode tables
12
+ are included only when the selected backend or trailing context needs them. It
13
+ can be loaded by a Ruby process that does not have the flexr gem installed:
12
14
 
13
15
  ```sh
14
16
  ruby -e 'load ARGV.fetch(0); p Lexer.new("12").tokens' lexer_standalone.rb
15
17
  ```
16
18
 
17
- Generation still needs the flexr gem and Prism on Ruby 3.3 or newer. Runtime
19
+ Generation still needs the flexr gem and Prism on Ruby 3.4 or newer. Runtime
18
20
  execution needs the generated file and Ruby's standard library only.
19
21
 
20
22
  Standalone output is larger and duplicates the runtime for every artifact.
@@ -13,9 +13,9 @@ Ruby constants, comments, requires, and non-DSL code remain in the file.
13
13
 
14
14
  ## Static expressions
15
15
 
16
- The default path accepts regexp and string literals, arrays of supported
17
- values, constants, ranges, interpolation whose values are static, `.freeze`,
18
- and `Regexp.union`. A call whose value depends on runtime state raises
16
+ The default path accepts regexp and string literals, arrays and hashes of
17
+ supported values, constants, ranges, interpolation whose values are static,
18
+ `.freeze`, and `Regexp.union`. Constants follow Ruby source order. A call whose value depends on runtime state raises
19
19
  `FLEXR-E017`. See the [static-expression reference](../reference/generated-artifacts.md)
20
20
  for the exact boundary.
21
21
 
@@ -29,4 +29,7 @@ instance with `lexer.on_error = ->(error) { :skip }`, `:raise`, `:token`, or
29
29
  byte position, line, and text.
30
30
 
31
31
  `error!` creates the same error from an action. `max_token_size` protects the
32
- runtime from unexpectedly large tokens and raises `FLEXR-E012` when exceeded.
32
+ runtime from unexpectedly large tokens and raises `FLEXR-E012` before the
33
+ action runs. Lookahead, buffer, step, state-stack, cancellation, and progress
34
+ guards likewise raise structured runtime errors; their stable codes are listed
35
+ in the [diagnostics catalog](../reference/diagnostics.md).
@@ -20,6 +20,11 @@ in chunks; `chunk_size:` controls the buffer size. `next_token` returns one
20
20
  token or `nil` at EOF, `tokens` drains the lexer into an array, and
21
21
  `each_token` returns an enumerator when no block is given.
22
22
 
23
+ For a long-lived stream that does not need old input, pass
24
+ `retain_input: false`. Combine it with `max_buffer_size:`, `max_token_size:`,
25
+ and `max_lookahead_size:` to keep memory bounded. Use `max_steps:` or a
26
+ `cancellation:` callback when request-level work must also be bounded.
27
+
23
28
  Use runtime mode when the specification deliberately depends on ordinary Ruby
24
29
  execution. Static generation is a better fit when startup cost, deployment
25
30
  reproducibility, or a generator-free runtime matters.
@@ -21,7 +21,8 @@ Locations use byte offsets for `byte_begin` and `byte_end`, with an exclusive
21
21
  end offset. Lines and columns are one-based. UTF-8 columns count characters;
22
22
  binary columns count bytes. Column values are computed lazily unless
23
23
  `option :eager_columns` is set. `filename:` on the lexer is copied into each
24
- location and lexical error.
24
+ location and lexical error. The lexer tracks line and column state
25
+ incrementally; locations do not rescan all previously consumed input.
25
26
 
26
27
  `more` joins successive matches into one `text` value, so `last_location` spans
27
28
  the complete assembled token. See the [token and location contract](../reference/tokens-and-locations.md).