flexr 1.1.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/CHANGELOG.md +116 -0
- data/CONTRIBUTING.md +46 -3
- data/README.md +5 -4
- data/Rakefile +191 -68
- data/Steepfile +7 -0
- data/benchmark/baselines/json.json +2 -2
- data/benchmark/golden/calculator_lexer.rb +276 -0
- data/benchmark/golden/graphql_lexer.rb +310 -0
- data/benchmark/golden/json_lexer.rb +292 -0
- data/benchmark/golden/regexp_tokenizer.rb +293 -0
- data/benchmark/golden/ruby_subset_lexer.rb +280 -0
- data/benchmark/golden/sql_lexer.rb +293 -0
- data/benchmark/golden/toml_lexer.rb +292 -0
- data/benchmark/golden/toy_lang_lexer.rb +273 -0
- data/benchmark/golden/with_lrama_lexer.rb +264 -0
- data/benchmark/golden/with_racc_lexer.rb +264 -0
- data/benchmark/lrama_lexer.rb +165 -0
- data/benchmark/matrix.rb +79 -0
- data/benchmark/profile.rb +109 -0
- data/benchmark/run.rb +4 -1
- data/benchmark/state_dispatch.rb +78 -0
- data/docs/README.md +1 -2
- data/docs/RELEASING.md +3 -2
- data/docs/explanation/backends.md +17 -0
- data/docs/how-to/deploy-a-standalone-lexer.md +1 -1
- data/docs/how-to/tune-performance.md +12 -0
- data/docs/migration-v2.md +74 -0
- data/docs/perf-log.md +196 -0
- data/docs/reference/README.md +2 -0
- data/docs/reference/cli.md +71 -5
- data/docs/reference/compatibility.md +10 -4
- data/docs/reference/diagnostics.md +338 -0
- data/docs/reference/dsl.md +12 -5
- data/docs/reference/errors.md +28 -0
- data/docs/reference/generated-artifacts.md +42 -4
- data/docs/reference/public-api.md +33 -8
- data/docs/reference/regexp.md +19 -7
- data/docs/reference/ruby-lsp.md +34 -0
- data/docs/reference/runtime.md +2 -2
- data/docs/reference/testing.md +57 -0
- data/docs/reference/tokens-and-locations.md +8 -0
- data/examples/graphql/README.md +14 -0
- data/examples/graphql/lexer.flexr.rb +29 -0
- data/examples/json/lexer.flexr.rb +6 -6
- data/examples/sql/README.md +16 -0
- data/examples/sql/lexer.flexr.rb +23 -0
- data/examples/toml/README.md +14 -0
- data/examples/toml/lexer.flexr.rb +23 -0
- data/gemfiles/ruby_lsp.gemfile +4 -0
- data/gemfiles/types.gemfile +7 -0
- data/lib/flexr/automaton/accel.rb +69 -1
- data/lib/flexr/automaton/analysis.rb +161 -7
- data/lib/flexr/automaton/compiler.rb +179 -25
- data/lib/flexr/automaton/types.rb +1 -1
- data/lib/flexr/cli.rb +117 -17
- data/lib/flexr/codegen/firstmatch.rb +1 -7
- data/lib/flexr/codegen/table.rb +8 -2
- data/lib/flexr/configuration.rb +8 -0
- data/lib/flexr/diagnostics.rb +39 -3
- data/lib/flexr/dsl.rb +12 -9
- data/lib/flexr/errors.rb +6 -6
- data/lib/flexr/generated.rb +65 -17
- data/lib/flexr/generator.rb +62 -34
- data/lib/flexr/grammar.rb +41 -0
- data/lib/flexr/ir.rb +2 -2
- data/lib/flexr/migration.rb +31 -0
- data/lib/flexr/minitest.rb +24 -0
- data/lib/flexr/options.rb +4 -2
- data/lib/flexr/regexp/matcher.rb +26 -0
- data/lib/flexr/regexp/normalizer.rb +4 -8
- data/lib/flexr/regexp/parser.rb +127 -21
- data/lib/flexr/regexp/tokenizer.rb +26 -26
- data/lib/flexr/rspec.rb +51 -0
- data/lib/flexr/runtime/core.rb +47 -17
- data/lib/flexr/runtime/errors.rb +6 -4
- data/lib/flexr/runtime/interpreter.rb +55 -46
- data/lib/flexr/runtime/location.rb +35 -12
- data/lib/flexr/runtime/record.rb +14 -0
- data/lib/flexr/runtime/token.rb +7 -1
- data/lib/flexr/runtime.rb +7 -2
- data/lib/flexr/source/passthrough.rb +14 -11
- data/lib/flexr/source/prism_reader.rb +53 -16
- data/lib/flexr/testing.rb +47 -0
- data/lib/flexr/trace.rb +197 -0
- data/lib/flexr/unicode/property.rb +1 -3
- data/lib/flexr/unicode/reference_regexp.rb +8 -10
- data/lib/flexr/version.rb +1 -1
- data/lib/flexr.rb +1 -0
- data/lib/ruby_lsp/flexr/addon.rb +114 -0
- data/sig/flexr.rbs +191 -0
- data/site/README.md +54 -3
- data/site/astro.config.mjs +3 -1
- data/site/package.json +12 -5
- data/site/playwright.config.mjs +16 -0
- data/site/pnpm-lock.yaml +218 -88
- data/site/ruby/playground.rb +57 -0
- data/site/scripts/prepare-playground.mjs +35 -0
- data/site/scripts/verify-site.mjs +5 -3
- data/site/src/content/docs/concepts/matching-semantics.md +4 -2
- data/site/src/content/docs/concepts/security-model.md +9 -1
- data/site/src/content/docs/learn/generation.md +1 -1
- data/site/src/content/docs/reference/dsl.md +2 -2
- data/site/src/content/docs/reference/public-api.md +4 -2
- data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
- data/site/src/layouts/SiteLayout.astro +11 -3
- data/site/src/pages/index.astro +69 -155
- data/site/src/pages/playground.astro +40 -38
- data/site/src/playground/client.js +211 -0
- data/site/src/playground/graph-worker.mjs +11 -0
- data/site/src/playground/presets.mjs +33 -0
- data/site/src/playground/runtime.mjs +34 -0
- data/site/src/playground/share.mjs +30 -0
- data/site/src/playground/worker.mjs +29 -0
- data/site/src/styles/custom.css +12 -287
- data/site/src/styles/landing.css +239 -0
- data/site/tests/browser/playground.spec.mjs +99 -0
- data/site/tests/playground-runtime.test.mjs +85 -0
- data/test/ruby_lsp_addon_spec.rb +135 -0
- data/tools/docs_verify.rb +3 -2
- data/tools/golden.rb +54 -0
- data/tools/verification_failure.rb +119 -0
- metadata +56 -13
- data/benchmark/golden/calculator_lexer.sha256 +0 -1
- data/benchmark/golden/json_lexer.sha256 +0 -1
- data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
- data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
- data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
- data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
- data/benchmark/golden/with_racc_lexer.sha256 +0 -1
- data/docs/releases/v1.1.0.md +0 -44
- data/site/public/playground.js +0 -189
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "tempfile"
|
|
4
|
+
require "json"
|
|
5
|
+
require_relative "../lib/flexr/cli"
|
|
6
|
+
|
|
7
|
+
module Flexr
|
|
8
|
+
module Benchmarking
|
|
9
|
+
module LramaLexer
|
|
10
|
+
LINE_BREAK = /\n/
|
|
11
|
+
module_function
|
|
12
|
+
|
|
13
|
+
def recognizer(patterns)
|
|
14
|
+
source = +"class GrammarRecognizer < Flexr::Lexer\nbackend :firstmatch\noption :experimental, :standalone\n"
|
|
15
|
+
patterns.each { |pattern, kind| source << "rule(#{pattern.inspect}, emit: #{kind.inspect})\n" }
|
|
16
|
+
source << "end\n"
|
|
17
|
+
Tempfile.create(["lrama-recognizer-", ".flexr.rb"]) do |file|
|
|
18
|
+
file.write(source)
|
|
19
|
+
file.flush
|
|
20
|
+
scope = Module.new
|
|
21
|
+
scope.module_eval(Generator.new(file.path).generate, file.path)
|
|
22
|
+
scope.const_get(:GrammarRecognizer, false)
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def patterns(native, path)
|
|
27
|
+
require "prism"
|
|
28
|
+
method = nil
|
|
29
|
+
visit = lambda do |node|
|
|
30
|
+
method = node if node.is_a?(Prism::DefNode) && node.name == :lex_token
|
|
31
|
+
node.compact_child_nodes.each { |child| visit.call(child) }
|
|
32
|
+
end
|
|
33
|
+
visit.call(Prism.parse_file(path).value)
|
|
34
|
+
raise "upstream lex_token was not found" unless method
|
|
35
|
+
|
|
36
|
+
scans = []
|
|
37
|
+
visit = lambda do |node|
|
|
38
|
+
if node.is_a?(Prism::CallNode) && node.name == :scan &&
|
|
39
|
+
node.receiver.is_a?(Prism::InstanceVariableReadNode) && node.receiver.name == :@scanner
|
|
40
|
+
argument = node.arguments.arguments.first
|
|
41
|
+
scans << case argument
|
|
42
|
+
when Prism::RegularExpressionNode then ::Regexp.new(argument.unescaped, argument.options)
|
|
43
|
+
when Prism::ConstantReadNode then native.const_get(argument.name)
|
|
44
|
+
else raise "unsupported upstream scan argument"
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
node.compact_child_nodes.each { |child| visit.call(child) }
|
|
48
|
+
end
|
|
49
|
+
visit.call(method)
|
|
50
|
+
raise "upstream lex_token scan layout changed" unless scans.length == 12
|
|
51
|
+
|
|
52
|
+
scans = scans.drop(4)
|
|
53
|
+
scans[0] = ::Regexp.union(native::SYMBOLS.map { |symbol| symbol.gsub(/\\(.)/, '\1') })
|
|
54
|
+
scans[1] = ::Regexp.union(native::PERCENT_TOKENS)
|
|
55
|
+
scans.zip(%i[SYMBOL DIRECTIVE SYMBOL TAG CHARACTER STRING INTEGER IDENTIFIER])
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def adapter(native, recognizer)
|
|
59
|
+
Class.new(native) do
|
|
60
|
+
define_method(:initialize) do |file|
|
|
61
|
+
super(file)
|
|
62
|
+
@recognizer = recognizer.new(file.text)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
define_method(:lex_token) do
|
|
66
|
+
loop do
|
|
67
|
+
if @scanner.scan(LINE_BREAK)
|
|
68
|
+
newline
|
|
69
|
+
elsif @scanner.scan(/[^\S\n]+/)
|
|
70
|
+
next
|
|
71
|
+
elsif @scanner.scan(%r{/\*})
|
|
72
|
+
lex_comment
|
|
73
|
+
elsif @scanner.scan(%r{//[^\n]*(?:\n|$)})
|
|
74
|
+
newline if @scanner.matched.end_with?("\n")
|
|
75
|
+
else
|
|
76
|
+
break
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
reset_first_position
|
|
80
|
+
return if @scanner.eos?
|
|
81
|
+
|
|
82
|
+
@recognizer.instance_variable_set(:@position, @scanner.pos)
|
|
83
|
+
kind, text = @recognizer.next_token
|
|
84
|
+
raise ::ParseError, location.generate_error_message("Unexpected token") if
|
|
85
|
+
kind == :DIRECTIVE && @scanner.string.byteslice(@recognizer.byte_pos, 1)&.match?(/[-A-Za-z0-9_.]/)
|
|
86
|
+
@scanner.pos = @recognizer.byte_pos
|
|
87
|
+
types = { TAG: :Tag, CHARACTER: :Char, STRING: :Str, INTEGER: :Int, IDENTIFIER: :Ident }
|
|
88
|
+
token_class = native::Token.const_get(types.fetch(kind, :Token))
|
|
89
|
+
value = kind == :INTEGER ? Integer(text) : text
|
|
90
|
+
type = case kind
|
|
91
|
+
when :SYMBOL, :DIRECTIVE then text
|
|
92
|
+
when :IDENTIFIER then identifier_colon? ? :IDENT_COLON : :IDENTIFIER
|
|
93
|
+
else kind
|
|
94
|
+
end
|
|
95
|
+
[type, token_class.new(s_value: value, location: location)]
|
|
96
|
+
rescue StandardError => e
|
|
97
|
+
raise unless e.class.name&.end_with?("LexError")
|
|
98
|
+
|
|
99
|
+
raise ::ParseError, location.generate_error_message("Unexpected token")
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def observe(klass, source, path)
|
|
105
|
+
original = Lrama::Lexer
|
|
106
|
+
events = []
|
|
107
|
+
recording = Class.new(klass) do
|
|
108
|
+
define_method(:next_token) do
|
|
109
|
+
super().tap do |token|
|
|
110
|
+
if token
|
|
111
|
+
type, value = token
|
|
112
|
+
loc = value.location
|
|
113
|
+
events << [type, value.s_value, loc.first_line, loc.first_column, loc.last_line, loc.last_column]
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
Lrama.send(:remove_const, :Lexer)
|
|
119
|
+
Lrama.const_set(:Lexer, recording)
|
|
120
|
+
error = nil
|
|
121
|
+
begin
|
|
122
|
+
Lrama::Parser.new(source, path).parse
|
|
123
|
+
rescue StandardError => e
|
|
124
|
+
error = [e.class.name, e.message]
|
|
125
|
+
end
|
|
126
|
+
[events, error]
|
|
127
|
+
ensure
|
|
128
|
+
Lrama.send(:remove_const, :Lexer)
|
|
129
|
+
Lrama.const_set(:Lexer, original)
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def run(root, iterations: 3, out: $stdout)
|
|
133
|
+
$LOAD_PATH.unshift(File.join(root, "lib"))
|
|
134
|
+
require "lrama"
|
|
135
|
+
native = Lrama::Lexer
|
|
136
|
+
candidate = adapter(native, recognizer(patterns(native, File.join(root, "lib/lrama/lexer.rb"))))
|
|
137
|
+
paths = Dir[File.join(root, "spec/fixtures/**/*.y")]
|
|
138
|
+
raise "no upstream fixture corpus" if paths.empty?
|
|
139
|
+
|
|
140
|
+
inputs = paths.map { |path| [path, File.read(path)] }
|
|
141
|
+
successful_parses = 0
|
|
142
|
+
tokens = 0
|
|
143
|
+
inputs.each do |path, source|
|
|
144
|
+
expected = observe(native, source, path)
|
|
145
|
+
actual = observe(candidate, source, path)
|
|
146
|
+
raise "lexer mismatch: #{path}" unless actual == expected
|
|
147
|
+
successful_parses += 1 unless expected.last
|
|
148
|
+
tokens += expected.first.length
|
|
149
|
+
end
|
|
150
|
+
timings = { native: native, standalone: candidate }.transform_values do |klass|
|
|
151
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
152
|
+
iterations.times { inputs.each { |path, source| observe(klass, source, path) } }
|
|
153
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
|
|
154
|
+
end
|
|
155
|
+
result = { ruby: RUBY_DESCRIPTION, fixtures: paths.length, successful_parses: successful_parses,
|
|
156
|
+
tokens_compared: tokens, iterations: iterations, seconds: timings,
|
|
157
|
+
scope: "normal grammar tokens; existing trivia, C-code scanning, locations, and parser retained" }
|
|
158
|
+
out.puts JSON.pretty_generate(result)
|
|
159
|
+
result
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
Flexr::Benchmarking::LramaLexer.run(ARGV.fetch(0)) if $PROGRAM_NAME == __FILE__
|
data/benchmark/matrix.rb
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "open3"
|
|
4
|
+
require "rbconfig"
|
|
5
|
+
require_relative "run"
|
|
6
|
+
|
|
7
|
+
module Flexr
|
|
8
|
+
module Benchmarking
|
|
9
|
+
class RubyMatrix
|
|
10
|
+
ROOT = File.expand_path("..", __dir__)
|
|
11
|
+
FLAGS = { "interpreter" => ["--disable=yjit,zjit"], "yjit" => ["--yjit"], "zjit" => ["--zjit"] }.freeze
|
|
12
|
+
PROBE = <<~RUBY.lines.map(&:strip).join(";").freeze
|
|
13
|
+
require "json"
|
|
14
|
+
engine = ARGV.fetch(0)
|
|
15
|
+
enabled = case engine
|
|
16
|
+
when "interpreter"
|
|
17
|
+
!(defined?(RubyVM::YJIT) && RubyVM::YJIT.enabled?) && !(defined?(RubyVM::ZJIT) && RubyVM::ZJIT.enabled?)
|
|
18
|
+
when "yjit" then defined?(RubyVM::YJIT) && RubyVM::YJIT.enabled?
|
|
19
|
+
when "zjit" then defined?(RubyVM::ZJIT) && RubyVM::ZJIT.enabled?
|
|
20
|
+
end
|
|
21
|
+
puts JSON.generate(available: !!enabled, ruby: RUBY_DESCRIPTION, reason: enabled ? nil : "engine is not enabled by this Ruby build")
|
|
22
|
+
RUBY
|
|
23
|
+
|
|
24
|
+
def initialize(arguments: [], probe: nil, benchmark: nil)
|
|
25
|
+
@arguments = arguments
|
|
26
|
+
@probe = probe || method(:probe_engine)
|
|
27
|
+
@benchmark = benchmark || method(:benchmark_engine)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def run(engines: FLAGS.keys)
|
|
31
|
+
results = engines.to_h do |engine|
|
|
32
|
+
availability = @probe.call(engine)
|
|
33
|
+
result = if availability.fetch("available")
|
|
34
|
+
{ "status" => "measured", "ruby" => availability["ruby"], "result" => @benchmark.call(engine) }
|
|
35
|
+
else
|
|
36
|
+
{ "status" => "unavailable", "reason" => availability.fetch("reason") }
|
|
37
|
+
end
|
|
38
|
+
[engine, result]
|
|
39
|
+
end
|
|
40
|
+
{ "schema" => 1, "engines" => results }
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def self.command(argv, out: $stdout, err: $stderr)
|
|
44
|
+
options = Benchmarking.parse(argv.dup)
|
|
45
|
+
return Benchmarking.usage(out) if options[:help]
|
|
46
|
+
raise ArgumentError, "matrix results cannot share a baseline output file" if options[:write_baseline]
|
|
47
|
+
|
|
48
|
+
out.puts JSON.pretty_generate(new(arguments: argv).run)
|
|
49
|
+
0
|
|
50
|
+
rescue ArgumentError, Errno::ENOENT => e
|
|
51
|
+
err.puts "matrix error: #{e.message}"
|
|
52
|
+
2
|
|
53
|
+
rescue RuntimeError => e
|
|
54
|
+
err.puts "matrix error: #{e.message}"
|
|
55
|
+
1
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
private
|
|
59
|
+
|
|
60
|
+
def probe_engine(engine)
|
|
61
|
+
stdout, stderr, status = Open3.capture3({ "RUBYOPT" => nil }, RbConfig.ruby, *FLAGS.fetch(engine),
|
|
62
|
+
"-e", PROBE, engine, chdir: ROOT)
|
|
63
|
+
return { "available" => false, "reason" => stderr.strip } unless status.success?
|
|
64
|
+
|
|
65
|
+
JSON.parse(stdout)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def benchmark_engine(engine)
|
|
69
|
+
stdout, stderr, status = Open3.capture3({ "RUBYOPT" => nil }, RbConfig.ruby, *FLAGS.fetch(engine),
|
|
70
|
+
"-Ilib", "benchmark/run.rb", *@arguments, "--json", chdir: ROOT)
|
|
71
|
+
raise "#{engine} benchmark failed (#{status.exitstatus}): #{stderr}#{stdout}" unless status.success?
|
|
72
|
+
|
|
73
|
+
JSON.parse(stdout)
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
exit Flexr::Benchmarking::RubyMatrix.command(ARGV) if $PROGRAM_NAME == __FILE__
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "run"
|
|
4
|
+
|
|
5
|
+
module Flexr
|
|
6
|
+
module Benchmarking
|
|
7
|
+
class PhaseProfiler
|
|
8
|
+
attr_reader :seconds, :elapsed
|
|
9
|
+
|
|
10
|
+
def initialize
|
|
11
|
+
@seconds = %w[scanning actions tokens locations other].to_h { |phase| [phase, 0.0] }
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def measure(&)
|
|
15
|
+
stack = ["other"]
|
|
16
|
+
started = previous = clock
|
|
17
|
+
@trace = TracePoint.new(:call, :return, :c_call, :c_return) do |event|
|
|
18
|
+
now = clock
|
|
19
|
+
@seconds[stack.last] += now - previous
|
|
20
|
+
previous = now
|
|
21
|
+
if %i[call c_call].include?(event.event)
|
|
22
|
+
stack << (phase(event) || stack.last)
|
|
23
|
+
elsif stack.length > 1
|
|
24
|
+
stack.pop
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
@trace.enable(target_thread: Thread.current, &)
|
|
28
|
+
ensure
|
|
29
|
+
@trace&.disable
|
|
30
|
+
finished = clock
|
|
31
|
+
@seconds[stack.last] += finished - previous
|
|
32
|
+
@elapsed = finished - started
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def enabled?
|
|
36
|
+
@trace&.enabled? || false
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
private
|
|
40
|
+
|
|
41
|
+
def clock
|
|
42
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def phase(event)
|
|
46
|
+
if event.self.is_a?(Runtime::Interpreter)
|
|
47
|
+
return "scanning" if event.method_id == :scan
|
|
48
|
+
elsif event.self.is_a?(Lexer)
|
|
49
|
+
case event.method_id
|
|
50
|
+
when :scan_one then return "scanning"
|
|
51
|
+
when :execute then return "actions"
|
|
52
|
+
when :emit, :text, :next_token, :tokens then return "tokens"
|
|
53
|
+
when :last_location, :update_position, :advance_location! then return "locations"
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
nil
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
class ProfileRunner < Runner
|
|
61
|
+
def run
|
|
62
|
+
path = File.expand_path(@options.fetch(:spec))
|
|
63
|
+
input = input_for(path)
|
|
64
|
+
runtime = profile_class(load_runtime(path), input)
|
|
65
|
+
generated = profile_class(load_generated(path), input)
|
|
66
|
+
{
|
|
67
|
+
"schema" => 1, "ruby" => RUBY_DESCRIPTION,
|
|
68
|
+
"spec" => relative_path(path), "input_bytes" => input.bytesize,
|
|
69
|
+
"input_sha256" => Digest::SHA256.hexdigest(input),
|
|
70
|
+
"iterations" => @options.fetch(:iterations),
|
|
71
|
+
"note" => "TracePoint changes execution costs and disables JIT optimizations; use phase timings only to locate work, not as throughput measurements.",
|
|
72
|
+
"modes" => { "runtime" => runtime, "generated" => generated }
|
|
73
|
+
}
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
private
|
|
77
|
+
|
|
78
|
+
def profile_class(klass, input)
|
|
79
|
+
iterations = @options.fetch(:iterations)
|
|
80
|
+
klass.new(input).tokens
|
|
81
|
+
profiler = PhaseProfiler.new
|
|
82
|
+
tokens = profiler.measure do
|
|
83
|
+
result = nil
|
|
84
|
+
iterations.times { result = klass.new(input).tokens }
|
|
85
|
+
result
|
|
86
|
+
end
|
|
87
|
+
{
|
|
88
|
+
"tokens" => tokens.length,
|
|
89
|
+
"instrumented_elapsed_seconds" => profiler.elapsed,
|
|
90
|
+
"phase_seconds" => profiler.seconds
|
|
91
|
+
}
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def self.profile(argv, out: $stdout, err: $stderr)
|
|
96
|
+
options = parse(argv.dup)
|
|
97
|
+
return usage(out) if options[:help]
|
|
98
|
+
raise ArgumentError, "profiling does not support baseline checks or writes" if options[:baseline] || options[:write_baseline]
|
|
99
|
+
|
|
100
|
+
out.puts JSON.pretty_generate(ProfileRunner.new(options).run)
|
|
101
|
+
0
|
|
102
|
+
rescue ArgumentError, Errno::ENOENT => e
|
|
103
|
+
err.puts "profile error: #{e.message}"
|
|
104
|
+
2
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
exit Flexr::Benchmarking.profile(ARGV) if $PROGRAM_NAME == __FILE__
|
data/benchmark/run.rb
CHANGED
|
@@ -13,7 +13,10 @@ module Flexr
|
|
|
13
13
|
DEFAULT_SPEC = File.expand_path("../examples/json/lexer.flexr.rb", __dir__)
|
|
14
14
|
DEFAULT_INPUTS = {
|
|
15
15
|
%r{/examples/json/} => ('{"answer": 42}' * 10_000).freeze,
|
|
16
|
-
%r{/examples/toy_lang/} => ("answer + 12" * 10_000).freeze
|
|
16
|
+
%r{/examples/toy_lang/} => ("answer + 12" * 10_000).freeze,
|
|
17
|
+
%r{/examples/toml/} => ('title = "flexr" ' * 10_000).freeze,
|
|
18
|
+
%r{/examples/graphql/} => ('query { user(id: 12) { name } } ' * 10_000).freeze,
|
|
19
|
+
%r{/examples/sql/} => ('select name from users where id >= 12; ' * 10_000).freeze
|
|
17
20
|
}.freeze
|
|
18
21
|
|
|
19
22
|
module_function
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "run"
|
|
4
|
+
require_relative "../lib/flexr/cli"
|
|
5
|
+
|
|
6
|
+
module Flexr
|
|
7
|
+
module Benchmarking
|
|
8
|
+
module StateDispatch
|
|
9
|
+
module_function
|
|
10
|
+
|
|
11
|
+
def lexers(path)
|
|
12
|
+
parsed = CLI.read_spec(path)
|
|
13
|
+
compiled = CLI.compile(parsed, overrides: { backend: :table })
|
|
14
|
+
indent = Source::Passthrough.indentation(parsed.source, parsed.first_dsl_offset)
|
|
15
|
+
scanner = Codegen::Table.new(compiled).source(indent: indent)
|
|
16
|
+
replacement = scanner.gsub("dfa.transition(current, byte)", "__flexr_state_transition(current, ec[byte])")
|
|
17
|
+
raise "generated scanner has no table dispatch" if replacement == scanner
|
|
18
|
+
|
|
19
|
+
replacement += dispatch_source(compiled, indent)
|
|
20
|
+
generated = Generator.new(path, options: { backend: :table }).generate
|
|
21
|
+
raise "generated scanner template changed" unless generated.include?(scanner)
|
|
22
|
+
|
|
23
|
+
[generated, Generator.new(path, options: { backend: :direct }).generate,
|
|
24
|
+
generated.sub(scanner, replacement)].map do |source|
|
|
25
|
+
scope = Module.new
|
|
26
|
+
scope.module_eval(source, path)
|
|
27
|
+
parsed.class_name.split("::").reject(&:empty?).inject(scope) { |parent, name| parent.const_get(name, false) }
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def dispatch_source(compiled, indent)
|
|
32
|
+
lines = ["def __flexr_state_transition(state, class_id)", "case @state"]
|
|
33
|
+
compiled.machines.each do |name, machine|
|
|
34
|
+
lines << "when #{name.inspect}" << "case state"
|
|
35
|
+
machine.dfa.transitions.each_with_index do |row, state|
|
|
36
|
+
if row.compact.empty?
|
|
37
|
+
lines << "when #{state} then nil"
|
|
38
|
+
next
|
|
39
|
+
end
|
|
40
|
+
lines << "when #{state}" << "case class_id"
|
|
41
|
+
row.each_with_index.group_by(&:first).each do |destination, pairs|
|
|
42
|
+
next unless destination
|
|
43
|
+
|
|
44
|
+
lines << "when #{pairs.map(&:last).join(', ')} then #{destination}"
|
|
45
|
+
end
|
|
46
|
+
lines << "else nil" << "end"
|
|
47
|
+
end
|
|
48
|
+
lines << "else nil" << "end"
|
|
49
|
+
end
|
|
50
|
+
(lines + ["else nil", "end", "end"]).map { |line| "#{indent}#{line}\n" }.join
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def run(argv, out: $stdout)
|
|
54
|
+
options = Benchmarking.parse(argv.dup)
|
|
55
|
+
return Benchmarking.usage(out) if options[:help]
|
|
56
|
+
raise ArgumentError, "--baseline and --write-baseline do not apply to this experiment" if options[:baseline] || options[:write_baseline]
|
|
57
|
+
|
|
58
|
+
path = File.expand_path(options.fetch(:spec))
|
|
59
|
+
input = options[:input_file] ? File.binread(options[:input_file]) : Benchmarking::DEFAULT_INPUTS.find { |pattern, _| path.match?(pattern) }&.last
|
|
60
|
+
raise ArgumentError, "--input-file is required for this spec" unless input
|
|
61
|
+
|
|
62
|
+
table, direct, switched = lexers(path)
|
|
63
|
+
expected = table.new(input).tokens
|
|
64
|
+
raise "state dispatch changed tokens" unless [direct, switched].all? { |lexer| lexer.new(input).tokens == expected }
|
|
65
|
+
|
|
66
|
+
runner = Runner.new(options)
|
|
67
|
+
result = { ruby: RUBY_DESCRIPTION, input_bytes: input.bytesize, iterations: options[:iterations],
|
|
68
|
+
modes: { table: runner.send(:measure_class, table, input),
|
|
69
|
+
direct: runner.send(:measure_class, direct, input),
|
|
70
|
+
state_cases: runner.send(:measure_class, switched, input) } }
|
|
71
|
+
out.puts JSON.pretty_generate(result)
|
|
72
|
+
result
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
Flexr::Benchmarking::StateDispatch.run(ARGV) if $PROGRAM_NAME == __FILE__
|
data/docs/README.md
CHANGED
|
@@ -61,5 +61,4 @@ pages explain how to use those specifications.
|
|
|
61
61
|
|
|
62
62
|
The [performance log](perf-log.md), [architecture decisions](adr/),
|
|
63
63
|
[internals notes](internals/README.md), and [release procedure](RELEASING.md)
|
|
64
|
-
are maintainer-facing documents.
|
|
65
|
-
[releases](releases/v1.1.0.md).
|
|
64
|
+
are maintainer-facing documents.
|
data/docs/RELEASING.md
CHANGED
|
@@ -18,7 +18,8 @@ output as SVG and should be run before a release that changes CLI visualization.
|
|
|
18
18
|
|
|
19
19
|
Then:
|
|
20
20
|
|
|
21
|
-
1. Review the commits since the previous release
|
|
21
|
+
1. Review the commits since the previous release, add a dated version section
|
|
22
|
+
to [the changelog](../CHANGELOG.md), and update `lib/flexr/version.rb`.
|
|
22
23
|
2. Verify the gemspec metadata URLs.
|
|
23
24
|
3. Build and inspect the package with `gem build flexr.gemspec`.
|
|
24
25
|
4. Confirm generated golden files and the Unicode snapshot are intentional.
|
|
@@ -26,5 +27,5 @@ Then:
|
|
|
26
27
|
|
|
27
28
|
Unicode snapshot changes are minor-version compatibility changes because they
|
|
28
29
|
can change generated output. Stable runtime and generated-file contracts are
|
|
29
|
-
preserved within a major version unless this document and the
|
|
30
|
+
preserved within a major version unless this document and the changelog
|
|
30
31
|
state a migration.
|
|
@@ -34,3 +34,20 @@ compression and packed Base64 output trade source size against loading cost;
|
|
|
34
34
|
packed transitions are queried without creating a second dense table, and a
|
|
35
35
|
dense row is materialized lazily only for inspection. These choices do not
|
|
36
36
|
change token streams.
|
|
37
|
+
|
|
38
|
+
Explicit `accel :strscan` also derives complete-token expressions from the DFA.
|
|
39
|
+
For each starting-byte branch, all accepting states must select the same rule.
|
|
40
|
+
The reachable graph may have self-loops but no other cycles, and every edge
|
|
41
|
+
must consume ASCII bytes. Outgoing byte sets are disjoint because they come
|
|
42
|
+
from deterministic transitions. A greedy self-loop cannot consume an exit
|
|
43
|
+
byte; greedy optional exits therefore find the longest accepted prefix without
|
|
44
|
+
adopting the original Ruby regexp's alternation priority. Unproven branches
|
|
45
|
+
fall back to DFA traversal.
|
|
46
|
+
|
|
47
|
+
Expression expansion is capped at 32 states per path and 4 KiB per expression.
|
|
48
|
+
Plans are shared by lexer instances through their compiled machine. Native
|
|
49
|
+
scans use `StringScanner#skip`, retaining byte lengths without materializing
|
|
50
|
+
matched text. The fast path is disabled for mutable or streaming input,
|
|
51
|
+
non-retained buffers, token/step/cancellation guards, anchors, trailing context,
|
|
52
|
+
and `firstmatch`. These restrictions preserve the DFA's boundary and resource
|
|
53
|
+
checks.
|
|
@@ -16,7 +16,7 @@ can be loaded by a Ruby process that does not have the flexr gem installed:
|
|
|
16
16
|
ruby -e 'load ARGV.fetch(0); p Lexer.new("12").tokens' lexer_standalone.rb
|
|
17
17
|
```
|
|
18
18
|
|
|
19
|
-
Generation still needs the flexr gem and Prism on Ruby 3.
|
|
19
|
+
Generation still needs the flexr gem and Prism on Ruby 3.4 or newer. Runtime
|
|
20
20
|
execution needs the generated file and Ruby's standard library only.
|
|
21
21
|
|
|
22
22
|
Standalone output is larger and duplicates the runtime for every artifact.
|
|
@@ -13,11 +13,23 @@ application startup. `backend :auto` compares dense memory, packed memory, and
|
|
|
13
13
|
lookup costs rather than using a fixed table-size threshold; `:table` and
|
|
14
14
|
`:direct` preserve the same longest-match semantics.
|
|
15
15
|
|
|
16
|
+
Use `emit:` for tokens whose value is the unchanged matched text. For example,
|
|
17
|
+
`rule(/:/, emit: :COLON)` avoids invoking an action block for every separator.
|
|
18
|
+
Keep blocks for value conversions and state transitions. The JSON example
|
|
19
|
+
uses this form for punctuation.
|
|
20
|
+
|
|
16
21
|
Acceleration is an optimization over the DFA, not a second matcher. `accel
|
|
17
22
|
:auto` selects an available safe path, while `:strscan`, `:regexp`, and `:none`
|
|
18
23
|
make the choice explicit. Trailing context disables region acceleration for the
|
|
19
24
|
affected rule and emits `FLEXR-W012` unless acceleration is disabled.
|
|
20
25
|
|
|
26
|
+
Explicit `accel :strscan` can also scan a complete token in one native call
|
|
27
|
+
when a conservative DFA check proves it safe. This path needs frozen String
|
|
28
|
+
input with retained bytes and no active scanning or token-size guard. Streaming
|
|
29
|
+
and mutable input, anchors, trailing context, and unproven patterns keep the DFA
|
|
30
|
+
path. Measure with your actual token lengths and Ruby/JIT configuration before
|
|
31
|
+
choosing `:strscan`; `:auto` keeps its region-selection policy.
|
|
32
|
+
|
|
21
33
|
The [performance log](../perf-log.md) records reproducible local observations,
|
|
22
34
|
and the [backend explanation](../explanation/backends.md) describes the memory
|
|
23
35
|
and compatibility trade-offs.
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# Migrating to flexr 2.0
|
|
2
|
+
|
|
3
|
+
This guide describes the v2 prerelease. The preceding v1 preparation release
|
|
4
|
+
keeps existing behavior and offers opt-in deprecation warnings. In v2 the
|
|
5
|
+
contracts below are enforced and those preparation warnings are removed.
|
|
6
|
+
|
|
7
|
+
Run static checks before upgrading:
|
|
8
|
+
|
|
9
|
+
```sh
|
|
10
|
+
flexr check lexer.flexr.rb --target 2.0
|
|
11
|
+
flexr check lexer.flexr.rb --target 2.0 --format json
|
|
12
|
+
RUBYOPT=-W:deprecated ruby application.rb
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
`--target 2.0` reports the new requirements and identifies structured-token
|
|
16
|
+
declarations. It includes migration advice even when ordinary compiler
|
|
17
|
+
warnings are disabled with `--warn none`. `--warn-as-error` fails for migration
|
|
18
|
+
warnings. The mode uses the static reader and cannot be combined with `--eval`.
|
|
19
|
+
It does not execute the specification or scan consumer files, dependencies,
|
|
20
|
+
or metaprogramming; a clean result does not prove the application is ready.
|
|
21
|
+
|
|
22
|
+
## Ruby version
|
|
23
|
+
|
|
24
|
+
Use Ruby 3.4 or newer for the runtime, generator, and standalone lexers. Upgrade
|
|
25
|
+
deployment images and development tools before upgrading flexr. Loading v1 on
|
|
26
|
+
an older Ruby emits a notice when deprecated-category warnings are enabled.
|
|
27
|
+
|
|
28
|
+
## Immutable tokens and locations
|
|
29
|
+
|
|
30
|
+
With `token_kind :struct`, tokens and locations become `Data` records. The
|
|
31
|
+
option name stays `:struct` to avoid changing lexer declarations. Attribute
|
|
32
|
+
readers, keyword construction, and read-only indexing remain available.
|
|
33
|
+
Field setters, `[]=`, and checks that require a `Struct` instance must change.
|
|
34
|
+
The default array token format is unaffected.
|
|
35
|
+
|
|
36
|
+
Replace mutation with a new value:
|
|
37
|
+
|
|
38
|
+
```ruby
|
|
39
|
+
# Before
|
|
40
|
+
token.value = replacement
|
|
41
|
+
|
|
42
|
+
# After upgrading to v2
|
|
43
|
+
token = token.with(value: replacement)
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Use attribute readers or `to_h` rather than relying on Struct-specific
|
|
47
|
+
enumeration or the raw storage of lazy location columns. Column readers
|
|
48
|
+
remain available without retaining the lexer. Immutability is shallow: a
|
|
49
|
+
mutable object emitted as a token value remains the application's responsibility.
|
|
50
|
+
When deprecated-category warnings are enabled, v1 warns on record field
|
|
51
|
+
assignments and `[]=` calls. It does not detect mutations of nested values or
|
|
52
|
+
every Struct mutator, so inspect consumer code and tests as well.
|
|
53
|
+
|
|
54
|
+
## Error policy
|
|
55
|
+
|
|
56
|
+
Only `error_mode: :raise`, `:token`, and `:panic` are valid in v2. An unknown
|
|
57
|
+
value raises `ArgumentError` during initialization, including for empty or
|
|
58
|
+
otherwise matching input. v1 warns at initialization when deprecation warnings
|
|
59
|
+
are enabled, while preserving its existing fallback behavior.
|
|
60
|
+
|
|
61
|
+
## Generated artifacts
|
|
62
|
+
|
|
63
|
+
Regenerate every lexer with the new generator after upgrading. Artifact schema
|
|
64
|
+
and runtime ABI change to version 2; loading an incompatible old artifact
|
|
65
|
+
produces `FLEXR-E020` with regeneration guidance. Standalone artifacts bundle
|
|
66
|
+
their runtime, so regenerate them as part of deployment as well. v1 reports an
|
|
67
|
+
opt-in warning when loading an ABI 1 artifact. Do not edit artifact metadata by
|
|
68
|
+
hand to bypass compatibility checks.
|
|
69
|
+
|
|
70
|
+
## Backends
|
|
71
|
+
|
|
72
|
+
`firstmatch` remains available with its explicit first-matching-rule semantics;
|
|
73
|
+
it does not acquire the longest-match behavior of `table`, `direct`, and `auto`.
|
|
74
|
+
No backend removal is required by this migration.
|