flexr 1.0.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +241 -72
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/RELEASING.md +3 -2
  25. data/docs/explanation/backends.md +30 -10
  26. data/docs/explanation/matching-semantics.md +4 -2
  27. data/docs/explanation/security-model.md +10 -3
  28. data/docs/explanation/unicode-and-encoding.md +3 -1
  29. data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
  30. data/docs/how-to/generate-a-lexer.md +3 -3
  31. data/docs/how-to/handle-errors.md +4 -1
  32. data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
  33. data/docs/how-to/track-token-locations.md +2 -1
  34. data/docs/how-to/tune-performance.md +15 -3
  35. data/docs/migration-v2.md +74 -0
  36. data/docs/perf-log.md +212 -0
  37. data/docs/reference/README.md +2 -0
  38. data/docs/reference/actions.md +5 -2
  39. data/docs/reference/cli.md +71 -5
  40. data/docs/reference/compatibility.md +10 -4
  41. data/docs/reference/diagnostics.md +346 -0
  42. data/docs/reference/dsl.md +30 -9
  43. data/docs/reference/errors.md +34 -3
  44. data/docs/reference/generated-artifacts.md +71 -6
  45. data/docs/reference/public-api.md +37 -7
  46. data/docs/reference/regexp.md +27 -13
  47. data/docs/reference/ruby-lsp.md +34 -0
  48. data/docs/reference/runtime.md +21 -4
  49. data/docs/reference/testing.md +57 -0
  50. data/docs/reference/tokens-and-locations.md +14 -4
  51. data/examples/graphql/README.md +14 -0
  52. data/examples/graphql/lexer.flexr.rb +29 -0
  53. data/examples/json/lexer.flexr.rb +6 -6
  54. data/examples/sql/README.md +16 -0
  55. data/examples/sql/lexer.flexr.rb +23 -0
  56. data/examples/toml/README.md +14 -0
  57. data/examples/toml/lexer.flexr.rb +23 -0
  58. data/gemfiles/ruby_lsp.gemfile +4 -0
  59. data/gemfiles/types.gemfile +7 -0
  60. data/lib/flexr/action_resolver.rb +15 -0
  61. data/lib/flexr/artifact_writer.rb +61 -0
  62. data/lib/flexr/automaton/accel.rb +78 -5
  63. data/lib/flexr/automaton/analysis.rb +202 -7
  64. data/lib/flexr/automaton/backend_cost_model.rb +40 -0
  65. data/lib/flexr/automaton/compiler.rb +229 -96
  66. data/lib/flexr/automaton/dfa.rb +106 -19
  67. data/lib/flexr/automaton/minimizer.rb +79 -31
  68. data/lib/flexr/automaton/nfa.rb +45 -13
  69. data/lib/flexr/automaton/types.rb +14 -0
  70. data/lib/flexr/cli.rb +128 -22
  71. data/lib/flexr/codegen/direct.rb +5 -41
  72. data/lib/flexr/codegen/firstmatch.rb +1 -7
  73. data/lib/flexr/codegen/table.rb +181 -69
  74. data/lib/flexr/codegen.rb +8 -0
  75. data/lib/flexr/configuration.rb +45 -0
  76. data/lib/flexr/diagnostics.rb +39 -3
  77. data/lib/flexr/dsl.rb +116 -47
  78. data/lib/flexr/errors.rb +7 -6
  79. data/lib/flexr/generated.rb +120 -46
  80. data/lib/flexr/generator.rb +102 -109
  81. data/lib/flexr/grammar.rb +41 -0
  82. data/lib/flexr/importer.rb +4 -40
  83. data/lib/flexr/ir.rb +2 -2
  84. data/lib/flexr/migration.rb +31 -0
  85. data/lib/flexr/minitest.rb +24 -0
  86. data/lib/flexr/options.rb +9 -5
  87. data/lib/flexr/rake_task.rb +6 -1
  88. data/lib/flexr/regexp/ast.rb +5 -2
  89. data/lib/flexr/regexp/matcher.rb +26 -0
  90. data/lib/flexr/regexp/normalizer.rb +63 -21
  91. data/lib/flexr/regexp/parser.rb +241 -60
  92. data/lib/flexr/regexp/tokenizer.rb +172 -47
  93. data/lib/flexr/rspec.rb +51 -0
  94. data/lib/flexr/runtime/buffer.rb +115 -9
  95. data/lib/flexr/runtime/core.rb +248 -51
  96. data/lib/flexr/runtime/errors.rb +66 -4
  97. data/lib/flexr/runtime/interpreter.rb +213 -97
  98. data/lib/flexr/runtime/location.rb +35 -12
  99. data/lib/flexr/runtime/record.rb +14 -0
  100. data/lib/flexr/runtime/token.rb +7 -1
  101. data/lib/flexr/runtime.rb +75 -0
  102. data/lib/flexr/source/passthrough.rb +70 -14
  103. data/lib/flexr/source/prism_reader.rb +217 -51
  104. data/lib/flexr/source/static_eval.rb +40 -4
  105. data/lib/flexr/source.rb +6 -0
  106. data/lib/flexr/testing.rb +47 -0
  107. data/lib/flexr/trace.rb +197 -0
  108. data/lib/flexr/unicode/data/properties.rb +1 -1
  109. data/lib/flexr/unicode/data.rb +11 -0
  110. data/lib/flexr/unicode/property.rb +4 -8
  111. data/lib/flexr/unicode/reference_regexp.rb +11 -9
  112. data/lib/flexr/unicode/version.rb +7 -0
  113. data/lib/flexr/version.rb +1 -1
  114. data/lib/flexr.rb +10 -76
  115. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  116. data/sig/flexr.rbs +191 -0
  117. data/site/README.md +54 -3
  118. data/site/astro.config.mjs +3 -1
  119. data/site/package.json +12 -5
  120. data/site/playwright.config.mjs +16 -0
  121. data/site/pnpm-lock.yaml +218 -88
  122. data/site/ruby/playground.rb +57 -0
  123. data/site/scripts/prepare-playground.mjs +35 -0
  124. data/site/scripts/verify-site.mjs +5 -3
  125. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  126. data/site/src/content/docs/concepts/security-model.md +9 -1
  127. data/site/src/content/docs/learn/generation.md +1 -1
  128. data/site/src/content/docs/reference/dsl.md +2 -2
  129. data/site/src/content/docs/reference/public-api.md +4 -2
  130. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  131. data/site/src/layouts/SiteLayout.astro +11 -3
  132. data/site/src/pages/index.astro +69 -155
  133. data/site/src/pages/playground.astro +40 -38
  134. data/site/src/playground/client.js +211 -0
  135. data/site/src/playground/graph-worker.mjs +11 -0
  136. data/site/src/playground/presets.mjs +33 -0
  137. data/site/src/playground/runtime.mjs +34 -0
  138. data/site/src/playground/share.mjs +30 -0
  139. data/site/src/playground/worker.mjs +29 -0
  140. data/site/src/styles/custom.css +12 -287
  141. data/site/src/styles/landing.css +239 -0
  142. data/site/tests/browser/playground.spec.mjs +99 -0
  143. data/site/tests/playground-runtime.test.mjs +85 -0
  144. data/test/ruby_lsp_addon_spec.rb +135 -0
  145. data/tools/coverage.rb +6 -1
  146. data/tools/docs_verify.rb +3 -2
  147. data/tools/golden.rb +54 -0
  148. data/tools/verification_failure.rb +119 -0
  149. metadata +66 -12
  150. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  151. data/benchmark/golden/json_lexer.sha256 +0 -1
  152. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  153. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  154. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  155. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  156. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  157. data/site/public/playground.js +0 -189
@@ -6,65 +6,113 @@ module Flexr
6
6
  module_function
7
7
 
8
8
  def minimize(dfa)
9
- partitions = initial_partitions(dfa)
10
- loop do
11
- groups = group_ids(partitions)
12
- refined = partitions.flat_map do |partition|
13
- partition.group_by do |state|
14
- [dfa.accepts[state], dfa.transitions[state].map { |destination| destination.nil? ? -1 : groups[destination] }]
15
- end.values
9
+ dead = dfa.states
10
+ rows = dfa.transitions.map { |row| row.map { |destination| destination.nil? ? dead : destination } }
11
+ rows << Array.new(dfa.class_count, dead)
12
+ partitions = initial_partitions(dfa, dead)
13
+ predecessors = predecessor_index(rows, dfa.class_count)
14
+ partitions = refine(partitions, predecessors, dfa.class_count)
15
+ rebuild(dfa, partitions, dead)
16
+ end
17
+
18
+ def initial_partitions(dfa, dead)
19
+ (0..dead).group_by { |state| state == dead ? [] : dfa.accepts[state] }.values
20
+ end
21
+
22
+ def predecessor_index(rows, class_count)
23
+ Array.new(class_count) { Array.new(rows.length) { [] } }.tap do |index|
24
+ rows.each_with_index do |row, state|
25
+ row.each_with_index { |destination, class_id| index[class_id][destination] << state }
16
26
  end
17
- break if refined == partitions
27
+ end
28
+ end
18
29
 
19
- partitions = refined
30
+ def refine(partitions, predecessors, class_count)
31
+ worklist = partitions.dup
32
+ pending = {}.compare_by_identity
33
+ partitions.each { |partition| pending[partition] = true }
34
+ cursor = 0
35
+ while cursor < worklist.length
36
+ splitter = worklist.fetch(cursor)
37
+ cursor += 1
38
+ next unless pending.delete(splitter)
39
+
40
+ class_count.times do |class_id|
41
+ marked = splitter.each_with_object({}) do |state, result|
42
+ predecessors[class_id][state].each { |predecessor| result[predecessor] = true }
43
+ end
44
+ next if marked.empty?
45
+
46
+ partitions = split_partitions(partitions, marked, worklist, pending)
47
+ end
20
48
  end
21
- rebuild(dfa, partitions)
49
+ partitions
22
50
  end
23
51
 
24
- def initial_partitions(dfa)
25
- (0...dfa.states).group_by { |state| dfa.accepts[state] }.values
52
+ def split_partitions(partitions, marked, worklist, pending)
53
+ partitions.flat_map do |partition|
54
+ inside, outside = partition.partition { |state| marked[state] }
55
+ next [partition] if inside.empty? || outside.empty?
56
+
57
+ if pending.delete(partition)
58
+ enqueue(inside, worklist, pending)
59
+ enqueue(outside, worklist, pending)
60
+ else
61
+ enqueue(inside.length <= outside.length ? inside : outside, worklist, pending)
62
+ end
63
+ [inside, outside]
64
+ end
26
65
  end
27
66
 
28
- def group_ids(partitions)
29
- ids = {}
30
- partitions.each_with_index { |partition, id| partition.each { |state| ids[state] = id } }
31
- ids
67
+ def enqueue(partition, worklist, pending)
68
+ worklist << partition
69
+ pending[partition] = true
32
70
  end
33
71
 
34
- def rebuild(dfa, partitions)
72
+ def rebuild(dfa, partitions, dead)
35
73
  groups = group_ids(partitions)
74
+ dead_group = groups.fetch(dead)
36
75
  start_group = groups.fetch(dfa.start)
37
- order = bfs_groups(dfa, groups, start_group)
76
+ representatives = partitions.map { |partition| partition.find { |state| state != dead } }
77
+ order = reachable_groups(dfa, groups, representatives, start_group, dead_group)
38
78
  index = order.each_with_index.to_h
39
79
  transitions = order.map do |group|
40
- representative = partitions[group].first
41
- dfa.transitions[representative].map { |destination| destination.nil? ? nil : index[groups[destination]] }
80
+ representative = representatives.fetch(group)
81
+ dfa.transitions.fetch(representative).map do |destination|
82
+ next nil if destination.nil? || groups.fetch(destination) == dead_group
83
+
84
+ index.fetch(groups.fetch(destination))
85
+ end
42
86
  end
43
- accepts = order.map { |group| dfa.accepts[partitions[group].first] }
87
+ accepts = order.map { |group| dfa.accepts.fetch(representatives.fetch(group)) }
44
88
  rule_ids = accepts.flatten.map(&:rule_index).uniq.sort
45
89
  DFA.new(transitions: transitions, accepts: accepts, ec: dfa.ec, class_count: dfa.class_count,
46
90
  start: 0, rule_ids: rule_ids)
47
91
  end
48
92
 
49
- def bfs_groups(dfa, groups, start_group)
93
+ def group_ids(partitions)
94
+ partitions.each_with_index.with_object({}) do |(partition, group), result|
95
+ partition.each { |state| result[state] = group }
96
+ end
97
+ end
98
+
99
+ def reachable_groups(dfa, groups, representatives, start_group, dead_group)
50
100
  result = []
51
101
  queue = [start_group]
52
102
  seen = {}
53
- until queue.empty?
54
- group = queue.shift
55
- next if seen[group]
103
+ cursor = 0
104
+ while cursor < queue.length
105
+ group = queue.fetch(cursor)
106
+ cursor += 1
107
+ next if seen[group] || (group == dead_group && group != start_group)
56
108
 
57
109
  seen[group] = true
58
110
  result << group
59
- representative = dfa.transitions[group_members(dfa, groups, group).first]
60
- representative.compact.each { |destination| queue << groups[destination] }
111
+ representative = representatives.fetch(group)
112
+ dfa.transitions.fetch(representative).compact.each { |destination| queue << groups.fetch(destination) }
61
113
  end
62
114
  result
63
115
  end
64
-
65
- def group_members(dfa, groups, group)
66
- (0...dfa.states).select { |state| groups[state] == group }
67
- end
68
116
  end
69
117
  end
70
118
  end
@@ -4,11 +4,6 @@ module Flexr
4
4
  module Automaton
5
5
  NFAState = Struct.new(:epsilon, :transitions, :accepts, keyword_init: true)
6
6
  NFATransition = Struct.new(:lo, :hi, :to, keyword_init: true)
7
- Acceptance = Struct.new(:rule_index, :pattern_index, :bol_only, :end_anchor, keyword_init: true) do
8
- def inspect
9
- [rule_index, pattern_index, bol_only, end_anchor].inspect
10
- end
11
- end
12
7
 
13
8
  class NFA
14
9
  attr_reader :states, :start, :byte_classes
@@ -56,6 +51,8 @@ module Flexr
56
51
  when Regexp::AST::Empty
57
52
  state = @nfa.new_state
58
53
  [state, state]
54
+ when Regexp::AST::Fail
55
+ [@nfa.new_state, @nfa.new_state]
59
56
  when Regexp::AST::ByteRange
60
57
  from = @nfa.new_state
61
58
  to = @nfa.new_state
@@ -75,18 +72,53 @@ module Flexr
75
72
  end
76
73
  [from, to]
77
74
  when Regexp::AST::Star
78
- from = @nfa.new_state
79
- to = @nfa.new_state
80
- child_start, child_end = fragment(node.child)
81
- @nfa.epsilon(from, to)
82
- @nfa.epsilon(from, child_start)
83
- @nfa.epsilon(child_end, child_start)
84
- @nfa.epsilon(child_end, to)
85
- [from, to]
75
+ repeat_fragment(node.child, 0, nil)
76
+ when Regexp::AST::Repeat
77
+ repeat_fragment(node.child, node.minimum, node.maximum)
86
78
  else
87
79
  raise CompileError, "cannot build NFA from #{node.class}"
88
80
  end
89
81
  end
82
+
83
+ def repeat_fragment(child, minimum, maximum)
84
+ return fragment(Regexp::AST::Empty.new(loc: nil)) if maximum&.zero?
85
+ return star_fragment(child) if minimum.zero? && maximum.nil?
86
+
87
+ from = @nfa.new_state
88
+ cursor = from
89
+ last_start = nil
90
+ minimum.times do
91
+ child_start, child_end = fragment(child)
92
+ @nfa.epsilon(cursor, child_start)
93
+ cursor = child_end
94
+ last_start = child_start
95
+ end
96
+ if maximum.nil?
97
+ @nfa.epsilon(cursor, last_start)
98
+ return [from, cursor]
99
+ end
100
+
101
+ (maximum - minimum).times do
102
+ child_start, child_end = fragment(child)
103
+ next_cursor = @nfa.new_state
104
+ @nfa.epsilon(cursor, next_cursor)
105
+ @nfa.epsilon(cursor, child_start)
106
+ @nfa.epsilon(child_end, next_cursor)
107
+ cursor = next_cursor
108
+ end
109
+ [from, cursor]
110
+ end
111
+
112
+ def star_fragment(child)
113
+ from = @nfa.new_state
114
+ to = @nfa.new_state
115
+ child_start, child_end = fragment(child)
116
+ @nfa.epsilon(from, to)
117
+ @nfa.epsilon(from, child_start)
118
+ @nfa.epsilon(child_end, child_start)
119
+ @nfa.epsilon(child_end, to)
120
+ [from, to]
121
+ end
90
122
  end
91
123
  end
92
124
  end
@@ -0,0 +1,14 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Flexr
4
+ module Automaton
5
+ Acceptance = Struct.new(:rule_index, :pattern_index, :bol_only, :end_anchor, keyword_init: true) do
6
+ def inspect
7
+ [rule_index, pattern_index, bol_only, end_anchor].inspect
8
+ end
9
+ end
10
+
11
+ CompiledSpec = Struct.new(:machines, :rules, :states, :stats, :diagnostics, keyword_init: true)
12
+ Machine = Struct.new(:dfa, :state_name, :whole_tokens, keyword_init: true)
13
+ end
14
+ end
data/lib/flexr/cli.rb CHANGED
@@ -1,5 +1,12 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative "runtime" unless defined?(Flexr::Lexer)
4
+ require_relative "artifact_writer"
5
+ require_relative "generator"
6
+ require_relative "importer"
7
+ require_relative "options"
8
+ require_relative "migration"
9
+
3
10
  module Flexr
4
11
  module CLI
5
12
  COMMANDS = %w[check stats tokens dot explain trace bench import].freeze
@@ -15,17 +22,19 @@ module Flexr
15
22
  return version(out) if args.delete("--version")
16
23
 
17
24
  command = COMMANDS.include?(args.first) ? args.shift.to_sym : :generate
18
- options, output, rule_number, benchmark_args, positionals = parse(args, command)
25
+ options, output, rule_number, benchmark_args, positionals, trace_args = parse(args, command)
26
+ raise ArgumentError, "lrama format is only valid for tokens" if options.format == :lrama && command != :tokens
19
27
  spec = positionals.shift
20
28
  raise ArgumentError, "a SPEC.rb path is required" unless spec
21
29
  raise ArgumentError, "unexpected argument: #{positionals.first}" unless positionals.empty?
22
30
 
23
- execute(command, spec, options, output, rule_number, benchmark_args, out, err)
31
+ execute(command, spec, options, output, rule_number, benchmark_args, out, err, trace_args: trace_args)
24
32
  rescue ArgumentError => e
25
33
  err.puts "error: #{e.message}"
26
34
  usage(err, status: EXIT_USAGE)
27
35
  rescue Flexr::Error, Errno::ENOENT, Errno::EACCES, SyntaxError => e
28
- err.puts render_error(e, options: options)
36
+ err.puts render_error(e, options: options, path: spec,
37
+ normalize_source: command == :generate || (command == :check && options&.eval_mode))
29
38
  EXIT_FAILURE
30
39
  rescue StandardError => e
31
40
  err.puts "error: #{e.class}: #{e.message}"
@@ -38,6 +47,7 @@ module Flexr
38
47
  rule_number = nil
39
48
  benchmark_args = []
40
49
  positionals = []
50
+ trace_args = {}
41
51
  parsing_options = true
42
52
 
43
53
  until args.empty?
@@ -78,10 +88,32 @@ module Flexr
78
88
  options.color = required_argument!(args, argument).to_sym
79
89
  when "--format"
80
90
  options.format = required_argument!(args, argument).to_sym
91
+ when "--target"
92
+ raise ArgumentError, "--target is only valid for the check command" unless command == :check
93
+ options.target = required_argument!(args, argument)
81
94
  when "--rule"
82
95
  rule_number = Integer(required_argument!(args, argument), 10)
83
96
  raise ArgumentError, "--rule must be non-negative" if rule_number.negative?
84
- when "--input-file", "--baseline", "--iterations"
97
+ when "--grammar"
98
+ raise ArgumentError, "--grammar is only valid for check" unless command == :check
99
+ options.grammar = required_argument!(args, argument)
100
+ when "--input", "--generated"
101
+ raise ArgumentError, "#{argument} is only valid for the trace command" unless command == :trace
102
+ if argument == "--generated"
103
+ trace_args[:generated] = true
104
+ else
105
+ trace_args[:input] = args.shift || raise(ArgumentError, "--input requires a value")
106
+ end
107
+ when "--input-file"
108
+ value = required_argument!(args, argument)
109
+ if command == :trace
110
+ trace_args[:input_file] = value
111
+ elsif command == :bench
112
+ benchmark_args.push(argument, value)
113
+ else
114
+ raise ArgumentError, "#{argument} is only valid for the trace or bench command"
115
+ end
116
+ when "--baseline", "--iterations"
85
117
  raise ArgumentError, "#{argument} is only valid for the bench command" unless command == :bench
86
118
  benchmark_args.push(argument, required_argument!(args, argument))
87
119
  else
@@ -90,14 +122,17 @@ module Flexr
90
122
  end
91
123
 
92
124
  options.validate!
93
- [options, output, rule_number, benchmark_args, positionals]
125
+ raise ArgumentError, "choose --input or --input-file" if trace_args.key?(:input) && trace_args[:input_file]
126
+ raise ArgumentError, "--generated requires --input or --input-file" if
127
+ trace_args[:generated] && !trace_args.key?(:input) && !trace_args[:input_file]
128
+ [options, output, rule_number, benchmark_args, positionals, trace_args]
94
129
  rescue ArgumentError => e
95
130
  raise e if e.message.start_with?("unsupported ", "--rule")
96
131
 
97
132
  raise ArgumentError, "invalid option value: #{e.message}"
98
133
  end
99
134
 
100
- def execute(command, spec, options, output, rule_number, benchmark_args, out, err)
135
+ def execute(command, spec, options, output, rule_number, benchmark_args, out, err, trace_args: {})
101
136
  case command
102
137
  when :check
103
138
  check(spec, options, out)
@@ -105,14 +140,24 @@ module Flexr
105
140
  print_stats(spec, options, out)
106
141
  when :tokens
107
142
  parsed = read_spec(spec)
108
- out.puts Array(parsed.config[:declared_tokens]).join(" ")
143
+ tokens = Array(parsed.config[:declared_tokens])
144
+ if options.format == :lrama
145
+ require_relative "grammar"
146
+ out.puts Grammar.declarations(tokens)
147
+ else
148
+ out.puts tokens.join(" ")
149
+ end
109
150
  EXIT_OK
110
151
  when :dot
111
152
  print_dot(spec, options, out)
112
153
  when :explain
113
154
  print_explanation(spec, rule_number, out)
114
155
  when :trace
115
- print_trace(spec, options, out)
156
+ if trace_args.key?(:input) || trace_args[:input_file]
157
+ print_input_trace(spec, options, trace_args, out)
158
+ else
159
+ print_trace(spec, options, out)
160
+ end
116
161
  when :bench
117
162
  run_benchmark(spec, options, benchmark_args, out)
118
163
  when :import
@@ -121,13 +166,13 @@ module Flexr
121
166
  return EXIT_FAILURE unless result.complete?
122
167
 
123
168
  if output
124
- File.binwrite(output, result.source)
169
+ ArtifactWriter.write!(output, result.source, source_path: spec)
125
170
  else
126
171
  out.write(result.source)
127
172
  end
128
173
  EXIT_OK
129
174
  when :generate
130
- target = output || spec.sub(/\.flexr\.rb\z/, ".rb")
175
+ target = output || ArtifactWriter.default_generated_path(spec)
131
176
  Generator.new(spec, output: target, eval_mode: options.eval_mode,
132
177
  options: options.generator_options).generate
133
178
  EXIT_OK
@@ -140,18 +185,26 @@ module Flexr
140
185
  generator.generate
141
186
  generator.diagnostics
142
187
  else
143
- compile(read_spec(spec), overrides: options.overrides)
188
+ parsed = read_spec(spec)
189
+ compile(parsed, overrides: options.overrides)
144
190
  end
145
- diagnostics = compiled.is_a?(Array) ? compiled : Array(compiled&.diagnostics)
191
+ diagnostics = (compiled.is_a?(Array) ? compiled : Array(compiled&.diagnostics)).dup
192
+ if options.grammar
193
+ require_relative "grammar"
194
+ diagnostics.concat(Grammar.diagnostics(read_spec(spec), options.grammar))
195
+ end
146
196
  diagnostics.select! do |diagnostic|
147
- options.warn_level == :all || (options.warn_level == :default && diagnostic.code != "FLEXR-W016")
197
+ options.warn_level == :all ||
198
+ (options.warn_level == :default && diagnostic.severity != :info && diagnostic.code != "FLEXR-W016")
148
199
  end
200
+ diagnostics.concat(Migration.diagnostics(parsed)) if options.target
149
201
  set = DiagnosticSet.new
150
202
  diagnostics.each { |diagnostic| set << diagnostic }
151
203
  if options.format == :json
152
204
  out.puts set.render(format: :json)
153
205
  elsif !diagnostics.empty?
154
- out.puts set.render(format: :human, color: options.color)
206
+ out.puts set.render(format: :human, color: options.color, path: spec,
207
+ source: diagnostic_source(spec, normalize: options.eval_mode))
155
208
  end
156
209
  return EXIT_FAILURE if options.warn_as_error && diagnostics.any?(&:warning?)
157
210
 
@@ -251,6 +304,34 @@ module Flexr
251
304
  EXIT_OK
252
305
  end
253
306
 
307
+ def print_input_trace(spec, options, trace_args, out)
308
+ require_relative "trace"
309
+ parsed = read_spec(spec)
310
+ source = trace_args[:generated] ? generate(spec, options) : File.read(spec)
311
+ scope = Module.new
312
+ scope.module_eval(source, spec, 1)
313
+ namespace = parsed.class_name.start_with?("::") ? Object : scope
314
+ klass = parsed.class_name.split("::").reject(&:empty?).inject(namespace) { |parent, name| parent.const_get(name, false) }
315
+ unless trace_args[:generated]
316
+ options.overrides.each do |name, value|
317
+ klass.public_send(name, value) if %i[backend token_kind accel].include?(name)
318
+ end
319
+ klass.__flexr_config.options.merge!(options.overrides.slice(:max_dfa_states, :warn_level))
320
+ compiled = klass.compile!
321
+ if options.warn_as_error
322
+ diagnostic = Array(compiled.diagnostics).find do |item|
323
+ item.warning? && (options.warn_level == :all || item.code != "FLEXR-W016")
324
+ end
325
+ raise CompileError.new(diagnostic.message, diagnostic: diagnostic) if diagnostic
326
+ end
327
+ end
328
+ input = trace_args[:input_file] ? File.binread(trace_args[:input_file]) : trace_args.fetch(:input)
329
+ trace = Trace.new(klass.new(input, filename: trace_args[:input_file]))
330
+ trace.run
331
+ out.puts(options.format == :json ? JSON.pretty_generate(steps: trace.steps) : trace.render)
332
+ EXIT_OK
333
+ end
334
+
254
335
  def run_benchmark(spec, options, benchmark_args, out)
255
336
  require_relative "../../benchmark/run"
256
337
  args = ["--spec", spec, *benchmark_args]
@@ -275,16 +356,16 @@ module Flexr
275
356
  klass.encoding(config.fetch(:encoding, Encoding::UTF_8))
276
357
  Array(config[:declared_tokens]).each { |token| klass.emits(token) }
277
358
  config_options = config.fetch(:options, {}).merge(overrides.slice(:experimental, :allow_empty_match))
278
- config_options.each do |name, value|
279
- value ? klass.option(name) : nil
359
+ config_options.slice(*Configuration::BOOLEAN_OPTIONS).each do |name, value|
360
+ klass.option(name) if value
280
361
  end
281
362
  klass.__flexr_config.options[:max_dfa_states] = overrides[:max_dfa_states] if overrides[:max_dfa_states]
363
+ klass.__flexr_config.options[:warn_level] = overrides[:warn_level] if overrides[:warn_level]
282
364
  accel = overrides.fetch(:accel, config_options[:accel])
283
365
  klass.accel(accel) if accel
284
366
  parsed.states.each do |name, value|
285
- next if name.to_sym == :initial
286
-
287
- klass.state(name, inclusive: value[:inclusive]) { nil }
367
+ klass.state(name, inclusive: value[:inclusive]) { nil } unless name.to_sym == :initial
368
+ klass.__flexr_states.fetch(name.to_sym).location = value[:location]
288
369
  end
289
370
  parsed.rules.each { |rule| klass.__flexr_add_generated_rule(rule.to_h) }
290
371
  klass.compile!
@@ -297,10 +378,30 @@ module Flexr
297
378
  value
298
379
  end
299
380
 
300
- def render_error(error, options: nil)
381
+ def diagnostic_source(path, normalize: false, max_bytes: nil)
382
+ return unless path
383
+
384
+ source = File.binread(path, max_bytes && (max_bytes + 1))
385
+ return if max_bytes && source.bytesize > max_bytes
386
+
387
+ source.force_encoding(Encoding::UTF_8)
388
+ normalize ? source.gsub(/\r\n?/, "\n") : source
389
+ rescue SystemCallError, IOError
390
+ nil
391
+ end
392
+
393
+ def render_error(error, options: nil, path: nil, normalize_source: false)
301
394
  if error.respond_to?(:diagnostic) && error.diagnostic
395
+ if error.is_a?(LexError) || (error.is_a?(Error) && error.byte_pos)
396
+ path = error.filename
397
+ normalize_source = false
398
+ # shortcut: omit input excerpts above 64 KiB; use bounded line reads if larger inputs need excerpts.
399
+ max_bytes = 64 * 1024
400
+ end
401
+ source = diagnostic_source(path, normalize: normalize_source, max_bytes: max_bytes) if
402
+ error.diagnostic.location && options&.format != :json
302
403
  return DiagnosticSet.new.tap { |set| set << error.diagnostic }.render(
303
- format: options&.format || :human, color: options&.color || :auto
404
+ format: options&.format || :human, color: options&.color || :auto, source: source, path: path
304
405
  )
305
406
  end
306
407
 
@@ -334,7 +435,12 @@ module Flexr
334
435
  -W, --warn LEVEL all | default | none
335
436
  --warn-as-error
336
437
  --color WHEN auto | always | never
337
- --format FMT human | json
438
+ --format FMT human | json | lrama (tokens only)
439
+ --grammar PATH compare emits with grammar %token declarations (check only)
440
+ --input STRING trace input (executes SPEC actions)
441
+ --input-file PATH trace or benchmark input
442
+ --generated trace generated scanner
443
+ --target 2.0 report migration requirements (static check only)
338
444
  USAGE
339
445
  status
340
446
  end
@@ -4,49 +4,13 @@ module Flexr
4
4
  module Codegen
5
5
  class Direct < Table
6
6
  def generate(state: :initial)
7
- table = super
8
- table.merge(dispatch: :case).freeze
7
+ dfa = compiled.machines.fetch(state).dfa
8
+ {
9
+ nxt: dfa.transitions.flatten.map { |value| value || -1 }.freeze,
10
+ classes: dfa.class_count
11
+ }.freeze
9
12
  end
10
13
 
11
- def source(indent: " ")
12
- lines = []
13
- lines << "#{indent}def self.__flexr_generated_direct_transition(state_name, state, byte)"
14
- lines << "#{indent} case state_name"
15
- compiled.machines.each do |state_name, machine|
16
- lines << "#{indent} when #{state_name.inspect}"
17
- lines << "#{indent} class_id = case byte"
18
- byte_classes(machine.dfa.ec).each do |first, last, class_id|
19
- selector = first == last ? first.to_s : "#{first}..#{last}"
20
- lines << "#{indent} when #{selector} then #{class_id}"
21
- end
22
- lines << "#{indent} end"
23
- lines << "#{indent} case state"
24
- machine.dfa.transitions.each_with_index do |row, state|
25
- lines << "#{indent} when #{state}"
26
- lines << "#{indent} case class_id"
27
- row.each_with_index do |destination, class_id|
28
- value = destination.nil? ? "nil" : destination
29
- lines << "#{indent} when #{class_id} then #{value}"
30
- end
31
- lines << "#{indent} else nil"
32
- lines << "#{indent} end"
33
- end
34
- lines << "#{indent} else nil"
35
- lines << "#{indent} end"
36
- end
37
- lines << "#{indent} else nil"
38
- lines << "#{indent} end"
39
- lines << "#{indent}end"
40
- "#{lines.join("\n")}\n"
41
- end
42
-
43
- private
44
-
45
- def byte_classes(ec)
46
- ec.each_with_index.chunk_while { |left, right| right[0] == left[0] }.map do |run|
47
- [run.first[1], run.last[1], run.first[0]]
48
- end
49
- end
50
14
  end
51
15
  end
52
16
  end
@@ -3,14 +3,8 @@
3
3
  module Flexr
4
4
  module Codegen
5
5
  class Firstmatch < Base
6
- def initialize(compiled, experimental: false)
7
- raise CompileError, "firstmatch requires option :experimental" unless experimental
8
-
9
- super(compiled)
10
- end
11
-
12
6
  def generate
13
- compiled.rules.map { |rule| rule.patterns }.flatten.freeze
7
+ compiled.rules.flat_map(&:patterns).freeze
14
8
  end
15
9
  end
16
10
  end