flexr 1.0.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +241 -72
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/RELEASING.md +3 -2
  25. data/docs/explanation/backends.md +30 -10
  26. data/docs/explanation/matching-semantics.md +4 -2
  27. data/docs/explanation/security-model.md +10 -3
  28. data/docs/explanation/unicode-and-encoding.md +3 -1
  29. data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
  30. data/docs/how-to/generate-a-lexer.md +3 -3
  31. data/docs/how-to/handle-errors.md +4 -1
  32. data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
  33. data/docs/how-to/track-token-locations.md +2 -1
  34. data/docs/how-to/tune-performance.md +15 -3
  35. data/docs/migration-v2.md +74 -0
  36. data/docs/perf-log.md +212 -0
  37. data/docs/reference/README.md +2 -0
  38. data/docs/reference/actions.md +5 -2
  39. data/docs/reference/cli.md +71 -5
  40. data/docs/reference/compatibility.md +10 -4
  41. data/docs/reference/diagnostics.md +346 -0
  42. data/docs/reference/dsl.md +30 -9
  43. data/docs/reference/errors.md +34 -3
  44. data/docs/reference/generated-artifacts.md +71 -6
  45. data/docs/reference/public-api.md +37 -7
  46. data/docs/reference/regexp.md +27 -13
  47. data/docs/reference/ruby-lsp.md +34 -0
  48. data/docs/reference/runtime.md +21 -4
  49. data/docs/reference/testing.md +57 -0
  50. data/docs/reference/tokens-and-locations.md +14 -4
  51. data/examples/graphql/README.md +14 -0
  52. data/examples/graphql/lexer.flexr.rb +29 -0
  53. data/examples/json/lexer.flexr.rb +6 -6
  54. data/examples/sql/README.md +16 -0
  55. data/examples/sql/lexer.flexr.rb +23 -0
  56. data/examples/toml/README.md +14 -0
  57. data/examples/toml/lexer.flexr.rb +23 -0
  58. data/gemfiles/ruby_lsp.gemfile +4 -0
  59. data/gemfiles/types.gemfile +7 -0
  60. data/lib/flexr/action_resolver.rb +15 -0
  61. data/lib/flexr/artifact_writer.rb +61 -0
  62. data/lib/flexr/automaton/accel.rb +78 -5
  63. data/lib/flexr/automaton/analysis.rb +202 -7
  64. data/lib/flexr/automaton/backend_cost_model.rb +40 -0
  65. data/lib/flexr/automaton/compiler.rb +229 -96
  66. data/lib/flexr/automaton/dfa.rb +106 -19
  67. data/lib/flexr/automaton/minimizer.rb +79 -31
  68. data/lib/flexr/automaton/nfa.rb +45 -13
  69. data/lib/flexr/automaton/types.rb +14 -0
  70. data/lib/flexr/cli.rb +128 -22
  71. data/lib/flexr/codegen/direct.rb +5 -41
  72. data/lib/flexr/codegen/firstmatch.rb +1 -7
  73. data/lib/flexr/codegen/table.rb +181 -69
  74. data/lib/flexr/codegen.rb +8 -0
  75. data/lib/flexr/configuration.rb +45 -0
  76. data/lib/flexr/diagnostics.rb +39 -3
  77. data/lib/flexr/dsl.rb +116 -47
  78. data/lib/flexr/errors.rb +7 -6
  79. data/lib/flexr/generated.rb +120 -46
  80. data/lib/flexr/generator.rb +102 -109
  81. data/lib/flexr/grammar.rb +41 -0
  82. data/lib/flexr/importer.rb +4 -40
  83. data/lib/flexr/ir.rb +2 -2
  84. data/lib/flexr/migration.rb +31 -0
  85. data/lib/flexr/minitest.rb +24 -0
  86. data/lib/flexr/options.rb +9 -5
  87. data/lib/flexr/rake_task.rb +6 -1
  88. data/lib/flexr/regexp/ast.rb +5 -2
  89. data/lib/flexr/regexp/matcher.rb +26 -0
  90. data/lib/flexr/regexp/normalizer.rb +63 -21
  91. data/lib/flexr/regexp/parser.rb +241 -60
  92. data/lib/flexr/regexp/tokenizer.rb +172 -47
  93. data/lib/flexr/rspec.rb +51 -0
  94. data/lib/flexr/runtime/buffer.rb +115 -9
  95. data/lib/flexr/runtime/core.rb +248 -51
  96. data/lib/flexr/runtime/errors.rb +66 -4
  97. data/lib/flexr/runtime/interpreter.rb +213 -97
  98. data/lib/flexr/runtime/location.rb +35 -12
  99. data/lib/flexr/runtime/record.rb +14 -0
  100. data/lib/flexr/runtime/token.rb +7 -1
  101. data/lib/flexr/runtime.rb +75 -0
  102. data/lib/flexr/source/passthrough.rb +70 -14
  103. data/lib/flexr/source/prism_reader.rb +217 -51
  104. data/lib/flexr/source/static_eval.rb +40 -4
  105. data/lib/flexr/source.rb +6 -0
  106. data/lib/flexr/testing.rb +47 -0
  107. data/lib/flexr/trace.rb +197 -0
  108. data/lib/flexr/unicode/data/properties.rb +1 -1
  109. data/lib/flexr/unicode/data.rb +11 -0
  110. data/lib/flexr/unicode/property.rb +4 -8
  111. data/lib/flexr/unicode/reference_regexp.rb +11 -9
  112. data/lib/flexr/unicode/version.rb +7 -0
  113. data/lib/flexr/version.rb +1 -1
  114. data/lib/flexr.rb +10 -76
  115. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  116. data/sig/flexr.rbs +191 -0
  117. data/site/README.md +54 -3
  118. data/site/astro.config.mjs +3 -1
  119. data/site/package.json +12 -5
  120. data/site/playwright.config.mjs +16 -0
  121. data/site/pnpm-lock.yaml +218 -88
  122. data/site/ruby/playground.rb +57 -0
  123. data/site/scripts/prepare-playground.mjs +35 -0
  124. data/site/scripts/verify-site.mjs +5 -3
  125. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  126. data/site/src/content/docs/concepts/security-model.md +9 -1
  127. data/site/src/content/docs/learn/generation.md +1 -1
  128. data/site/src/content/docs/reference/dsl.md +2 -2
  129. data/site/src/content/docs/reference/public-api.md +4 -2
  130. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  131. data/site/src/layouts/SiteLayout.astro +11 -3
  132. data/site/src/pages/index.astro +69 -155
  133. data/site/src/pages/playground.astro +40 -38
  134. data/site/src/playground/client.js +211 -0
  135. data/site/src/playground/graph-worker.mjs +11 -0
  136. data/site/src/playground/presets.mjs +33 -0
  137. data/site/src/playground/runtime.mjs +34 -0
  138. data/site/src/playground/share.mjs +30 -0
  139. data/site/src/playground/worker.mjs +29 -0
  140. data/site/src/styles/custom.css +12 -287
  141. data/site/src/styles/landing.css +239 -0
  142. data/site/tests/browser/playground.spec.mjs +99 -0
  143. data/site/tests/playground-runtime.test.mjs +85 -0
  144. data/test/ruby_lsp_addon_spec.rb +135 -0
  145. data/tools/coverage.rb +6 -1
  146. data/tools/docs_verify.rb +3 -2
  147. data/tools/golden.rb +54 -0
  148. data/tools/verification_failure.rb +119 -0
  149. metadata +66 -12
  150. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  151. data/benchmark/golden/json_lexer.sha256 +0 -1
  152. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  153. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  154. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  155. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  156. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  157. data/site/public/playground.js +0 -189
@@ -0,0 +1,47 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../flexr"
4
+
5
+ module Flexr
6
+ module Testing
7
+ module Snapshot
8
+ module_function
9
+
10
+ def render(tokens)
11
+ "#{JSON.pretty_generate(encode(tokens))}\n"
12
+ end
13
+
14
+ def expected(path, actual, update:)
15
+ File.binwrite(path, actual) if update
16
+ File.binread(path).force_encoding(Encoding::UTF_8)
17
+ end
18
+
19
+ def encode(value)
20
+ case value
21
+ when NilClass, TrueClass, FalseClass, Integer
22
+ value
23
+ when Float
24
+ raise ArgumentError, "unsupported snapshot value: non-finite Float" unless value.finite?
25
+
26
+ value
27
+ when String
28
+ utf8 = value.dup.force_encoding(Encoding::UTF_8)
29
+ utf8.valid_encoding? ? utf8 : { "bytes" => value.bytes }
30
+ when Symbol
31
+ { "symbol" => value.to_s }
32
+ when Array
33
+ value.map { |item| encode(item) }
34
+ when Hash
35
+ pairs = value.map { |key, item| [encode(key), encode(item)] }
36
+ { "hash" => pairs.sort_by { |key, _item| JSON.generate(key) } }
37
+ when Runtime::Token
38
+ { "token" => encode(value.to_h) }
39
+ when Runtime::Location
40
+ { "location" => encode(value.members.to_h { |name| [name, value.public_send(name)] }) }
41
+ else
42
+ raise ArgumentError, "unsupported snapshot value: #{value.class}; convert it to built-in values first"
43
+ end
44
+ end
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,197 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "runtime" unless defined?(Flexr::Lexer)
4
+
5
+ module Flexr
6
+ class Trace
7
+ module Actions
8
+ private
9
+
10
+ def execute(rule)
11
+ @flexr_trace.record(rule) { super }
12
+ end
13
+
14
+ public
15
+
16
+ def instance_exec(*args, &block)
17
+ if block.equal?(self.class.__flexr_spec.eof_rules[state])
18
+ @flexr_trace.record_eof { super }
19
+ else
20
+ super
21
+ end
22
+ end
23
+
24
+ private
25
+
26
+ def handle_unmatched_byte
27
+ @flexr_trace.record_unmatched { super }
28
+ end
29
+ end
30
+
31
+ attr_reader :steps
32
+
33
+ def initialize(lexer)
34
+ @lexer = lexer
35
+ @steps = []
36
+ @config = lexer.class.__flexr_config
37
+ @compiled = lexer.class.__flexr_compiled
38
+ @rules = lexer.class.__flexr_rules
39
+ @lexer.singleton_class.prepend(Actions)
40
+ @lexer.instance_variable_set(:@flexr_trace, self)
41
+ end
42
+
43
+ def run
44
+ @lexer.tokens
45
+ end
46
+
47
+ def record(rule)
48
+ start = @lexer.instance_variable_get(:@match_start)
49
+ ending = @lexer.instance_variable_get(:@match_end)
50
+ state = @lexer.state
51
+ candidates = candidates_at(start)
52
+ selected = candidates.find { |candidate| candidate[:rule] == rule.index }
53
+ reason = if @config.backend == :firstmatch
54
+ "first_match"
55
+ elsif candidates.length == 1
56
+ "only_match"
57
+ elsif selected && candidates.count { |candidate| candidate[:total_length] == selected[:total_length] } > 1
58
+ "rule_order"
59
+ else
60
+ "longest"
61
+ end
62
+ yield
63
+ pending = @lexer.instance_variable_get(:@pending)
64
+ bytes = @lexer.buffer.byteslice(start...ending)
65
+ @steps << { byte_begin: start, byte_end: ending, next_byte: @lexer.byte_pos, state: state, next_state: @lexer.state,
66
+ text: display_text(bytes), bytes: bytes.bytes, candidates: candidates,
67
+ rule: rule.index, patterns: rule.patterns.map(&:inspect), reason: reason,
68
+ action: pending ? "emit" : "skip", token: pending&.inspect }
69
+ end
70
+
71
+ def record_eof
72
+ position = @lexer.byte_pos
73
+ state = @lexer.state
74
+ yield
75
+ pending = @lexer.instance_variable_get(:@pending)
76
+ @steps << { byte_begin: position, byte_end: position, next_byte: @lexer.byte_pos,
77
+ state: state, next_state: @lexer.state, text: "", bytes: [], candidates: [],
78
+ rule: nil, patterns: [], reason: "eof", action: pending ? "emit" : "skip", token: pending&.inspect }
79
+ end
80
+
81
+ def record_unmatched
82
+ position = @lexer.byte_pos
83
+ state = @lexer.state
84
+ bytes = @lexer.buffer.byteslice(position, 1)
85
+ token = yield
86
+ @steps << { byte_begin: position, byte_end: position + 1, next_byte: @lexer.byte_pos,
87
+ state: state, next_state: @lexer.state, text: display_text(bytes), bytes: bytes.bytes,
88
+ candidates: [], rule: nil, patterns: [], reason: "unmatched",
89
+ action: token ? "emit" : "skip", token: token&.inspect }
90
+ token
91
+ end
92
+
93
+ def render
94
+ steps.map do |step|
95
+ candidates = step[:candidates].map do |candidate|
96
+ "rule #{candidate[:rule]}(#{candidate[:length]}/#{candidate[:total_length]})"
97
+ end.join(" ")
98
+ target = step[:rule] ? "rule #{step[:rule]}" : step[:reason]
99
+ "@#{step[:byte_begin]} #{step[:text].inspect} state=#{step[:state]} " \
100
+ "candidates: #{candidates} -> #{target} #{step[:action]} (#{step[:reason]}) " \
101
+ "state=#{step[:next_state]}#{step[:token] && " #{step[:token]}"}"
102
+ end.join("\n")
103
+ end
104
+
105
+ private
106
+
107
+ def candidates_at(start)
108
+ @config.backend == :firstmatch ? firstmatch_candidates(start) : dfa_candidates(start)
109
+ end
110
+
111
+ def dfa_candidates(start)
112
+ dfa = @compiled.machines.fetch(@lexer.state).dfa
113
+ best = {}
114
+ walk(dfa, start) do |acceptance, ending|
115
+ next if acceptance.bol_only && !@lexer.beginning_of_line?
116
+
117
+ rule = @rules.fetch(acceptance.rule_index)
118
+ candidate = candidate_for(rule, start, ending)
119
+ next unless candidate
120
+ previous = best[rule.index]
121
+ best[rule.index] = candidate if !previous || candidate[:total_length] >= previous[:total_length]
122
+ end
123
+ best.values.sort_by { |candidate| candidate[:rule] }
124
+ end
125
+
126
+ def firstmatch_candidates(start)
127
+ state = @config.states.fetch(@lexer.state)
128
+ @lexer.buffer.read_to_end
129
+ ending = @lexer.buffer.bytesize
130
+ if @config.encoding == Encoding::UTF_8
131
+ ending = start
132
+ ending += @lexer.buffer.utf8_character_length(ending) while
133
+ ending < @lexer.buffer.bytesize && @lexer.buffer.valid_utf8_at?(ending)
134
+ end
135
+ subject = @lexer.buffer.byteslice(start...ending).to_s
136
+ subject.force_encoding(@config.encoding)
137
+ @rules.filter_map do |rule|
138
+ next unless rule.states.include?(@lexer.state) || (state.inclusive && rule.states.include?(:initial))
139
+ matches = rule.patterns.each_with_index.filter_map do |pattern, index|
140
+ condition = rule.pattern_conditions.fetch(index)
141
+ next if condition.bol_only && !@lexer.beginning_of_line?
142
+ regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(::Regexp.escape(pattern))
143
+ reference = Regexp::Matcher.reference?(regexp, encoding: @config.encoding, unicode: @config.options[:unicode] == true)
144
+ match = Regexp::Matcher.match(regexp, subject, encoding: @config.encoding,
145
+ unicode: @config.options[:unicode] == true, reference: reference)
146
+ [match[0].bytesize, condition] if match&.begin(0)&.zero?
147
+ end
148
+ length, condition = matches.max_by(&:first)
149
+ next unless length
150
+ next if condition.end_anchor && !end_anchor?(start + length)
151
+
152
+ candidate_for(rule, start, start + length)
153
+ end
154
+ end
155
+
156
+ def candidate_for(rule, start, ending)
157
+ trailing = if rule.trailing
158
+ dfa = rule.trailing_dfa || (@trailing_dfas ||= {})[rule.index] ||= Flexr.compile_pattern(
159
+ rule.trailing, encoding: @config.encoding, options: { unicode: @config.options[:unicode] == true }
160
+ )
161
+ longest = nil
162
+ walk(dfa, ending) { |_acceptance, cursor| longest = cursor - ending }
163
+ longest
164
+ else
165
+ 0
166
+ end
167
+ return if trailing.nil?
168
+
169
+ { rule: rule.index, patterns: rule.patterns.map(&:inspect), length: ending - start,
170
+ trailing_length: trailing, total_length: ending - start + trailing }
171
+ end
172
+
173
+ def walk(dfa, start)
174
+ state = dfa.start
175
+ cursor = start
176
+ loop do
177
+ if @lexer.utf8_boundary?(cursor)
178
+ dfa.accepts[state].each do |acceptance|
179
+ yield acceptance, cursor unless acceptance.end_anchor && !end_anchor?(cursor)
180
+ end
181
+ end
182
+ break unless @lexer.buffer.ensure_available?(cursor + 1)
183
+ break unless (state = dfa.transition(state, @lexer.buffer.getbyte(cursor)))
184
+
185
+ cursor += 1
186
+ end
187
+ end
188
+
189
+ def end_anchor?(position)
190
+ @lexer.buffer.eof?(position) || @lexer.buffer.getbyte(position) == 0x0a
191
+ end
192
+
193
+ def display_text(bytes)
194
+ bytes.dup.force_encoding(Encoding::UTF_8).scrub
195
+ end
196
+ end
197
+ end