flexr 1.0.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +241 -72
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/RELEASING.md +3 -2
  25. data/docs/explanation/backends.md +30 -10
  26. data/docs/explanation/matching-semantics.md +4 -2
  27. data/docs/explanation/security-model.md +10 -3
  28. data/docs/explanation/unicode-and-encoding.md +3 -1
  29. data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
  30. data/docs/how-to/generate-a-lexer.md +3 -3
  31. data/docs/how-to/handle-errors.md +4 -1
  32. data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
  33. data/docs/how-to/track-token-locations.md +2 -1
  34. data/docs/how-to/tune-performance.md +15 -3
  35. data/docs/migration-v2.md +74 -0
  36. data/docs/perf-log.md +212 -0
  37. data/docs/reference/README.md +2 -0
  38. data/docs/reference/actions.md +5 -2
  39. data/docs/reference/cli.md +71 -5
  40. data/docs/reference/compatibility.md +10 -4
  41. data/docs/reference/diagnostics.md +346 -0
  42. data/docs/reference/dsl.md +30 -9
  43. data/docs/reference/errors.md +34 -3
  44. data/docs/reference/generated-artifacts.md +71 -6
  45. data/docs/reference/public-api.md +37 -7
  46. data/docs/reference/regexp.md +27 -13
  47. data/docs/reference/ruby-lsp.md +34 -0
  48. data/docs/reference/runtime.md +21 -4
  49. data/docs/reference/testing.md +57 -0
  50. data/docs/reference/tokens-and-locations.md +14 -4
  51. data/examples/graphql/README.md +14 -0
  52. data/examples/graphql/lexer.flexr.rb +29 -0
  53. data/examples/json/lexer.flexr.rb +6 -6
  54. data/examples/sql/README.md +16 -0
  55. data/examples/sql/lexer.flexr.rb +23 -0
  56. data/examples/toml/README.md +14 -0
  57. data/examples/toml/lexer.flexr.rb +23 -0
  58. data/gemfiles/ruby_lsp.gemfile +4 -0
  59. data/gemfiles/types.gemfile +7 -0
  60. data/lib/flexr/action_resolver.rb +15 -0
  61. data/lib/flexr/artifact_writer.rb +61 -0
  62. data/lib/flexr/automaton/accel.rb +78 -5
  63. data/lib/flexr/automaton/analysis.rb +202 -7
  64. data/lib/flexr/automaton/backend_cost_model.rb +40 -0
  65. data/lib/flexr/automaton/compiler.rb +229 -96
  66. data/lib/flexr/automaton/dfa.rb +106 -19
  67. data/lib/flexr/automaton/minimizer.rb +79 -31
  68. data/lib/flexr/automaton/nfa.rb +45 -13
  69. data/lib/flexr/automaton/types.rb +14 -0
  70. data/lib/flexr/cli.rb +128 -22
  71. data/lib/flexr/codegen/direct.rb +5 -41
  72. data/lib/flexr/codegen/firstmatch.rb +1 -7
  73. data/lib/flexr/codegen/table.rb +181 -69
  74. data/lib/flexr/codegen.rb +8 -0
  75. data/lib/flexr/configuration.rb +45 -0
  76. data/lib/flexr/diagnostics.rb +39 -3
  77. data/lib/flexr/dsl.rb +116 -47
  78. data/lib/flexr/errors.rb +7 -6
  79. data/lib/flexr/generated.rb +120 -46
  80. data/lib/flexr/generator.rb +102 -109
  81. data/lib/flexr/grammar.rb +41 -0
  82. data/lib/flexr/importer.rb +4 -40
  83. data/lib/flexr/ir.rb +2 -2
  84. data/lib/flexr/migration.rb +31 -0
  85. data/lib/flexr/minitest.rb +24 -0
  86. data/lib/flexr/options.rb +9 -5
  87. data/lib/flexr/rake_task.rb +6 -1
  88. data/lib/flexr/regexp/ast.rb +5 -2
  89. data/lib/flexr/regexp/matcher.rb +26 -0
  90. data/lib/flexr/regexp/normalizer.rb +63 -21
  91. data/lib/flexr/regexp/parser.rb +241 -60
  92. data/lib/flexr/regexp/tokenizer.rb +172 -47
  93. data/lib/flexr/rspec.rb +51 -0
  94. data/lib/flexr/runtime/buffer.rb +115 -9
  95. data/lib/flexr/runtime/core.rb +248 -51
  96. data/lib/flexr/runtime/errors.rb +66 -4
  97. data/lib/flexr/runtime/interpreter.rb +213 -97
  98. data/lib/flexr/runtime/location.rb +35 -12
  99. data/lib/flexr/runtime/record.rb +14 -0
  100. data/lib/flexr/runtime/token.rb +7 -1
  101. data/lib/flexr/runtime.rb +75 -0
  102. data/lib/flexr/source/passthrough.rb +70 -14
  103. data/lib/flexr/source/prism_reader.rb +217 -51
  104. data/lib/flexr/source/static_eval.rb +40 -4
  105. data/lib/flexr/source.rb +6 -0
  106. data/lib/flexr/testing.rb +47 -0
  107. data/lib/flexr/trace.rb +197 -0
  108. data/lib/flexr/unicode/data/properties.rb +1 -1
  109. data/lib/flexr/unicode/data.rb +11 -0
  110. data/lib/flexr/unicode/property.rb +4 -8
  111. data/lib/flexr/unicode/reference_regexp.rb +11 -9
  112. data/lib/flexr/unicode/version.rb +7 -0
  113. data/lib/flexr/version.rb +1 -1
  114. data/lib/flexr.rb +10 -76
  115. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  116. data/sig/flexr.rbs +191 -0
  117. data/site/README.md +54 -3
  118. data/site/astro.config.mjs +3 -1
  119. data/site/package.json +12 -5
  120. data/site/playwright.config.mjs +16 -0
  121. data/site/pnpm-lock.yaml +218 -88
  122. data/site/ruby/playground.rb +57 -0
  123. data/site/scripts/prepare-playground.mjs +35 -0
  124. data/site/scripts/verify-site.mjs +5 -3
  125. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  126. data/site/src/content/docs/concepts/security-model.md +9 -1
  127. data/site/src/content/docs/learn/generation.md +1 -1
  128. data/site/src/content/docs/reference/dsl.md +2 -2
  129. data/site/src/content/docs/reference/public-api.md +4 -2
  130. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  131. data/site/src/layouts/SiteLayout.astro +11 -3
  132. data/site/src/pages/index.astro +69 -155
  133. data/site/src/pages/playground.astro +40 -38
  134. data/site/src/playground/client.js +211 -0
  135. data/site/src/playground/graph-worker.mjs +11 -0
  136. data/site/src/playground/presets.mjs +33 -0
  137. data/site/src/playground/runtime.mjs +34 -0
  138. data/site/src/playground/share.mjs +30 -0
  139. data/site/src/playground/worker.mjs +29 -0
  140. data/site/src/styles/custom.css +12 -287
  141. data/site/src/styles/landing.css +239 -0
  142. data/site/tests/browser/playground.spec.mjs +99 -0
  143. data/site/tests/playground-runtime.test.mjs +85 -0
  144. data/test/ruby_lsp_addon_spec.rb +135 -0
  145. data/tools/coverage.rb +6 -1
  146. data/tools/docs_verify.rb +3 -2
  147. data/tools/golden.rb +54 -0
  148. data/tools/verification_failure.rb +119 -0
  149. metadata +66 -12
  150. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  151. data/benchmark/golden/json_lexer.sha256 +0 -1
  152. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  153. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  154. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  155. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  156. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  157. data/site/public/playground.js +0 -189
@@ -19,13 +19,7 @@ module Flexr
19
19
  end
20
20
 
21
21
  def self_loop_set(dfa, state)
22
- dfa.transitions[state].each_with_index.with_object([]) do |(destination, class_id), result|
23
- next unless destination == state
24
-
25
- dfa.ec.each_with_index do |value, byte|
26
- result << byte if value == class_id
27
- end
28
- end
22
+ Accel.self_loop_set(dfa, state)
29
23
  end
30
24
 
31
25
  def dead_states(dfa)
@@ -33,6 +27,207 @@ module Flexr
33
27
  dfa.accepts[state].empty? && dfa.transitions[state].compact.all? { |destination| destination == state }
34
28
  end
35
29
  end
30
+
31
+ def firstmatch_counterexample(first, second)
32
+ queue = [[first.start, second.start, false, +"".b]]
33
+ visited = { [first.start, second.start, false] => true }
34
+ bytes = joint_byte_representatives(first, second)
35
+ cursor = 0
36
+
37
+ while cursor < queue.length
38
+ first_state, second_state, first_seen, input = queue.fetch(cursor)
39
+ cursor += 1
40
+ bytes.each do |byte|
41
+ next_first = first_state && first.transition(first_state, byte)
42
+ next_second = second_state && second.transition(second_state, byte)
43
+ next unless next_second
44
+
45
+ next_input = input + byte.chr(Encoding::BINARY)
46
+ next_first_seen = first_seen || accepting?(first, next_first)
47
+ return next_input if next_first_seen && accepting?(second, next_second) && !accepting?(first, next_first)
48
+
49
+ key = [next_first, next_second, next_first_seen]
50
+ next if visited[key]
51
+
52
+ visited[key] = true
53
+ queue << [next_first, next_second, next_first_seen, next_input]
54
+ end
55
+ end
56
+ nil
57
+ end
58
+
59
+ def joint_byte_representatives(first, second)
60
+ representatives = {}
61
+ 256.times do |byte|
62
+ key = [first.ec[byte], second.ec[byte]]
63
+ representatives[key] ||= byte
64
+ end
65
+ representatives.values.sort_by { |byte| [byte.between?(32, 126) ? 0 : 1, byte] }
66
+ end
67
+
68
+ def shadowing_examples(dfa, rule_indices, include_empty: false, include_unshadowed: false)
69
+ remaining = (rule_indices & dfa.rule_ids).to_h { |index| [index, true] }
70
+ return {} if remaining.empty?
71
+
72
+ examples = {}
73
+ queue = [[dfa.start, nil, nil]]
74
+ # Visiting the start again permits a nonempty example when it accepts and loops.
75
+ visited = {}
76
+ bytes = joint_byte_representatives(dfa, dfa)
77
+ cursor = 0
78
+ while cursor < queue.length
79
+ state = queue.fetch(cursor).first
80
+ accepts = dfa.accepts.fetch(state)
81
+ winner = accepts.min_by(&:rule_index)&.rule_index
82
+ if (include_empty || cursor.positive?) && winner
83
+ shadowed = accepts.map(&:rule_index).uniq.select do |index|
84
+ (include_unshadowed || index > winner) && remaining.key?(index)
85
+ end
86
+ unless shadowed.empty?
87
+ input = example_input(queue, cursor)
88
+ shadowed.each do |index|
89
+ examples[index] = [input, winner]
90
+ remaining.delete(index)
91
+ end
92
+ return examples if remaining.empty?
93
+ end
94
+ end
95
+
96
+ bytes.each do |byte|
97
+ target = dfa.transition(state, byte)
98
+ next if target.nil? || visited[target]
99
+
100
+ visited[target] = true
101
+ queue << [target, cursor, byte]
102
+ end
103
+ cursor += 1
104
+ end
105
+ examples
106
+ end
107
+
108
+ def example_input(paths, cursor)
109
+ bytes = []
110
+ while (parent = paths.fetch(cursor)[1])
111
+ bytes << paths.fetch(cursor)[2]
112
+ cursor = parent
113
+ end
114
+ bytes.reverse.pack("C*")
115
+ end
116
+
117
+ def shortest_input(dfa)
118
+ queue = [[dfa.start, nil, nil]]
119
+ visited = { dfa.start => true }
120
+ bytes = joint_byte_representatives(dfa, dfa)
121
+ cursor = 0
122
+ while cursor < queue.length
123
+ state = queue.fetch(cursor).first
124
+ return example_input(queue, cursor) if accepting?(dfa, state)
125
+
126
+ bytes.each do |byte|
127
+ target = dfa.transition(state, byte)
128
+ next if target.nil? || visited[target]
129
+
130
+ visited[target] = true
131
+ queue << [target, cursor, byte]
132
+ end
133
+ cursor += 1
134
+ end
135
+ nil
136
+ end
137
+
138
+ def trailing_example(dfa, rules, target, input, trailing_dfas, encoding:, unicode:, firstmatch:, same_length: false)
139
+ lengths = if firstmatch
140
+ rules.filter_map do |rule|
141
+ next if rule.index > target.index
142
+
143
+ length = firstmatch_length(rule, input, encoding: encoding, unicode: unicode)
144
+ [rule.index, length] if length
145
+ end
146
+ else
147
+ dfa_match_lengths(dfa, input, encoding: encoding)
148
+ end
149
+ matches = {}
150
+ lengths.each do |index, length|
151
+ trailing = trailing_dfas[index]
152
+ lookahead = trailing ? dfa_match_lengths(trailing, input, offset: length).map(&:last).max : 0
153
+ next unless lookahead
154
+
155
+ total = length + lookahead
156
+ next if matches[index] && matches[index].last > total
157
+
158
+ matches[index] = [length, total]
159
+ end
160
+ return unless matches.key?(target.index)
161
+
162
+ winner, (length, total) = matches.max_by { |index, (_, size)| firstmatch ? -index : [size, -index] }
163
+ return if same_length && total != matches.fetch(target.index).last
164
+
165
+ [winner, length, total - length]
166
+ rescue EncodingError
167
+ nil
168
+ end
169
+
170
+ def dfa_match_lengths(dfa, input, offset: 0, encoding: Encoding::BINARY)
171
+ state = dfa.start
172
+ cursor = offset
173
+ matches = []
174
+ loop do
175
+ if encoding != Encoding::UTF_8 || input.byteslice(0, cursor).valid_encoding?
176
+ dfa.accepts.fetch(state).each do |acceptance|
177
+ next if acceptance.end_anchor && cursor < input.bytesize && input.getbyte(cursor) != 0x0a
178
+
179
+ matches << [acceptance.rule_index, cursor - offset]
180
+ end
181
+ end
182
+ break if cursor == input.bytesize
183
+
184
+ state = dfa.transition(state, input.getbyte(cursor))
185
+ break unless state
186
+
187
+ cursor += 1
188
+ end
189
+ matches
190
+ end
191
+
192
+ def firstmatch_example(rules, target, input, encoding:, unicode: false, same_length: false)
193
+ target_length = firstmatch_length(target, input, encoding: encoding, unicode: unicode)
194
+ return unless target_length
195
+
196
+ rules.sort_by(&:index).each do |rule|
197
+ break if rule.index >= target.index
198
+
199
+ length = firstmatch_length(rule, input, encoding: encoding, unicode: unicode)
200
+ next unless length
201
+
202
+ return nil if same_length && length != target_length
203
+
204
+ return [rule.index, length]
205
+ end
206
+ [target.index, target_length]
207
+ rescue EncodingError
208
+ nil
209
+ end
210
+
211
+ def firstmatch_length(rule, input, encoding:, unicode:)
212
+ matches = rule.patterns.each_with_index.filter_map do |pattern, index|
213
+ regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(::Regexp.escape(pattern.to_s))
214
+ reference = Regexp::Matcher.reference?(regexp, encoding: encoding, unicode: unicode)
215
+ match = Regexp::Matcher.match(regexp, input, encoding: encoding, unicode: unicode, reference: reference)
216
+ next unless match&.begin(0)&.zero?
217
+
218
+ [match[0].bytesize, rule.pattern_conditions.fetch(index)]
219
+ end
220
+ length, condition = matches.max_by(&:first)
221
+ return unless length
222
+ return if encoding == Encoding::UTF_8 && !input.byteslice(0, length).valid_encoding?
223
+ return if condition.end_anchor && length < input.bytesize && input.getbyte(length) != 0x0a
224
+
225
+ length
226
+ end
227
+
228
+ def accepting?(dfa, state)
229
+ state && !dfa.accepts.fetch(state).empty?
230
+ end
36
231
  end
37
232
  end
38
233
  end
@@ -0,0 +1,40 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Flexr
4
+ module Automaton
5
+ module BackendCostModel
6
+ Metrics = Struct.new(:dense_bytes, :packed_bytes, :lookup_samples, keyword_init: true) do
7
+ def direct_score
8
+ dense_bytes + lookup_samples
9
+ end
10
+
11
+ def table_score
12
+ packed_bytes + (lookup_samples * 3)
13
+ end
14
+ end
15
+
16
+ module_function
17
+
18
+ def choose(compiled)
19
+ metrics = compiled.machines.values.map { |machine| metrics_for(machine.dfa) }
20
+ direct_score = metrics.sum(&:direct_score)
21
+ table_score = metrics.sum(&:table_score)
22
+ direct_score < table_score ? :direct : :table
23
+ end
24
+
25
+ def metrics_for(dfa)
26
+ cells = dfa.states * dfa.class_count
27
+ overrides = dfa.transitions.sum do |row|
28
+ default = row.tally.max_by { |_value, count| count }&.first
29
+ row.count { |value| value != default }
30
+ end
31
+ packed_integers = (overrides * 2) + (dfa.states * 2)
32
+ Metrics.new(
33
+ dense_bytes: cells * 4,
34
+ packed_bytes: packed_integers * 4,
35
+ lookup_samples: dfa.states * [dfa.class_count, 16].min
36
+ ).freeze
37
+ end
38
+ end
39
+ end
40
+ end