flexr 1.0.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +241 -72
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/RELEASING.md +3 -2
  25. data/docs/explanation/backends.md +30 -10
  26. data/docs/explanation/matching-semantics.md +4 -2
  27. data/docs/explanation/security-model.md +10 -3
  28. data/docs/explanation/unicode-and-encoding.md +3 -1
  29. data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
  30. data/docs/how-to/generate-a-lexer.md +3 -3
  31. data/docs/how-to/handle-errors.md +4 -1
  32. data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
  33. data/docs/how-to/track-token-locations.md +2 -1
  34. data/docs/how-to/tune-performance.md +15 -3
  35. data/docs/migration-v2.md +74 -0
  36. data/docs/perf-log.md +212 -0
  37. data/docs/reference/README.md +2 -0
  38. data/docs/reference/actions.md +5 -2
  39. data/docs/reference/cli.md +71 -5
  40. data/docs/reference/compatibility.md +10 -4
  41. data/docs/reference/diagnostics.md +346 -0
  42. data/docs/reference/dsl.md +30 -9
  43. data/docs/reference/errors.md +34 -3
  44. data/docs/reference/generated-artifacts.md +71 -6
  45. data/docs/reference/public-api.md +37 -7
  46. data/docs/reference/regexp.md +27 -13
  47. data/docs/reference/ruby-lsp.md +34 -0
  48. data/docs/reference/runtime.md +21 -4
  49. data/docs/reference/testing.md +57 -0
  50. data/docs/reference/tokens-and-locations.md +14 -4
  51. data/examples/graphql/README.md +14 -0
  52. data/examples/graphql/lexer.flexr.rb +29 -0
  53. data/examples/json/lexer.flexr.rb +6 -6
  54. data/examples/sql/README.md +16 -0
  55. data/examples/sql/lexer.flexr.rb +23 -0
  56. data/examples/toml/README.md +14 -0
  57. data/examples/toml/lexer.flexr.rb +23 -0
  58. data/gemfiles/ruby_lsp.gemfile +4 -0
  59. data/gemfiles/types.gemfile +7 -0
  60. data/lib/flexr/action_resolver.rb +15 -0
  61. data/lib/flexr/artifact_writer.rb +61 -0
  62. data/lib/flexr/automaton/accel.rb +78 -5
  63. data/lib/flexr/automaton/analysis.rb +202 -7
  64. data/lib/flexr/automaton/backend_cost_model.rb +40 -0
  65. data/lib/flexr/automaton/compiler.rb +229 -96
  66. data/lib/flexr/automaton/dfa.rb +106 -19
  67. data/lib/flexr/automaton/minimizer.rb +79 -31
  68. data/lib/flexr/automaton/nfa.rb +45 -13
  69. data/lib/flexr/automaton/types.rb +14 -0
  70. data/lib/flexr/cli.rb +128 -22
  71. data/lib/flexr/codegen/direct.rb +5 -41
  72. data/lib/flexr/codegen/firstmatch.rb +1 -7
  73. data/lib/flexr/codegen/table.rb +181 -69
  74. data/lib/flexr/codegen.rb +8 -0
  75. data/lib/flexr/configuration.rb +45 -0
  76. data/lib/flexr/diagnostics.rb +39 -3
  77. data/lib/flexr/dsl.rb +116 -47
  78. data/lib/flexr/errors.rb +7 -6
  79. data/lib/flexr/generated.rb +120 -46
  80. data/lib/flexr/generator.rb +102 -109
  81. data/lib/flexr/grammar.rb +41 -0
  82. data/lib/flexr/importer.rb +4 -40
  83. data/lib/flexr/ir.rb +2 -2
  84. data/lib/flexr/migration.rb +31 -0
  85. data/lib/flexr/minitest.rb +24 -0
  86. data/lib/flexr/options.rb +9 -5
  87. data/lib/flexr/rake_task.rb +6 -1
  88. data/lib/flexr/regexp/ast.rb +5 -2
  89. data/lib/flexr/regexp/matcher.rb +26 -0
  90. data/lib/flexr/regexp/normalizer.rb +63 -21
  91. data/lib/flexr/regexp/parser.rb +241 -60
  92. data/lib/flexr/regexp/tokenizer.rb +172 -47
  93. data/lib/flexr/rspec.rb +51 -0
  94. data/lib/flexr/runtime/buffer.rb +115 -9
  95. data/lib/flexr/runtime/core.rb +248 -51
  96. data/lib/flexr/runtime/errors.rb +66 -4
  97. data/lib/flexr/runtime/interpreter.rb +213 -97
  98. data/lib/flexr/runtime/location.rb +35 -12
  99. data/lib/flexr/runtime/record.rb +14 -0
  100. data/lib/flexr/runtime/token.rb +7 -1
  101. data/lib/flexr/runtime.rb +75 -0
  102. data/lib/flexr/source/passthrough.rb +70 -14
  103. data/lib/flexr/source/prism_reader.rb +217 -51
  104. data/lib/flexr/source/static_eval.rb +40 -4
  105. data/lib/flexr/source.rb +6 -0
  106. data/lib/flexr/testing.rb +47 -0
  107. data/lib/flexr/trace.rb +197 -0
  108. data/lib/flexr/unicode/data/properties.rb +1 -1
  109. data/lib/flexr/unicode/data.rb +11 -0
  110. data/lib/flexr/unicode/property.rb +4 -8
  111. data/lib/flexr/unicode/reference_regexp.rb +11 -9
  112. data/lib/flexr/unicode/version.rb +7 -0
  113. data/lib/flexr/version.rb +1 -1
  114. data/lib/flexr.rb +10 -76
  115. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  116. data/sig/flexr.rbs +191 -0
  117. data/site/README.md +54 -3
  118. data/site/astro.config.mjs +3 -1
  119. data/site/package.json +12 -5
  120. data/site/playwright.config.mjs +16 -0
  121. data/site/pnpm-lock.yaml +218 -88
  122. data/site/ruby/playground.rb +57 -0
  123. data/site/scripts/prepare-playground.mjs +35 -0
  124. data/site/scripts/verify-site.mjs +5 -3
  125. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  126. data/site/src/content/docs/concepts/security-model.md +9 -1
  127. data/site/src/content/docs/learn/generation.md +1 -1
  128. data/site/src/content/docs/reference/dsl.md +2 -2
  129. data/site/src/content/docs/reference/public-api.md +4 -2
  130. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  131. data/site/src/layouts/SiteLayout.astro +11 -3
  132. data/site/src/pages/index.astro +69 -155
  133. data/site/src/pages/playground.astro +40 -38
  134. data/site/src/playground/client.js +211 -0
  135. data/site/src/playground/graph-worker.mjs +11 -0
  136. data/site/src/playground/presets.mjs +33 -0
  137. data/site/src/playground/runtime.mjs +34 -0
  138. data/site/src/playground/share.mjs +30 -0
  139. data/site/src/playground/worker.mjs +29 -0
  140. data/site/src/styles/custom.css +12 -287
  141. data/site/src/styles/landing.css +239 -0
  142. data/site/tests/browser/playground.spec.mjs +99 -0
  143. data/site/tests/playground-runtime.test.mjs +85 -0
  144. data/test/ruby_lsp_addon_spec.rb +135 -0
  145. data/tools/coverage.rb +6 -1
  146. data/tools/docs_verify.rb +3 -2
  147. data/tools/golden.rb +54 -0
  148. data/tools/verification_failure.rb +119 -0
  149. metadata +66 -12
  150. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  151. data/benchmark/golden/json_lexer.sha256 +0 -1
  152. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  153. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  154. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  155. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  156. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  157. data/site/public/playground.js +0 -189
@@ -27,57 +27,146 @@ module Flexr
27
27
 
28
28
  def source(indent: " ")
29
29
  lines = []
30
+ whole_tokens = compiled.machines.values.any?(&:whole_tokens)
30
31
  lines << "#{indent}def scan_one"
31
- lines << "#{indent} return Flexr::Runtime::Interpreter.new(self).scan unless __flexr_generated_fast_path?"
32
- lines << "#{indent} machine = self.class.__flexr_compiled.machines.fetch(state)"
33
- lines << "#{indent} dfa = machine.dfa"
32
+ lines << "#{indent} if @__flexr_generated_scan_state != @state"
33
+ lines << "#{indent} machine = self.class.__flexr_compiled.machines.fetch(@state)"
34
+ lines << "#{indent} @__flexr_generated_scan_state = @state"
35
+ lines << "#{indent} @__flexr_generated_scan_dfa = machine.dfa"
36
+ lines << "#{indent} @__flexr_generated_whole_tokens = machine.respond_to?(:whole_tokens) ? machine.whole_tokens : nil" if whole_tokens
37
+ lines << "#{indent} @__flexr_generated_scan_fast = __flexr_generated_fast_path?(@__flexr_generated_scan_dfa)"
38
+ lines << "#{indent} @__flexr_generated_scan_simple = __flexr_generated_simple_fast_path?(@__flexr_generated_scan_dfa)"
39
+ lines << "#{indent} end"
40
+ lines << "#{indent} dfa = @__flexr_generated_scan_dfa"
41
+ lines << "#{indent} return Flexr::Runtime::Interpreter.new(self).scan unless @__flexr_generated_scan_fast"
34
42
  lines << "#{indent} position = byte_pos"
35
43
  lines << "#{indent} return nil unless valid_utf8_at?(position)"
44
+ if whole_tokens
45
+ lines << "#{indent} match = __flexr_whole_token_match(@__flexr_generated_whole_tokens, position) if @__flexr_generated_whole_tokens"
46
+ lines << "#{indent} return match if match"
47
+ end
48
+ lines << "#{indent} return __flexr_generated_scan_simple(dfa, position) if @__flexr_generated_scan_simple"
36
49
  lines << "#{indent} cursor = position"
37
50
  lines << "#{indent} current = dfa.start"
38
51
  lines << "#{indent} direct = dfa.direct"
39
- lines << "#{indent} source = buffer.source"
40
- simple = if simple_fast_path?
41
- lines << "#{indent} best = nil"
42
- true
43
- else
44
- lines << "#{indent} best = __flexr_generated_acceptance(dfa, current, position, position, nil)"
45
- false
46
- end
47
- lines << "#{indent} accelerate = self.class.__flexr_config.options.fetch(:accel, :auto) != :none && !utf8_input?"
48
- lines << "#{indent} while cursor < source.bytesize || buffer.ensure_available?(cursor + 1)"
52
+ lines << "#{indent} direct_nxt = direct&.fetch(:nxt)"
53
+ lines << "#{indent} direct_classes = direct&.fetch(:classes)"
54
+ lines << "#{indent} ec = dfa.ec"
55
+ lines << "#{indent} source = buffer.stable_source"
56
+ lines << "#{indent} guarded_steps = scan_steps_guarded?"
57
+ lines << "#{indent} token_start = more_text_start || position"
58
+ lines << "#{indent} token_limit = max_token_size"
59
+ lines << "#{indent} check_token_limit = token_limit_required?"
60
+ lines << "#{indent} best = nil"
61
+ lines << "#{indent} accel_mode = @accel_mode"
62
+ lines << "#{indent} accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)"
63
+ lines << "#{indent} while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)"
49
64
  lines << "#{indent} if accelerate"
50
65
  lines << "#{indent} region = __flexr_generated_acceleration_region(dfa, current)"
51
- lines << "#{indent} accelerated_end = __flexr_generated_accelerate(region, cursor) if region"
66
+ lines << "#{indent} acceptance = dfa.accepts[current].first"
67
+ lines << "#{indent} token_rule = acceptance&.rule_index"
68
+ lines << "#{indent} accelerated_end = __flexr_generated_accelerate(region, cursor, token_rule, position) if region"
52
69
  lines << "#{indent} if accelerated_end && accelerated_end > cursor"
53
70
  lines << "#{indent} cursor = accelerated_end"
54
71
  lines << "#{indent} best = __flexr_generated_acceptance(dfa, current, position, cursor, best)"
55
72
  lines << "#{indent} next"
56
73
  lines << "#{indent} end"
57
74
  lines << "#{indent} end"
58
- lines << "#{indent} byte = source.getbyte(cursor)"
59
- lines << "#{indent} current = if direct"
60
- lines << "#{indent} class_id = dfa.ec[byte]"
61
- lines << "#{indent} value = direct[:nxt][(current * direct[:classes]) + class_id]"
75
+ lines << "#{indent} byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)"
76
+ lines << "#{indent} consume_step! if guarded_steps"
77
+ lines << "#{indent} current = if direct_nxt"
78
+ lines << "#{indent} class_id = ec[byte]"
79
+ lines << "#{indent} value = direct_nxt[(current * direct_classes) + class_id]"
62
80
  lines << "#{indent} value >= 0 ? value : nil"
63
81
  lines << "#{indent} else"
64
82
  lines << "#{indent} dfa.transition(current, byte)"
65
83
  lines << "#{indent} end"
66
84
  lines << "#{indent} break unless current"
67
85
  lines << "#{indent} cursor += 1"
68
- if simple
69
- lines << "#{indent} acceptance = dfa.accepts[current].first"
70
- lines << "#{indent} if acceptance"
71
- lines << "#{indent} rule = self.class.__flexr_rules.fetch(acceptance.rule_index)"
72
- lines << "#{indent} best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)"
73
- lines << "#{indent} best.rule = rule"
74
- lines << "#{indent} best.start_pos = position"
75
- lines << "#{indent} best.end_pos = cursor"
76
- lines << "#{indent} best.total_end_pos = cursor"
77
- lines << "#{indent} end"
78
- else
79
- lines << "#{indent} best = __flexr_generated_acceptance(dfa, current, position, cursor, best)"
80
- end
86
+ lines << "#{indent} resource_acceptance = dfa.accepts[current].first"
87
+ lines << "#{indent} resource_rule = resource_acceptance&.rule_index"
88
+ lines << "#{indent} if check_token_limit"
89
+ lines << "#{indent} size = cursor - token_start"
90
+ lines << "#{indent} defer_token_size_check!(size, rule: resource_rule) if size > token_limit"
91
+ lines << "#{indent} end"
92
+ lines << "#{indent} best = __flexr_generated_acceptance(dfa, current, position, cursor, best)"
93
+ lines << "#{indent} end"
94
+ lines << "#{indent} best"
95
+ lines << "#{indent}end"
96
+ lines << "#{indent}def __flexr_generated_scan_simple(dfa, position)"
97
+ lines << "#{indent} current = dfa.start"
98
+ lines << "#{indent} direct = dfa.direct"
99
+ lines << "#{indent} direct_nxt = direct&.fetch(:nxt)"
100
+ lines << "#{indent} direct_classes = direct&.fetch(:classes)"
101
+ lines << "#{indent} ec = dfa.ec"
102
+ lines << "#{indent} accepts = dfa.accepts"
103
+ lines << "#{indent} rules = self.class.__flexr_rules"
104
+ lines << "#{indent} source = buffer.stable_source"
105
+ lines << "#{indent} guarded_steps = scan_steps_guarded?"
106
+ lines << "#{indent} token_start = more_text_start || position"
107
+ lines << "#{indent} token_limit = max_token_size"
108
+ lines << "#{indent} check_token_limit = token_limit_required?"
109
+ lines << "#{indent} cursor = position"
110
+ lines << "#{indent} best = nil"
111
+ lines << "#{indent} accel_mode = @accel_mode"
112
+ lines << "#{indent} accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)"
113
+ lines << "#{indent} if source && !guarded_steps && !check_token_limit && !accelerate"
114
+ lines << "#{indent} while cursor < source.bytesize"
115
+ lines << "#{indent} byte = source.getbyte(cursor)"
116
+ lines << "#{indent} current = if direct_nxt"
117
+ lines << "#{indent} class_id = ec[byte]"
118
+ lines << "#{indent} value = direct_nxt[(current * direct_classes) + class_id]"
119
+ lines << "#{indent} value >= 0 ? value : nil"
120
+ lines << "#{indent} else"
121
+ lines << "#{indent} dfa.transition(current, byte)"
122
+ lines << "#{indent} end"
123
+ lines << "#{indent} break unless current"
124
+ lines << "#{indent} cursor += 1"
125
+ lines << "#{indent} acceptance = accepts[current].first"
126
+ lines << "#{indent} next unless acceptance"
127
+ lines << "#{indent} rule = rules.fetch(acceptance.rule_index)"
128
+ lines << "#{indent} best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)"
129
+ lines << "#{indent} best.rule = rule"
130
+ lines << "#{indent} best.start_pos = position"
131
+ lines << "#{indent} best.end_pos = cursor"
132
+ lines << "#{indent} best.total_end_pos = cursor"
133
+ lines << "#{indent} end"
134
+ lines << "#{indent} return best"
135
+ lines << "#{indent} end"
136
+ lines << "#{indent} while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)"
137
+ lines << "#{indent} if accelerate"
138
+ lines << "#{indent} region = __flexr_generated_acceleration_region(dfa, current)"
139
+ lines << "#{indent} acceptance = accepts[current].first"
140
+ lines << "#{indent} accelerated_end = __flexr_generated_accelerate(region, cursor, acceptance&.rule_index, position) if region"
141
+ lines << "#{indent} if accelerated_end && accelerated_end > cursor"
142
+ lines << "#{indent} cursor = accelerated_end"
143
+ lines << "#{indent} best = __flexr_generated_acceptance(dfa, current, position, cursor, best)"
144
+ lines << "#{indent} next"
145
+ lines << "#{indent} end"
146
+ lines << "#{indent} end"
147
+ lines << "#{indent} byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)"
148
+ lines << "#{indent} consume_step! if guarded_steps"
149
+ lines << "#{indent} current = if direct_nxt"
150
+ lines << "#{indent} class_id = ec[byte]"
151
+ lines << "#{indent} value = direct_nxt[(current * direct_classes) + class_id]"
152
+ lines << "#{indent} value >= 0 ? value : nil"
153
+ lines << "#{indent} else"
154
+ lines << "#{indent} dfa.transition(current, byte)"
155
+ lines << "#{indent} end"
156
+ lines << "#{indent} break unless current"
157
+ lines << "#{indent} cursor += 1"
158
+ lines << "#{indent} acceptance = accepts[current].first"
159
+ lines << "#{indent} if check_token_limit"
160
+ lines << "#{indent} size = cursor - token_start"
161
+ lines << "#{indent} defer_token_size_check!(size, rule: acceptance&.rule_index) if size > token_limit"
162
+ lines << "#{indent} end"
163
+ lines << "#{indent} next unless acceptance"
164
+ lines << "#{indent} rule = rules.fetch(acceptance.rule_index)"
165
+ lines << "#{indent} best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)"
166
+ lines << "#{indent} best.rule = rule"
167
+ lines << "#{indent} best.start_pos = position"
168
+ lines << "#{indent} best.end_pos = cursor"
169
+ lines << "#{indent} best.total_end_pos = cursor"
81
170
  lines << "#{indent} end"
82
171
  lines << "#{indent} best"
83
172
  lines << "#{indent}end"
@@ -86,44 +175,75 @@ module Flexr
86
175
  lines << "#{indent} regions = (@__flexr_generated_accel_regions[dfa] ||= Flexr::Automaton::Accel.extract(dfa).to_h { |region| [region.state, region] })"
87
176
  lines << "#{indent} region = regions[state]"
88
177
  lines << "#{indent} return unless region"
178
+ lines << "#{indent} return if @__flexr_generated_disabled_auto_accel&.key?(region)"
179
+ lines << "#{indent} return if utf8_input? && region.bytes.any? { |byte| byte >= 128 }"
89
180
  lines << "#{indent} return if dfa.accepts[state].any? do |acceptance|"
90
181
  lines << "#{indent} rule = self.class.__flexr_rules.fetch(acceptance.rule_index)"
91
182
  lines << "#{indent} acceptance.bol_only || acceptance.end_anchor || rule.trailing"
92
183
  lines << "#{indent} end"
93
184
  lines << "#{indent} region"
94
185
  lines << "#{indent}end"
95
- lines << "#{indent}def __flexr_generated_accelerate(region, position)"
96
- lines << "#{indent} binary = buffer.source.b"
97
- lines << "#{indent} mode = self.class.__flexr_config.options.fetch(:accel, :auto)"
98
- lines << "#{indent} match_end = if %i[strscan auto].include?(mode) && defined?(StringScanner)"
99
- lines << "#{indent} scanner = StringScanner.new(binary)"
100
- lines << "#{indent} scanner.pos = position"
101
- lines << "#{indent} length = scanner.skip(region.regexp)"
102
- lines << "#{indent} length && scanner.pos"
103
- lines << "#{indent} else"
104
- lines << "#{indent} match = region.regexp.match(binary, position)"
105
- lines << "#{indent} match && match.begin(0) == position ? match.end(0) : nil"
106
- lines << "#{indent} end"
107
- lines << "#{indent} return match_end if match_end && match_end < buffer.bytesize"
108
- lines << "#{indent} return match_end if match_end && buffer.eof_loaded?"
109
- lines << "#{indent} return unless buffer.ensure_available?(buffer.bytesize + 1)"
110
- lines << "#{indent} __flexr_generated_accelerate(region, position)"
111
- lines << "#{indent}rescue ArgumentError"
112
- lines << "#{indent} nil"
113
- lines << "#{indent}end"
114
- lines << "#{indent}def __flexr_generated_fast_path?"
115
- lines << "#{indent} return @__flexr_generated_fast_path if defined?(@__flexr_generated_fast_path)"
116
- lines << "#{indent} @__flexr_generated_fast_path = if self.class.__flexr_config.backend == :firstmatch ||"
117
- lines << "#{indent} self.class.__flexr_config.options[:allow_empty_match] == true"
118
- lines << "#{indent} false"
119
- lines << "#{indent} else"
120
- lines << "#{indent} self.class.__flexr_rules.none? do |rule|"
121
- lines << "#{indent} !rule.trailing.nil? || rule.patterns.any? do |pattern|"
122
- lines << "#{indent} next false unless pattern.is_a?(Regexp)"
123
- lines << "#{indent} next true if pattern.source.match?(/\\\\[pP]\\\\{/) || pattern.source.match?(/\\[:(?:\\^)?[a-z]+:\\]/)"
124
- lines << "#{indent} self.class.__flexr_config.options[:unicode] == true && utf8_input? && pattern.source.match?(/\\\\[dDwWsS]/)"
186
+ lines << "#{indent}def __flexr_generated_accelerate(region, position, token_rule, token_start)"
187
+ lines << "#{indent} mode = @accel_mode"
188
+ lines << "#{indent} cursor = position"
189
+ lines << "#{indent} matched = false"
190
+ lines << "#{indent} loop do"
191
+ lines << "#{indent} if cursor >= buffer.bytesize"
192
+ lines << "#{indent} return matched ? cursor : nil if buffer.eof_loaded?"
193
+ lines << "#{indent} return matched ? cursor : nil unless buffer.ensure_available?(cursor + 1)"
194
+ lines << "#{indent} end"
195
+ lines << "#{indent} segment = buffer.source"
196
+ lines << "#{indent} segment_position = cursor - buffer.base_offset"
197
+ lines << "#{indent} if segment.encoding == ::Encoding::UTF_8 && region.utf8_regexp"
198
+ lines << "#{indent} regexp = region.utf8_regexp"
199
+ lines << "#{indent} else"
200
+ lines << "#{indent} segment = segment.b unless segment.encoding == ::Encoding::BINARY"
201
+ lines << "#{indent} regexp = region.regexp"
202
+ lines << "#{indent} end"
203
+ lines << "#{indent} length = begin"
204
+ lines << "#{indent} if %i[strscan auto].include?(mode) && defined?(::StringScanner)"
205
+ lines << "#{indent} @__flexr_generated_accel_scanner ||= ::StringScanner.new(\"\".b)"
206
+ lines << "#{indent} @__flexr_generated_accel_scanner.string = segment"
207
+ lines << "#{indent} @__flexr_generated_accel_scanner.pos = segment_position"
208
+ lines << "#{indent} @__flexr_generated_accel_scanner.skip(regexp)"
209
+ lines << "#{indent} else"
210
+ lines << "#{indent} match = regexp.match(segment.byteslice(segment_position..).to_s, 0)"
211
+ lines << "#{indent} match && match.begin(0).zero? ? match[0].bytesize : nil"
125
212
  lines << "#{indent} end"
213
+ lines << "#{indent} rescue ArgumentError"
214
+ lines << "#{indent} nil"
126
215
  lines << "#{indent} end"
216
+ lines << "#{indent} __flexr_generated_record_auto_accel_miss(region) if mode == :auto && length.to_i < 8"
217
+ lines << "#{indent} return matched ? cursor : nil unless length&.positive?"
218
+ lines << "#{indent} matched = true"
219
+ lines << "#{indent} cursor += length"
220
+ lines << "#{indent} consume_step!(length)"
221
+ lines << "#{indent} defer_token_size_check!(cursor - (more_text_start || token_start), rule: token_rule)"
222
+ lines << "#{indent} return cursor if cursor < buffer.bytesize || buffer.eof_loaded?"
223
+ lines << "#{indent} end"
224
+ lines << "#{indent}end"
225
+ lines << "#{indent}def __flexr_generated_record_auto_accel_miss(region)"
226
+ lines << "#{indent} @__flexr_generated_auto_accel_misses ||= Hash.new(0)"
227
+ lines << "#{indent} @__flexr_generated_auto_accel_misses[region] += 1"
228
+ lines << "#{indent} @__flexr_generated_auto_accel_miss_total = @__flexr_generated_auto_accel_miss_total.to_i + 1"
229
+ lines << "#{indent} @__flexr_generated_auto_accel_off = true if @__flexr_generated_auto_accel_miss_total >= 9"
230
+ lines << "#{indent} return if @__flexr_generated_auto_accel_misses[region] < 3"
231
+ lines << "#{indent} @__flexr_generated_disabled_auto_accel ||= {}"
232
+ lines << "#{indent} @__flexr_generated_disabled_auto_accel[region] = true"
233
+ lines << "#{indent}end"
234
+ lines << "#{indent}def __flexr_generated_fast_path?(dfa)"
235
+ lines << "#{indent} @__flexr_generated_fast_paths ||= {}"
236
+ lines << "#{indent} return @__flexr_generated_fast_paths[dfa] if @__flexr_generated_fast_paths.key?(dfa)"
237
+ lines << "#{indent} rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }"
238
+ lines << "#{indent} @__flexr_generated_fast_paths[dfa] = self.class.__flexr_config.backend != :firstmatch &&"
239
+ lines << "#{indent} self.class.__flexr_config.options[:allow_empty_match] != true && rules.none?(&:trailing)"
240
+ lines << "#{indent}end"
241
+ lines << "#{indent}def __flexr_generated_simple_fast_path?(dfa)"
242
+ lines << "#{indent} @__flexr_generated_simple_fast_paths ||= {}"
243
+ lines << "#{indent} return @__flexr_generated_simple_fast_paths[dfa] if @__flexr_generated_simple_fast_paths.key?(dfa)"
244
+ lines << "#{indent} rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }"
245
+ lines << "#{indent} @__flexr_generated_simple_fast_paths[dfa] = rules.all? do |rule|"
246
+ lines << "#{indent} rule.pattern_conditions.all? { |condition| condition && !condition.bol_only && !condition.end_anchor }"
127
247
  lines << "#{indent} end"
128
248
  lines << "#{indent}end"
129
249
  lines << "#{indent}def __flexr_generated_acceptance(dfa, state, start_position, cursor, best)"
@@ -131,7 +251,6 @@ module Flexr
131
251
  lines << "#{indent} next if acceptance.bol_only && !beginning_of_line?"
132
252
  lines << "#{indent} next if acceptance.end_anchor && !(buffer.eof?(cursor) || buffer.getbyte(cursor) == 0x0a)"
133
253
  lines << "#{indent} rule = self.class.__flexr_rules.fetch(acceptance.rule_index)"
134
- lines << "#{indent} defer_token_size_check!(cursor - start_position)"
135
254
  lines << "#{indent} next if best && cursor < best.total_end_pos"
136
255
  lines << "#{indent} next if best && cursor == best.total_end_pos &&"
137
256
  lines << "#{indent} rule.index > best.rule.index"
@@ -146,13 +265,6 @@ module Flexr
146
265
  "#{lines.join("\n")}\n"
147
266
  end
148
267
 
149
- def simple_fast_path?
150
- compiled.rules.all? do |rule|
151
- rule.trailing.nil? && rule.pattern_conditions.all? do |condition|
152
- condition && !condition.bol_only && !condition.end_anchor
153
- end
154
- end
155
- end
156
268
  end
157
269
  end
158
270
  end
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "runtime" unless defined?(Flexr::Lexer)
4
+ require_relative "codegen/base"
5
+ require_relative "codegen/table"
6
+ require_relative "codegen/direct"
7
+ require_relative "codegen/firstmatch"
8
+ require_relative "codegen/table_packer"
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Flexr
4
+ module Configuration
5
+ BACKENDS = %i[table direct firstmatch auto].freeze
6
+ TOKEN_KINDS = %i[array struct yield].freeze
7
+ ACCELERATORS = %i[auto strscan regexp none].freeze
8
+ BOOLEAN_OPTIONS = %i[unicode eager_columns allow_empty_match experimental standalone].freeze
9
+
10
+ module_function
11
+
12
+ def backend!(value)
13
+ validated_symbol!(value, BACKENDS, :backend)
14
+ end
15
+
16
+ def token_kind!(value)
17
+ validated_symbol!(value, TOKEN_KINDS, :token_kind)
18
+ end
19
+
20
+ def accelerator!(value)
21
+ validated_symbol!(value, ACCELERATORS, :accel)
22
+ end
23
+
24
+ def encoding!(value)
25
+ encoding = value.is_a?(Encoding) ? value : Encoding.find(value.to_s)
26
+ return encoding if [Encoding::UTF_8, Encoding::BINARY].include?(encoding)
27
+
28
+ diagnostic = Diagnostics.error("FLEXR-E011", "flexr supports UTF-8 and BINARY only")
29
+ raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
30
+ end
31
+
32
+ def option!(value)
33
+ validated_symbol!(value, BOOLEAN_OPTIONS, :option)
34
+ end
35
+
36
+ def validated_symbol!(value, allowed, name)
37
+ symbol = value.to_sym
38
+ return symbol if allowed.include?(symbol)
39
+
40
+ raise ArgumentError, "unsupported #{name}: #{value.inspect}"
41
+ rescue NoMethodError
42
+ raise ArgumentError, "unsupported #{name}: #{value.inspect}"
43
+ end
44
+ end
45
+ end
@@ -50,23 +50,46 @@ module Flexr
50
50
  @items.dup
51
51
  end
52
52
 
53
- def render(format: :human, color: :auto)
53
+ def render(format: :human, color: :auto, source: nil, path: nil)
54
54
  return JSON.generate(@items.map(&:to_h)) if format.to_sym == :json
55
55
 
56
- @items.map { |item| render_one(item, color: color_enabled?(color)) }.join("\n")
56
+ @items.map { |item| render_one(item, color: color_enabled?(color), source: source, path: path) }.join("\n")
57
57
  end
58
58
 
59
59
  private
60
60
 
61
- def render_one(item, color:)
61
+ def render_one(item, color:, source:, path:)
62
62
  prefix = "#{item.severity}[#{item.code}]: #{item.message}"
63
63
  prefix = "\e[31m#{prefix}\e[0m" if [true, :always].include?(color)
64
64
  lines = [prefix]
65
+ lines.concat(render_source(item.location, source, path))
65
66
  lines << " help: #{item.help}" if item.help
66
67
  lines << " note: #{item.note}" if item.note
67
68
  lines.join("\n")
68
69
  end
69
70
 
71
+ def render_source(location, source, path)
72
+ return [] unless source.is_a?(String) && location.is_a?(Array) && location.length == 2 &&
73
+ location.all?(Integer)
74
+
75
+ first, last = location
76
+ return [] unless first.between?(0, source.bytesize) && last >= first
77
+
78
+ before = source.byteslice(0, first).b
79
+ line = before.count("\n") + 1
80
+ line_start = (before.rindex("\n") || -1) + 1
81
+ line_end = source.b.index("\n", first) || source.bytesize
82
+ text = source.byteslice(line_start...line_end).delete_suffix("\r")
83
+ prefix = source.byteslice(line_start...first).delete_suffix("\r").scrub
84
+ last = last.clamp(first, [line_start + text.bytesize, first].max)
85
+ width = [source.byteslice(first...last).scrub.length, 1].max
86
+ gutter = " " * line.to_s.length
87
+ # shortcut: carets count characters; use display-cell widths to align wide or combining glyphs.
88
+ indentation = prefix.gsub(/[^\t]/, " ")
89
+ [" --> #{path || '(source)'}:#{line}:#{prefix.length + 1}",
90
+ "#{gutter} |", "#{line} | #{text.scrub}", "#{gutter} | #{indentation}#{'^' * width}"]
91
+ end
92
+
70
93
  def color_enabled?(color)
71
94
  return false if ENV.key?("NO_COLOR") || color == :never || color == false
72
95
  return true if [:always, true].include?(color)
@@ -78,6 +101,15 @@ module Flexr
78
101
  module Diagnostics
79
102
  module_function
80
103
 
104
+ def rule_label(rule)
105
+ return if rule.nil?
106
+ return "rule #{rule}" unless rule.is_a?(IR::Rule)
107
+
108
+ pattern = rule.patterns.one? ? rule.patterns.first : rule.patterns
109
+ label = "rule #{pattern.inspect}"
110
+ rule.trailing ? "#{label}, followed_by: #{rule.trailing.inspect}" : label
111
+ end
112
+
81
113
  def error(code, message, location: nil, help: nil, note: nil)
82
114
  Diagnostic.new(code: code, severity: :error, message: message, location: location, help: help, note: note)
83
115
  end
@@ -86,6 +118,10 @@ module Flexr
86
118
  Diagnostic.new(code: code, severity: :warning, message: message, location: location, help: help, note: note)
87
119
  end
88
120
 
121
+ def info(code, message, location: nil, help: nil, note: nil)
122
+ Diagnostic.new(code: code, severity: :info, message: message, location: location, help: help, note: note)
123
+ end
124
+
89
125
  def raise!(diagnostic)
90
126
  klass = diagnostic.code == "FLEXR-E014" ? UnsupportedRegexpError : CompileError
91
127
  raise klass.new(diagnostic.message, diagnostic: diagnostic)