flexr 1.0.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -1
  3. data/CHANGELOG.md +116 -0
  4. data/CONTRIBUTING.md +46 -3
  5. data/README.md +5 -4
  6. data/Rakefile +241 -72
  7. data/Steepfile +7 -0
  8. data/benchmark/baselines/json.json +2 -2
  9. data/benchmark/golden/calculator_lexer.rb +276 -0
  10. data/benchmark/golden/graphql_lexer.rb +310 -0
  11. data/benchmark/golden/json_lexer.rb +292 -0
  12. data/benchmark/golden/regexp_tokenizer.rb +293 -0
  13. data/benchmark/golden/ruby_subset_lexer.rb +280 -0
  14. data/benchmark/golden/sql_lexer.rb +293 -0
  15. data/benchmark/golden/toml_lexer.rb +292 -0
  16. data/benchmark/golden/toy_lang_lexer.rb +273 -0
  17. data/benchmark/golden/with_lrama_lexer.rb +264 -0
  18. data/benchmark/golden/with_racc_lexer.rb +264 -0
  19. data/benchmark/lrama_lexer.rb +165 -0
  20. data/benchmark/matrix.rb +79 -0
  21. data/benchmark/profile.rb +109 -0
  22. data/benchmark/run.rb +4 -1
  23. data/benchmark/state_dispatch.rb +78 -0
  24. data/docs/RELEASING.md +3 -2
  25. data/docs/explanation/backends.md +30 -10
  26. data/docs/explanation/matching-semantics.md +4 -2
  27. data/docs/explanation/security-model.md +10 -3
  28. data/docs/explanation/unicode-and-encoding.md +3 -1
  29. data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
  30. data/docs/how-to/generate-a-lexer.md +3 -3
  31. data/docs/how-to/handle-errors.md +4 -1
  32. data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
  33. data/docs/how-to/track-token-locations.md +2 -1
  34. data/docs/how-to/tune-performance.md +15 -3
  35. data/docs/migration-v2.md +74 -0
  36. data/docs/perf-log.md +212 -0
  37. data/docs/reference/README.md +2 -0
  38. data/docs/reference/actions.md +5 -2
  39. data/docs/reference/cli.md +71 -5
  40. data/docs/reference/compatibility.md +10 -4
  41. data/docs/reference/diagnostics.md +346 -0
  42. data/docs/reference/dsl.md +30 -9
  43. data/docs/reference/errors.md +34 -3
  44. data/docs/reference/generated-artifacts.md +71 -6
  45. data/docs/reference/public-api.md +37 -7
  46. data/docs/reference/regexp.md +27 -13
  47. data/docs/reference/ruby-lsp.md +34 -0
  48. data/docs/reference/runtime.md +21 -4
  49. data/docs/reference/testing.md +57 -0
  50. data/docs/reference/tokens-and-locations.md +14 -4
  51. data/examples/graphql/README.md +14 -0
  52. data/examples/graphql/lexer.flexr.rb +29 -0
  53. data/examples/json/lexer.flexr.rb +6 -6
  54. data/examples/sql/README.md +16 -0
  55. data/examples/sql/lexer.flexr.rb +23 -0
  56. data/examples/toml/README.md +14 -0
  57. data/examples/toml/lexer.flexr.rb +23 -0
  58. data/gemfiles/ruby_lsp.gemfile +4 -0
  59. data/gemfiles/types.gemfile +7 -0
  60. data/lib/flexr/action_resolver.rb +15 -0
  61. data/lib/flexr/artifact_writer.rb +61 -0
  62. data/lib/flexr/automaton/accel.rb +78 -5
  63. data/lib/flexr/automaton/analysis.rb +202 -7
  64. data/lib/flexr/automaton/backend_cost_model.rb +40 -0
  65. data/lib/flexr/automaton/compiler.rb +229 -96
  66. data/lib/flexr/automaton/dfa.rb +106 -19
  67. data/lib/flexr/automaton/minimizer.rb +79 -31
  68. data/lib/flexr/automaton/nfa.rb +45 -13
  69. data/lib/flexr/automaton/types.rb +14 -0
  70. data/lib/flexr/cli.rb +128 -22
  71. data/lib/flexr/codegen/direct.rb +5 -41
  72. data/lib/flexr/codegen/firstmatch.rb +1 -7
  73. data/lib/flexr/codegen/table.rb +181 -69
  74. data/lib/flexr/codegen.rb +8 -0
  75. data/lib/flexr/configuration.rb +45 -0
  76. data/lib/flexr/diagnostics.rb +39 -3
  77. data/lib/flexr/dsl.rb +116 -47
  78. data/lib/flexr/errors.rb +7 -6
  79. data/lib/flexr/generated.rb +120 -46
  80. data/lib/flexr/generator.rb +102 -109
  81. data/lib/flexr/grammar.rb +41 -0
  82. data/lib/flexr/importer.rb +4 -40
  83. data/lib/flexr/ir.rb +2 -2
  84. data/lib/flexr/migration.rb +31 -0
  85. data/lib/flexr/minitest.rb +24 -0
  86. data/lib/flexr/options.rb +9 -5
  87. data/lib/flexr/rake_task.rb +6 -1
  88. data/lib/flexr/regexp/ast.rb +5 -2
  89. data/lib/flexr/regexp/matcher.rb +26 -0
  90. data/lib/flexr/regexp/normalizer.rb +63 -21
  91. data/lib/flexr/regexp/parser.rb +241 -60
  92. data/lib/flexr/regexp/tokenizer.rb +172 -47
  93. data/lib/flexr/rspec.rb +51 -0
  94. data/lib/flexr/runtime/buffer.rb +115 -9
  95. data/lib/flexr/runtime/core.rb +248 -51
  96. data/lib/flexr/runtime/errors.rb +66 -4
  97. data/lib/flexr/runtime/interpreter.rb +213 -97
  98. data/lib/flexr/runtime/location.rb +35 -12
  99. data/lib/flexr/runtime/record.rb +14 -0
  100. data/lib/flexr/runtime/token.rb +7 -1
  101. data/lib/flexr/runtime.rb +75 -0
  102. data/lib/flexr/source/passthrough.rb +70 -14
  103. data/lib/flexr/source/prism_reader.rb +217 -51
  104. data/lib/flexr/source/static_eval.rb +40 -4
  105. data/lib/flexr/source.rb +6 -0
  106. data/lib/flexr/testing.rb +47 -0
  107. data/lib/flexr/trace.rb +197 -0
  108. data/lib/flexr/unicode/data/properties.rb +1 -1
  109. data/lib/flexr/unicode/data.rb +11 -0
  110. data/lib/flexr/unicode/property.rb +4 -8
  111. data/lib/flexr/unicode/reference_regexp.rb +11 -9
  112. data/lib/flexr/unicode/version.rb +7 -0
  113. data/lib/flexr/version.rb +1 -1
  114. data/lib/flexr.rb +10 -76
  115. data/lib/ruby_lsp/flexr/addon.rb +114 -0
  116. data/sig/flexr.rbs +191 -0
  117. data/site/README.md +54 -3
  118. data/site/astro.config.mjs +3 -1
  119. data/site/package.json +12 -5
  120. data/site/playwright.config.mjs +16 -0
  121. data/site/pnpm-lock.yaml +218 -88
  122. data/site/ruby/playground.rb +57 -0
  123. data/site/scripts/prepare-playground.mjs +35 -0
  124. data/site/scripts/verify-site.mjs +5 -3
  125. data/site/src/content/docs/concepts/matching-semantics.md +4 -2
  126. data/site/src/content/docs/concepts/security-model.md +9 -1
  127. data/site/src/content/docs/learn/generation.md +1 -1
  128. data/site/src/content/docs/reference/dsl.md +2 -2
  129. data/site/src/content/docs/reference/public-api.md +4 -2
  130. data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
  131. data/site/src/layouts/SiteLayout.astro +11 -3
  132. data/site/src/pages/index.astro +69 -155
  133. data/site/src/pages/playground.astro +40 -38
  134. data/site/src/playground/client.js +211 -0
  135. data/site/src/playground/graph-worker.mjs +11 -0
  136. data/site/src/playground/presets.mjs +33 -0
  137. data/site/src/playground/runtime.mjs +34 -0
  138. data/site/src/playground/share.mjs +30 -0
  139. data/site/src/playground/worker.mjs +29 -0
  140. data/site/src/styles/custom.css +12 -287
  141. data/site/src/styles/landing.css +239 -0
  142. data/site/tests/browser/playground.spec.mjs +99 -0
  143. data/site/tests/playground-runtime.test.mjs +85 -0
  144. data/test/ruby_lsp_addon_spec.rb +135 -0
  145. data/tools/coverage.rb +6 -1
  146. data/tools/docs_verify.rb +3 -2
  147. data/tools/golden.rb +54 -0
  148. data/tools/verification_failure.rb +119 -0
  149. metadata +66 -12
  150. data/benchmark/golden/calculator_lexer.sha256 +0 -1
  151. data/benchmark/golden/json_lexer.sha256 +0 -1
  152. data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
  153. data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
  154. data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
  155. data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
  156. data/benchmark/golden/with_racc_lexer.sha256 +0 -1
  157. data/site/public/playground.js +0 -189
@@ -0,0 +1,292 @@
1
+ # frozen_string_literal: true
2
+ # Generated by flexr. DO NOT EDIT.
3
+ # source: examples/json/lexer.flexr.rb
4
+ # spec-digest: <normalized>
5
+ # artifact-schema: 2
6
+ # runtime-abi: 2
7
+ # unicode: 15.1.0
8
+ # backend: direct
9
+ # compiled: true
10
+ # eval: false
11
+ # standalone: false
12
+
13
+ require "flexr"
14
+
15
+ module JsonExample
16
+ class Lexer < Flexr::Lexer
17
+ Flexr::Generated.install_compiled!(self, { artifact: {schema_version: 2, compiler_version: "<compiler-version>", runtime_abi_version: 2, unicode_version: "15.1.0"}, rules: [{ index: 0, patterns: [/[ \t\r\n]+/], pattern_conditions: [[0, 0, false, false]], trailing: nil, action: :skip, states: [:initial], bol_only: false, end_anchor: nil }, { index: 1, patterns: [/\{/], pattern_conditions: [[1, 0, false, false]], trailing: nil, action: [:emit, :LBRACE], states: [:initial], bol_only: false, end_anchor: nil }, { index: 2, patterns: [/\}/], pattern_conditions: [[2, 0, false, false]], trailing: nil, action: [:emit, :RBRACE], states: [:initial], bol_only: false, end_anchor: nil }, { index: 3, patterns: [/\[/], pattern_conditions: [[3, 0, false, false]], trailing: nil, action: [:emit, :LBRACKET], states: [:initial], bol_only: false, end_anchor: nil }, { index: 4, patterns: [/\]/], pattern_conditions: [[4, 0, false, false]], trailing: nil, action: [:emit, :RBRACKET], states: [:initial], bol_only: false, end_anchor: nil }, { index: 5, patterns: [/:/], pattern_conditions: [[5, 0, false, false]], trailing: nil, action: [:emit, :COLON], states: [:initial], bol_only: false, end_anchor: nil }, { index: 6, patterns: [/,/], pattern_conditions: [[6, 0, false, false]], trailing: nil, action: [:emit, :COMMA], states: [:initial], bol_only: false, end_anchor: nil }, { index: 7, patterns: [/true/], pattern_conditions: [[7, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 8, patterns: [/false/], pattern_conditions: [[8, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 9, patterns: [/null/], pattern_conditions: [[9, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 10, patterns: [/-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?/], pattern_conditions: [[10, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 11, patterns: [/"(?:\\.|[^"\\])*"/], pattern_conditions: [[11, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }], backend: :direct, token_kind: :array, encoding: Encoding::UTF_8, declared_tokens: [:LBRACE, :RBRACE, :LBRACKET, :RBRACKET, :COLON, :COMMA, :STRING, :NUMBER, :TRUE, :FALSE, :NULL], options: {}, eof_rules: { }, states: [], inclusive_states: {initial: true}, compiled: { machines: { :initial => { state_name: :initial, dfa: {accepts: [[], [[0, 0, false, false]], [], [[6, 0, false, false]], [], [[10, 0, false, false]], [[10, 0, false, false]], [[5, 0, false, false]], [[3, 0, false, false]], [[4, 0, false, false]], [], [], [], [[1, 0, false, false]], [[2, 0, false, false]], [[11, 0, false, false]], [], [], [], [], [], [], [], [], [], [[10, 0, false, false]], [], [[10, 0, false, false]], [], [], [], [], [[9, 0, false, false]], [[7, 0, false, false]], [[8, 0, false, false]]], ec: [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 3, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 7, 8, 9, 9, 9, 9, 9, 9, 9, 9, 10, 11, 12, 13, 14, 15, 16, 16, 16, 16, 16, 16, 16, 16, 16, 17, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 19, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 21, 22, 23, 24, 24, 24, 25, 26, 26, 26, 27, 28, 29, 29, 29, 29, 29, 30, 31, 32, 33, 33, 33, 34, 35, 36, 37, 38, 38, 38, 38, 38, 39, 40, 41, 42, 42, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 44, 44, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 47, 47, 47, 47, 47, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48], class_count: 49, state_count: 35, start: 0, rule_ids: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], direct: {nxt: [-1, 1, 1, -1, 1, -1, 1, -1, 2, -1, -1, 3, 4, -1, -1, 5, 6, 7, -1, -1, -1, 8, -1, 9, -1, -1, -1, -1, 10, -1, -1, -1, 11, -1, -1, -1, 12, -1, -1, 13, -1, 14, -1, -1, -1, -1, -1, -1, -1, -1, 1, 1, -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, 2, 2, 2, 2, 2, 2, 2, 15, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 16, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -1, -1, 17, 18, 19, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 5, 6, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 20, -1, -1, -1, -1, -1, 21, -1, -1, -1, -1, -1, -1, -1, 21, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 20, -1, 6, 6, -1, -1, 21, -1, -1, -1, -1, -1, -1, -1, 21, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 22, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 24, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, 2, -1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -1, -1, 17, 18, 19, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 17, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 18, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 25, 25, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 26, -1, 26, -1, -1, 27, 27, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 28, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 29, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 30, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 25, 25, -1, -1, 21, -1, -1, -1, -1, -1, -1, -1, 21, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 27, 27, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 27, 27, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 31, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 32, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 33, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 34, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], classes: 49}} } }, states: [:initial], stats: {initial: {states: 35, classes: 49, accepting_states: 15}}, diagnostics: [] } })
18
+ def scan_one
19
+ if @__flexr_generated_scan_state != @state
20
+ machine = self.class.__flexr_compiled.machines.fetch(@state)
21
+ @__flexr_generated_scan_state = @state
22
+ @__flexr_generated_scan_dfa = machine.dfa
23
+ @__flexr_generated_scan_fast = __flexr_generated_fast_path?(@__flexr_generated_scan_dfa)
24
+ @__flexr_generated_scan_simple = __flexr_generated_simple_fast_path?(@__flexr_generated_scan_dfa)
25
+ end
26
+ dfa = @__flexr_generated_scan_dfa
27
+ return Flexr::Runtime::Interpreter.new(self).scan unless @__flexr_generated_scan_fast
28
+ position = byte_pos
29
+ return nil unless valid_utf8_at?(position)
30
+ return __flexr_generated_scan_simple(dfa, position) if @__flexr_generated_scan_simple
31
+ cursor = position
32
+ current = dfa.start
33
+ direct = dfa.direct
34
+ direct_nxt = direct&.fetch(:nxt)
35
+ direct_classes = direct&.fetch(:classes)
36
+ ec = dfa.ec
37
+ source = buffer.stable_source
38
+ guarded_steps = scan_steps_guarded?
39
+ token_start = more_text_start || position
40
+ token_limit = max_token_size
41
+ check_token_limit = token_limit_required?
42
+ best = nil
43
+ accel_mode = @accel_mode
44
+ accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)
45
+ while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)
46
+ if accelerate
47
+ region = __flexr_generated_acceleration_region(dfa, current)
48
+ acceptance = dfa.accepts[current].first
49
+ token_rule = acceptance&.rule_index
50
+ accelerated_end = __flexr_generated_accelerate(region, cursor, token_rule, position) if region
51
+ if accelerated_end && accelerated_end > cursor
52
+ cursor = accelerated_end
53
+ best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
54
+ next
55
+ end
56
+ end
57
+ byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)
58
+ consume_step! if guarded_steps
59
+ current = if direct_nxt
60
+ class_id = ec[byte]
61
+ value = direct_nxt[(current * direct_classes) + class_id]
62
+ value >= 0 ? value : nil
63
+ else
64
+ dfa.transition(current, byte)
65
+ end
66
+ break unless current
67
+ cursor += 1
68
+ resource_acceptance = dfa.accepts[current].first
69
+ resource_rule = resource_acceptance&.rule_index
70
+ if check_token_limit
71
+ size = cursor - token_start
72
+ defer_token_size_check!(size, rule: resource_rule) if size > token_limit
73
+ end
74
+ best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
75
+ end
76
+ best
77
+ end
78
+ def __flexr_generated_scan_simple(dfa, position)
79
+ current = dfa.start
80
+ direct = dfa.direct
81
+ direct_nxt = direct&.fetch(:nxt)
82
+ direct_classes = direct&.fetch(:classes)
83
+ ec = dfa.ec
84
+ accepts = dfa.accepts
85
+ rules = self.class.__flexr_rules
86
+ source = buffer.stable_source
87
+ guarded_steps = scan_steps_guarded?
88
+ token_start = more_text_start || position
89
+ token_limit = max_token_size
90
+ check_token_limit = token_limit_required?
91
+ cursor = position
92
+ best = nil
93
+ accel_mode = @accel_mode
94
+ accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)
95
+ if source && !guarded_steps && !check_token_limit && !accelerate
96
+ while cursor < source.bytesize
97
+ byte = source.getbyte(cursor)
98
+ current = if direct_nxt
99
+ class_id = ec[byte]
100
+ value = direct_nxt[(current * direct_classes) + class_id]
101
+ value >= 0 ? value : nil
102
+ else
103
+ dfa.transition(current, byte)
104
+ end
105
+ break unless current
106
+ cursor += 1
107
+ acceptance = accepts[current].first
108
+ next unless acceptance
109
+ rule = rules.fetch(acceptance.rule_index)
110
+ best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
111
+ best.rule = rule
112
+ best.start_pos = position
113
+ best.end_pos = cursor
114
+ best.total_end_pos = cursor
115
+ end
116
+ return best
117
+ end
118
+ while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)
119
+ if accelerate
120
+ region = __flexr_generated_acceleration_region(dfa, current)
121
+ acceptance = accepts[current].first
122
+ accelerated_end = __flexr_generated_accelerate(region, cursor, acceptance&.rule_index, position) if region
123
+ if accelerated_end && accelerated_end > cursor
124
+ cursor = accelerated_end
125
+ best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
126
+ next
127
+ end
128
+ end
129
+ byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)
130
+ consume_step! if guarded_steps
131
+ current = if direct_nxt
132
+ class_id = ec[byte]
133
+ value = direct_nxt[(current * direct_classes) + class_id]
134
+ value >= 0 ? value : nil
135
+ else
136
+ dfa.transition(current, byte)
137
+ end
138
+ break unless current
139
+ cursor += 1
140
+ acceptance = accepts[current].first
141
+ if check_token_limit
142
+ size = cursor - token_start
143
+ defer_token_size_check!(size, rule: acceptance&.rule_index) if size > token_limit
144
+ end
145
+ next unless acceptance
146
+ rule = rules.fetch(acceptance.rule_index)
147
+ best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
148
+ best.rule = rule
149
+ best.start_pos = position
150
+ best.end_pos = cursor
151
+ best.total_end_pos = cursor
152
+ end
153
+ best
154
+ end
155
+ def __flexr_generated_acceleration_region(dfa, state)
156
+ @__flexr_generated_accel_regions ||= {}
157
+ regions = (@__flexr_generated_accel_regions[dfa] ||= Flexr::Automaton::Accel.extract(dfa).to_h { |region| [region.state, region] })
158
+ region = regions[state]
159
+ return unless region
160
+ return if @__flexr_generated_disabled_auto_accel&.key?(region)
161
+ return if utf8_input? && region.bytes.any? { |byte| byte >= 128 }
162
+ return if dfa.accepts[state].any? do |acceptance|
163
+ rule = self.class.__flexr_rules.fetch(acceptance.rule_index)
164
+ acceptance.bol_only || acceptance.end_anchor || rule.trailing
165
+ end
166
+ region
167
+ end
168
+ def __flexr_generated_accelerate(region, position, token_rule, token_start)
169
+ mode = @accel_mode
170
+ cursor = position
171
+ matched = false
172
+ loop do
173
+ if cursor >= buffer.bytesize
174
+ return matched ? cursor : nil if buffer.eof_loaded?
175
+ return matched ? cursor : nil unless buffer.ensure_available?(cursor + 1)
176
+ end
177
+ segment = buffer.source
178
+ segment_position = cursor - buffer.base_offset
179
+ if segment.encoding == ::Encoding::UTF_8 && region.utf8_regexp
180
+ regexp = region.utf8_regexp
181
+ else
182
+ segment = segment.b unless segment.encoding == ::Encoding::BINARY
183
+ regexp = region.regexp
184
+ end
185
+ length = begin
186
+ if %i[strscan auto].include?(mode) && defined?(::StringScanner)
187
+ @__flexr_generated_accel_scanner ||= ::StringScanner.new("".b)
188
+ @__flexr_generated_accel_scanner.string = segment
189
+ @__flexr_generated_accel_scanner.pos = segment_position
190
+ @__flexr_generated_accel_scanner.skip(regexp)
191
+ else
192
+ match = regexp.match(segment.byteslice(segment_position..).to_s, 0)
193
+ match && match.begin(0).zero? ? match[0].bytesize : nil
194
+ end
195
+ rescue ArgumentError
196
+ nil
197
+ end
198
+ __flexr_generated_record_auto_accel_miss(region) if mode == :auto && length.to_i < 8
199
+ return matched ? cursor : nil unless length&.positive?
200
+ matched = true
201
+ cursor += length
202
+ consume_step!(length)
203
+ defer_token_size_check!(cursor - (more_text_start || token_start), rule: token_rule)
204
+ return cursor if cursor < buffer.bytesize || buffer.eof_loaded?
205
+ end
206
+ end
207
+ def __flexr_generated_record_auto_accel_miss(region)
208
+ @__flexr_generated_auto_accel_misses ||= Hash.new(0)
209
+ @__flexr_generated_auto_accel_misses[region] += 1
210
+ @__flexr_generated_auto_accel_miss_total = @__flexr_generated_auto_accel_miss_total.to_i + 1
211
+ @__flexr_generated_auto_accel_off = true if @__flexr_generated_auto_accel_miss_total >= 9
212
+ return if @__flexr_generated_auto_accel_misses[region] < 3
213
+ @__flexr_generated_disabled_auto_accel ||= {}
214
+ @__flexr_generated_disabled_auto_accel[region] = true
215
+ end
216
+ def __flexr_generated_fast_path?(dfa)
217
+ @__flexr_generated_fast_paths ||= {}
218
+ return @__flexr_generated_fast_paths[dfa] if @__flexr_generated_fast_paths.key?(dfa)
219
+ rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }
220
+ @__flexr_generated_fast_paths[dfa] = self.class.__flexr_config.backend != :firstmatch &&
221
+ self.class.__flexr_config.options[:allow_empty_match] != true && rules.none?(&:trailing)
222
+ end
223
+ def __flexr_generated_simple_fast_path?(dfa)
224
+ @__flexr_generated_simple_fast_paths ||= {}
225
+ return @__flexr_generated_simple_fast_paths[dfa] if @__flexr_generated_simple_fast_paths.key?(dfa)
226
+ rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }
227
+ @__flexr_generated_simple_fast_paths[dfa] = rules.all? do |rule|
228
+ rule.pattern_conditions.all? { |condition| condition && !condition.bol_only && !condition.end_anchor }
229
+ end
230
+ end
231
+ def __flexr_generated_acceptance(dfa, state, start_position, cursor, best)
232
+ dfa.accepts[state].each do |acceptance|
233
+ next if acceptance.bol_only && !beginning_of_line?
234
+ next if acceptance.end_anchor && !(buffer.eof?(cursor) || buffer.getbyte(cursor) == 0x0a)
235
+ rule = self.class.__flexr_rules.fetch(acceptance.rule_index)
236
+ next if best && cursor < best.total_end_pos
237
+ next if best && cursor == best.total_end_pos &&
238
+ rule.index > best.rule.index
239
+ best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
240
+ best.rule = rule
241
+ best.start_pos = start_position
242
+ best.end_pos = cursor
243
+ best.total_end_pos = cursor
244
+ end
245
+ best
246
+ end
247
+ def __flexr_generated_execute(rule)
248
+ case rule.index
249
+ when 0
250
+ when 1
251
+ emit(:LBRACE, text)
252
+ when 2
253
+ emit(:RBRACE, text)
254
+ when 3
255
+ emit(:LBRACKET, text)
256
+ when 4
257
+ emit(:RBRACKET, text)
258
+ when 5
259
+ emit(:COLON, text)
260
+ when 6
261
+ emit(:COMMA, text)
262
+ when 7
263
+ emit :TRUE, true
264
+ when 8
265
+ emit :FALSE, false
266
+ when 9
267
+ emit :NULL, nil
268
+ when 10
269
+ emit :NUMBER, text.to_f
270
+ when 11
271
+ emit :STRING, text.byteslice(1...-1)
272
+ else
273
+ instance_exec(&rule.action)
274
+ end
275
+ end
276
+
277
+
278
+
279
+
280
+
281
+
282
+
283
+
284
+
285
+
286
+
287
+
288
+
289
+
290
+
291
+ end
292
+ end
@@ -0,0 +1,293 @@
1
+ # frozen_string_literal: true
2
+ # Generated by flexr. DO NOT EDIT.
3
+ # source: lib/flexr/regexp/tokenizer.flexr.rb
4
+ # spec-digest: <normalized>
5
+ # artifact-schema: 2
6
+ # runtime-abi: 2
7
+ # unicode: 15.1.0
8
+ # backend: table
9
+ # compiled: true
10
+ # eval: false
11
+ # standalone: false
12
+
13
+ require "flexr"
14
+
15
+ module Flexr
16
+ module Regexp
17
+ # This specification is intentionally small: it is a dogfood target for
18
+ # the source lexer path, not the parser's implementation.
19
+ class SourceLexer < Flexr::Lexer
20
+ Flexr::Generated.install_compiled!(self, { artifact: {schema_version: 2, compiler_version: "<compiler-version>", runtime_abi_version: 2, unicode_version: "15.1.0"}, rules: [{ index: 0, patterns: [/[ \t\r\n]+/], pattern_conditions: [[0, 0, false, false]], trailing: nil, action: :skip, states: [:initial], bol_only: false, end_anchor: nil }, { index: 1, patterns: [/\\p\{[A-Za-z_][A-Za-z0-9_]*\}/], pattern_conditions: [[1, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 2, patterns: [/\\./], pattern_conditions: [[2, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 3, patterns: [/\[[^\]\n]*\]/], pattern_conditions: [[3, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 4, patterns: [/[?*+]|\{[0-9]+(?:,[0-9]*)?\}/], pattern_conditions: [[4, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 5, patterns: [/\|/], pattern_conditions: [[5, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 6, patterns: [/\(/], pattern_conditions: [[6, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 7, patterns: [/\)/], pattern_conditions: [[7, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 8, patterns: [/\^|\$/], pattern_conditions: [[8, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 9, patterns: [/\./], pattern_conditions: [[9, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 10, patterns: [/[^\\\[\]().|?*+{}^$ \t\r\n]/], pattern_conditions: [[10, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }], backend: :table, token_kind: :array, encoding: Encoding::BINARY, declared_tokens: [:PROPERTY, :ESCAPE, :CHAR_CLASS, :QUANTIFIER, :ALTERNATION, :GROUP_OPEN, :GROUP_CLOSE, :ANCHOR, :DOT, :LITERAL], options: {}, eof_rules: { }, states: [], inclusive_states: {initial: true}, compiled: { machines: { :initial => { state_name: :initial, dfa: {accepts: [[], [[10, 0, false, false]], [[0, 0, false, false]], [[8, 0, false, false]], [[6, 0, false, false]], [[7, 0, false, false]], [[4, 0, false, false]], [[9, 0, false, false]], [], [], [], [[5, 0, false, false]], [[3, 0, false, false]], [[2, 0, false, false]], [[2, 0, false, false]], [], [], [], [], [[1, 0, false, false]]], ec: [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 3, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 7, 7, 7, 8, 9, 9, 9, 10, 11, 12, 12, 13, 14, 15, 16, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, 19, 20, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 22, 23, 24, 25, 26, 27, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 29, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 31, 32, 33, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34], class_count: 35, state_count: 20, start: 0, rule_ids: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], packed: {base: [0, 0, 12, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 3, 22, 15, 4, 21, 0], default: [1, nil, nil, nil, nil, nil, nil, nil, 8, 13, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil], next: [nil, 2, 2, nil, 2, nil, 2, nil, 3, nil, 4, 5, 6, 2, 2, 7, 2, 15, 2, 6, nil, 17, 8, 9, nil, 3, nil, 12, nil, nil, 14, 10, 11, nil, 16, 17, 18, 6, 18, 15, nil, 18, 18, 18, 18, 18, nil, 18, nil, 18, 18, 18, nil, nil, 19, 6], check: [nil, 0, 0, 9, 0, 8, 0, nil, 0, nil, 0, 0, 0, 2, 2, 0, 2, 10, 2, 0, nil, 17, 0, 0, 0, 0, nil, 8, nil, nil, 9, 0, 0, 0, 14, 15, 16, 17, 18, 15, nil, 16, 18, 16, 16, 16, nil, 18, nil, 18, 18, 18, nil, nil, 18, 15]}} } }, states: [:initial], stats: {initial: {states: 20, classes: 35, accepting_states: 12}}, diagnostics: [] } })
21
+ def scan_one
22
+ if @__flexr_generated_scan_state != @state
23
+ machine = self.class.__flexr_compiled.machines.fetch(@state)
24
+ @__flexr_generated_scan_state = @state
25
+ @__flexr_generated_scan_dfa = machine.dfa
26
+ @__flexr_generated_scan_fast = __flexr_generated_fast_path?(@__flexr_generated_scan_dfa)
27
+ @__flexr_generated_scan_simple = __flexr_generated_simple_fast_path?(@__flexr_generated_scan_dfa)
28
+ end
29
+ dfa = @__flexr_generated_scan_dfa
30
+ return Flexr::Runtime::Interpreter.new(self).scan unless @__flexr_generated_scan_fast
31
+ position = byte_pos
32
+ return nil unless valid_utf8_at?(position)
33
+ return __flexr_generated_scan_simple(dfa, position) if @__flexr_generated_scan_simple
34
+ cursor = position
35
+ current = dfa.start
36
+ direct = dfa.direct
37
+ direct_nxt = direct&.fetch(:nxt)
38
+ direct_classes = direct&.fetch(:classes)
39
+ ec = dfa.ec
40
+ source = buffer.stable_source
41
+ guarded_steps = scan_steps_guarded?
42
+ token_start = more_text_start || position
43
+ token_limit = max_token_size
44
+ check_token_limit = token_limit_required?
45
+ best = nil
46
+ accel_mode = @accel_mode
47
+ accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)
48
+ while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)
49
+ if accelerate
50
+ region = __flexr_generated_acceleration_region(dfa, current)
51
+ acceptance = dfa.accepts[current].first
52
+ token_rule = acceptance&.rule_index
53
+ accelerated_end = __flexr_generated_accelerate(region, cursor, token_rule, position) if region
54
+ if accelerated_end && accelerated_end > cursor
55
+ cursor = accelerated_end
56
+ best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
57
+ next
58
+ end
59
+ end
60
+ byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)
61
+ consume_step! if guarded_steps
62
+ current = if direct_nxt
63
+ class_id = ec[byte]
64
+ value = direct_nxt[(current * direct_classes) + class_id]
65
+ value >= 0 ? value : nil
66
+ else
67
+ dfa.transition(current, byte)
68
+ end
69
+ break unless current
70
+ cursor += 1
71
+ resource_acceptance = dfa.accepts[current].first
72
+ resource_rule = resource_acceptance&.rule_index
73
+ if check_token_limit
74
+ size = cursor - token_start
75
+ defer_token_size_check!(size, rule: resource_rule) if size > token_limit
76
+ end
77
+ best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
78
+ end
79
+ best
80
+ end
81
+ def __flexr_generated_scan_simple(dfa, position)
82
+ current = dfa.start
83
+ direct = dfa.direct
84
+ direct_nxt = direct&.fetch(:nxt)
85
+ direct_classes = direct&.fetch(:classes)
86
+ ec = dfa.ec
87
+ accepts = dfa.accepts
88
+ rules = self.class.__flexr_rules
89
+ source = buffer.stable_source
90
+ guarded_steps = scan_steps_guarded?
91
+ token_start = more_text_start || position
92
+ token_limit = max_token_size
93
+ check_token_limit = token_limit_required?
94
+ cursor = position
95
+ best = nil
96
+ accel_mode = @accel_mode
97
+ accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)
98
+ if source && !guarded_steps && !check_token_limit && !accelerate
99
+ while cursor < source.bytesize
100
+ byte = source.getbyte(cursor)
101
+ current = if direct_nxt
102
+ class_id = ec[byte]
103
+ value = direct_nxt[(current * direct_classes) + class_id]
104
+ value >= 0 ? value : nil
105
+ else
106
+ dfa.transition(current, byte)
107
+ end
108
+ break unless current
109
+ cursor += 1
110
+ acceptance = accepts[current].first
111
+ next unless acceptance
112
+ rule = rules.fetch(acceptance.rule_index)
113
+ best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
114
+ best.rule = rule
115
+ best.start_pos = position
116
+ best.end_pos = cursor
117
+ best.total_end_pos = cursor
118
+ end
119
+ return best
120
+ end
121
+ while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)
122
+ if accelerate
123
+ region = __flexr_generated_acceleration_region(dfa, current)
124
+ acceptance = accepts[current].first
125
+ accelerated_end = __flexr_generated_accelerate(region, cursor, acceptance&.rule_index, position) if region
126
+ if accelerated_end && accelerated_end > cursor
127
+ cursor = accelerated_end
128
+ best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
129
+ next
130
+ end
131
+ end
132
+ byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)
133
+ consume_step! if guarded_steps
134
+ current = if direct_nxt
135
+ class_id = ec[byte]
136
+ value = direct_nxt[(current * direct_classes) + class_id]
137
+ value >= 0 ? value : nil
138
+ else
139
+ dfa.transition(current, byte)
140
+ end
141
+ break unless current
142
+ cursor += 1
143
+ acceptance = accepts[current].first
144
+ if check_token_limit
145
+ size = cursor - token_start
146
+ defer_token_size_check!(size, rule: acceptance&.rule_index) if size > token_limit
147
+ end
148
+ next unless acceptance
149
+ rule = rules.fetch(acceptance.rule_index)
150
+ best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
151
+ best.rule = rule
152
+ best.start_pos = position
153
+ best.end_pos = cursor
154
+ best.total_end_pos = cursor
155
+ end
156
+ best
157
+ end
158
+ def __flexr_generated_acceleration_region(dfa, state)
159
+ @__flexr_generated_accel_regions ||= {}
160
+ regions = (@__flexr_generated_accel_regions[dfa] ||= Flexr::Automaton::Accel.extract(dfa).to_h { |region| [region.state, region] })
161
+ region = regions[state]
162
+ return unless region
163
+ return if @__flexr_generated_disabled_auto_accel&.key?(region)
164
+ return if utf8_input? && region.bytes.any? { |byte| byte >= 128 }
165
+ return if dfa.accepts[state].any? do |acceptance|
166
+ rule = self.class.__flexr_rules.fetch(acceptance.rule_index)
167
+ acceptance.bol_only || acceptance.end_anchor || rule.trailing
168
+ end
169
+ region
170
+ end
171
+ def __flexr_generated_accelerate(region, position, token_rule, token_start)
172
+ mode = @accel_mode
173
+ cursor = position
174
+ matched = false
175
+ loop do
176
+ if cursor >= buffer.bytesize
177
+ return matched ? cursor : nil if buffer.eof_loaded?
178
+ return matched ? cursor : nil unless buffer.ensure_available?(cursor + 1)
179
+ end
180
+ segment = buffer.source
181
+ segment_position = cursor - buffer.base_offset
182
+ if segment.encoding == ::Encoding::UTF_8 && region.utf8_regexp
183
+ regexp = region.utf8_regexp
184
+ else
185
+ segment = segment.b unless segment.encoding == ::Encoding::BINARY
186
+ regexp = region.regexp
187
+ end
188
+ length = begin
189
+ if %i[strscan auto].include?(mode) && defined?(::StringScanner)
190
+ @__flexr_generated_accel_scanner ||= ::StringScanner.new("".b)
191
+ @__flexr_generated_accel_scanner.string = segment
192
+ @__flexr_generated_accel_scanner.pos = segment_position
193
+ @__flexr_generated_accel_scanner.skip(regexp)
194
+ else
195
+ match = regexp.match(segment.byteslice(segment_position..).to_s, 0)
196
+ match && match.begin(0).zero? ? match[0].bytesize : nil
197
+ end
198
+ rescue ArgumentError
199
+ nil
200
+ end
201
+ __flexr_generated_record_auto_accel_miss(region) if mode == :auto && length.to_i < 8
202
+ return matched ? cursor : nil unless length&.positive?
203
+ matched = true
204
+ cursor += length
205
+ consume_step!(length)
206
+ defer_token_size_check!(cursor - (more_text_start || token_start), rule: token_rule)
207
+ return cursor if cursor < buffer.bytesize || buffer.eof_loaded?
208
+ end
209
+ end
210
+ def __flexr_generated_record_auto_accel_miss(region)
211
+ @__flexr_generated_auto_accel_misses ||= Hash.new(0)
212
+ @__flexr_generated_auto_accel_misses[region] += 1
213
+ @__flexr_generated_auto_accel_miss_total = @__flexr_generated_auto_accel_miss_total.to_i + 1
214
+ @__flexr_generated_auto_accel_off = true if @__flexr_generated_auto_accel_miss_total >= 9
215
+ return if @__flexr_generated_auto_accel_misses[region] < 3
216
+ @__flexr_generated_disabled_auto_accel ||= {}
217
+ @__flexr_generated_disabled_auto_accel[region] = true
218
+ end
219
+ def __flexr_generated_fast_path?(dfa)
220
+ @__flexr_generated_fast_paths ||= {}
221
+ return @__flexr_generated_fast_paths[dfa] if @__flexr_generated_fast_paths.key?(dfa)
222
+ rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }
223
+ @__flexr_generated_fast_paths[dfa] = self.class.__flexr_config.backend != :firstmatch &&
224
+ self.class.__flexr_config.options[:allow_empty_match] != true && rules.none?(&:trailing)
225
+ end
226
+ def __flexr_generated_simple_fast_path?(dfa)
227
+ @__flexr_generated_simple_fast_paths ||= {}
228
+ return @__flexr_generated_simple_fast_paths[dfa] if @__flexr_generated_simple_fast_paths.key?(dfa)
229
+ rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }
230
+ @__flexr_generated_simple_fast_paths[dfa] = rules.all? do |rule|
231
+ rule.pattern_conditions.all? { |condition| condition && !condition.bol_only && !condition.end_anchor }
232
+ end
233
+ end
234
+ def __flexr_generated_acceptance(dfa, state, start_position, cursor, best)
235
+ dfa.accepts[state].each do |acceptance|
236
+ next if acceptance.bol_only && !beginning_of_line?
237
+ next if acceptance.end_anchor && !(buffer.eof?(cursor) || buffer.getbyte(cursor) == 0x0a)
238
+ rule = self.class.__flexr_rules.fetch(acceptance.rule_index)
239
+ next if best && cursor < best.total_end_pos
240
+ next if best && cursor == best.total_end_pos &&
241
+ rule.index > best.rule.index
242
+ best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
243
+ best.rule = rule
244
+ best.start_pos = start_position
245
+ best.end_pos = cursor
246
+ best.total_end_pos = cursor
247
+ end
248
+ best
249
+ end
250
+ def __flexr_generated_execute(rule)
251
+ case rule.index
252
+ when 0
253
+ when 1
254
+ emit :PROPERTY, text.byteslice(3...-1)
255
+ when 2
256
+ emit :ESCAPE, text
257
+ when 3
258
+ emit :CHAR_CLASS, text
259
+ when 4
260
+ emit :QUANTIFIER, text
261
+ when 5
262
+ emit :ALTERNATION, text
263
+ when 6
264
+ emit :GROUP_OPEN, text
265
+ when 7
266
+ emit :GROUP_CLOSE, text
267
+ when 8
268
+ emit :ANCHOR, text
269
+ when 9
270
+ emit :DOT, text
271
+ when 10
272
+ emit :LITERAL, text
273
+ else
274
+ instance_exec(&rule.action)
275
+ end
276
+ end
277
+
278
+
279
+
280
+
281
+
282
+
283
+
284
+
285
+
286
+
287
+
288
+
289
+
290
+
291
+ end
292
+ end
293
+ end