flexr 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/CHANGELOG.md +116 -0
- data/CONTRIBUTING.md +46 -3
- data/README.md +5 -4
- data/Rakefile +241 -72
- data/Steepfile +7 -0
- data/benchmark/baselines/json.json +2 -2
- data/benchmark/golden/calculator_lexer.rb +276 -0
- data/benchmark/golden/graphql_lexer.rb +310 -0
- data/benchmark/golden/json_lexer.rb +292 -0
- data/benchmark/golden/regexp_tokenizer.rb +293 -0
- data/benchmark/golden/ruby_subset_lexer.rb +280 -0
- data/benchmark/golden/sql_lexer.rb +293 -0
- data/benchmark/golden/toml_lexer.rb +292 -0
- data/benchmark/golden/toy_lang_lexer.rb +273 -0
- data/benchmark/golden/with_lrama_lexer.rb +264 -0
- data/benchmark/golden/with_racc_lexer.rb +264 -0
- data/benchmark/lrama_lexer.rb +165 -0
- data/benchmark/matrix.rb +79 -0
- data/benchmark/profile.rb +109 -0
- data/benchmark/run.rb +4 -1
- data/benchmark/state_dispatch.rb +78 -0
- data/docs/RELEASING.md +3 -2
- data/docs/explanation/backends.md +30 -10
- data/docs/explanation/matching-semantics.md +4 -2
- data/docs/explanation/security-model.md +10 -3
- data/docs/explanation/unicode-and-encoding.md +3 -1
- data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
- data/docs/how-to/generate-a-lexer.md +3 -3
- data/docs/how-to/handle-errors.md +4 -1
- data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
- data/docs/how-to/track-token-locations.md +2 -1
- data/docs/how-to/tune-performance.md +15 -3
- data/docs/migration-v2.md +74 -0
- data/docs/perf-log.md +212 -0
- data/docs/reference/README.md +2 -0
- data/docs/reference/actions.md +5 -2
- data/docs/reference/cli.md +71 -5
- data/docs/reference/compatibility.md +10 -4
- data/docs/reference/diagnostics.md +346 -0
- data/docs/reference/dsl.md +30 -9
- data/docs/reference/errors.md +34 -3
- data/docs/reference/generated-artifacts.md +71 -6
- data/docs/reference/public-api.md +37 -7
- data/docs/reference/regexp.md +27 -13
- data/docs/reference/ruby-lsp.md +34 -0
- data/docs/reference/runtime.md +21 -4
- data/docs/reference/testing.md +57 -0
- data/docs/reference/tokens-and-locations.md +14 -4
- data/examples/graphql/README.md +14 -0
- data/examples/graphql/lexer.flexr.rb +29 -0
- data/examples/json/lexer.flexr.rb +6 -6
- data/examples/sql/README.md +16 -0
- data/examples/sql/lexer.flexr.rb +23 -0
- data/examples/toml/README.md +14 -0
- data/examples/toml/lexer.flexr.rb +23 -0
- data/gemfiles/ruby_lsp.gemfile +4 -0
- data/gemfiles/types.gemfile +7 -0
- data/lib/flexr/action_resolver.rb +15 -0
- data/lib/flexr/artifact_writer.rb +61 -0
- data/lib/flexr/automaton/accel.rb +78 -5
- data/lib/flexr/automaton/analysis.rb +202 -7
- data/lib/flexr/automaton/backend_cost_model.rb +40 -0
- data/lib/flexr/automaton/compiler.rb +229 -96
- data/lib/flexr/automaton/dfa.rb +106 -19
- data/lib/flexr/automaton/minimizer.rb +79 -31
- data/lib/flexr/automaton/nfa.rb +45 -13
- data/lib/flexr/automaton/types.rb +14 -0
- data/lib/flexr/cli.rb +128 -22
- data/lib/flexr/codegen/direct.rb +5 -41
- data/lib/flexr/codegen/firstmatch.rb +1 -7
- data/lib/flexr/codegen/table.rb +181 -69
- data/lib/flexr/codegen.rb +8 -0
- data/lib/flexr/configuration.rb +45 -0
- data/lib/flexr/diagnostics.rb +39 -3
- data/lib/flexr/dsl.rb +116 -47
- data/lib/flexr/errors.rb +7 -6
- data/lib/flexr/generated.rb +120 -46
- data/lib/flexr/generator.rb +102 -109
- data/lib/flexr/grammar.rb +41 -0
- data/lib/flexr/importer.rb +4 -40
- data/lib/flexr/ir.rb +2 -2
- data/lib/flexr/migration.rb +31 -0
- data/lib/flexr/minitest.rb +24 -0
- data/lib/flexr/options.rb +9 -5
- data/lib/flexr/rake_task.rb +6 -1
- data/lib/flexr/regexp/ast.rb +5 -2
- data/lib/flexr/regexp/matcher.rb +26 -0
- data/lib/flexr/regexp/normalizer.rb +63 -21
- data/lib/flexr/regexp/parser.rb +241 -60
- data/lib/flexr/regexp/tokenizer.rb +172 -47
- data/lib/flexr/rspec.rb +51 -0
- data/lib/flexr/runtime/buffer.rb +115 -9
- data/lib/flexr/runtime/core.rb +248 -51
- data/lib/flexr/runtime/errors.rb +66 -4
- data/lib/flexr/runtime/interpreter.rb +213 -97
- data/lib/flexr/runtime/location.rb +35 -12
- data/lib/flexr/runtime/record.rb +14 -0
- data/lib/flexr/runtime/token.rb +7 -1
- data/lib/flexr/runtime.rb +75 -0
- data/lib/flexr/source/passthrough.rb +70 -14
- data/lib/flexr/source/prism_reader.rb +217 -51
- data/lib/flexr/source/static_eval.rb +40 -4
- data/lib/flexr/source.rb +6 -0
- data/lib/flexr/testing.rb +47 -0
- data/lib/flexr/trace.rb +197 -0
- data/lib/flexr/unicode/data/properties.rb +1 -1
- data/lib/flexr/unicode/data.rb +11 -0
- data/lib/flexr/unicode/property.rb +4 -8
- data/lib/flexr/unicode/reference_regexp.rb +11 -9
- data/lib/flexr/unicode/version.rb +7 -0
- data/lib/flexr/version.rb +1 -1
- data/lib/flexr.rb +10 -76
- data/lib/ruby_lsp/flexr/addon.rb +114 -0
- data/sig/flexr.rbs +191 -0
- data/site/README.md +54 -3
- data/site/astro.config.mjs +3 -1
- data/site/package.json +12 -5
- data/site/playwright.config.mjs +16 -0
- data/site/pnpm-lock.yaml +218 -88
- data/site/ruby/playground.rb +57 -0
- data/site/scripts/prepare-playground.mjs +35 -0
- data/site/scripts/verify-site.mjs +5 -3
- data/site/src/content/docs/concepts/matching-semantics.md +4 -2
- data/site/src/content/docs/concepts/security-model.md +9 -1
- data/site/src/content/docs/learn/generation.md +1 -1
- data/site/src/content/docs/reference/dsl.md +2 -2
- data/site/src/content/docs/reference/public-api.md +4 -2
- data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
- data/site/src/layouts/SiteLayout.astro +11 -3
- data/site/src/pages/index.astro +69 -155
- data/site/src/pages/playground.astro +40 -38
- data/site/src/playground/client.js +211 -0
- data/site/src/playground/graph-worker.mjs +11 -0
- data/site/src/playground/presets.mjs +33 -0
- data/site/src/playground/runtime.mjs +34 -0
- data/site/src/playground/share.mjs +30 -0
- data/site/src/playground/worker.mjs +29 -0
- data/site/src/styles/custom.css +12 -287
- data/site/src/styles/landing.css +239 -0
- data/site/tests/browser/playground.spec.mjs +99 -0
- data/site/tests/playground-runtime.test.mjs +85 -0
- data/test/ruby_lsp_addon_spec.rb +135 -0
- data/tools/coverage.rb +6 -1
- data/tools/docs_verify.rb +3 -2
- data/tools/golden.rb +54 -0
- data/tools/verification_failure.rb +119 -0
- metadata +66 -12
- data/benchmark/golden/calculator_lexer.sha256 +0 -1
- data/benchmark/golden/json_lexer.sha256 +0 -1
- data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
- data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
- data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
- data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
- data/benchmark/golden/with_racc_lexer.sha256 +0 -1
- data/site/public/playground.js +0 -189
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# Generated by flexr. DO NOT EDIT.
|
|
3
|
+
# source: examples/json/lexer.flexr.rb
|
|
4
|
+
# spec-digest: <normalized>
|
|
5
|
+
# artifact-schema: 2
|
|
6
|
+
# runtime-abi: 2
|
|
7
|
+
# unicode: 15.1.0
|
|
8
|
+
# backend: direct
|
|
9
|
+
# compiled: true
|
|
10
|
+
# eval: false
|
|
11
|
+
# standalone: false
|
|
12
|
+
|
|
13
|
+
require "flexr"
|
|
14
|
+
|
|
15
|
+
module JsonExample
|
|
16
|
+
class Lexer < Flexr::Lexer
|
|
17
|
+
Flexr::Generated.install_compiled!(self, { artifact: {schema_version: 2, compiler_version: "<compiler-version>", runtime_abi_version: 2, unicode_version: "15.1.0"}, rules: [{ index: 0, patterns: [/[ \t\r\n]+/], pattern_conditions: [[0, 0, false, false]], trailing: nil, action: :skip, states: [:initial], bol_only: false, end_anchor: nil }, { index: 1, patterns: [/\{/], pattern_conditions: [[1, 0, false, false]], trailing: nil, action: [:emit, :LBRACE], states: [:initial], bol_only: false, end_anchor: nil }, { index: 2, patterns: [/\}/], pattern_conditions: [[2, 0, false, false]], trailing: nil, action: [:emit, :RBRACE], states: [:initial], bol_only: false, end_anchor: nil }, { index: 3, patterns: [/\[/], pattern_conditions: [[3, 0, false, false]], trailing: nil, action: [:emit, :LBRACKET], states: [:initial], bol_only: false, end_anchor: nil }, { index: 4, patterns: [/\]/], pattern_conditions: [[4, 0, false, false]], trailing: nil, action: [:emit, :RBRACKET], states: [:initial], bol_only: false, end_anchor: nil }, { index: 5, patterns: [/:/], pattern_conditions: [[5, 0, false, false]], trailing: nil, action: [:emit, :COLON], states: [:initial], bol_only: false, end_anchor: nil }, { index: 6, patterns: [/,/], pattern_conditions: [[6, 0, false, false]], trailing: nil, action: [:emit, :COMMA], states: [:initial], bol_only: false, end_anchor: nil }, { index: 7, patterns: [/true/], pattern_conditions: [[7, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 8, patterns: [/false/], pattern_conditions: [[8, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 9, patterns: [/null/], pattern_conditions: [[9, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 10, patterns: [/-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?/], pattern_conditions: [[10, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 11, patterns: [/"(?:\\.|[^"\\])*"/], pattern_conditions: [[11, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }], backend: :direct, token_kind: :array, encoding: Encoding::UTF_8, declared_tokens: [:LBRACE, :RBRACE, :LBRACKET, :RBRACKET, :COLON, :COMMA, :STRING, :NUMBER, :TRUE, :FALSE, :NULL], options: {}, eof_rules: { }, states: [], inclusive_states: {initial: true}, compiled: { machines: { :initial => { state_name: :initial, dfa: {accepts: [[], [[0, 0, false, false]], [], [[6, 0, false, false]], [], [[10, 0, false, false]], [[10, 0, false, false]], [[5, 0, false, false]], [[3, 0, false, false]], [[4, 0, false, false]], [], [], [], [[1, 0, false, false]], [[2, 0, false, false]], [[11, 0, false, false]], [], [], [], [], [], [], [], [], [], [[10, 0, false, false]], [], [[10, 0, false, false]], [], [], [], [], [[9, 0, false, false]], [[7, 0, false, false]], [[8, 0, false, false]]], ec: [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 3, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 7, 8, 9, 9, 9, 9, 9, 9, 9, 9, 10, 11, 12, 13, 14, 15, 16, 16, 16, 16, 16, 16, 16, 16, 16, 17, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 19, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 21, 22, 23, 24, 24, 24, 25, 26, 26, 26, 27, 28, 29, 29, 29, 29, 29, 30, 31, 32, 33, 33, 33, 34, 35, 36, 37, 38, 38, 38, 38, 38, 39, 40, 41, 42, 42, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 44, 44, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 47, 47, 47, 47, 47, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48], class_count: 49, state_count: 35, start: 0, rule_ids: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], direct: {nxt: [-1, 1, 1, -1, 1, -1, 1, -1, 2, -1, -1, 3, 4, -1, -1, 5, 6, 7, -1, -1, -1, 8, -1, 9, -1, -1, -1, -1, 10, -1, -1, -1, 11, -1, -1, -1, 12, -1, -1, 13, -1, 14, -1, -1, -1, -1, -1, -1, -1, -1, 1, 1, -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, 2, 2, 2, 2, 2, 2, 2, 15, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 16, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -1, -1, 17, 18, 19, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 5, 6, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 20, -1, -1, -1, -1, -1, 21, -1, -1, -1, -1, -1, -1, -1, 21, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 20, -1, 6, 6, -1, -1, 21, -1, -1, -1, -1, -1, -1, -1, 21, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 22, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 24, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, 2, -1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -1, -1, 17, 18, 19, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 17, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 18, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 25, 25, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 26, -1, 26, -1, -1, 27, 27, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 28, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 29, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 30, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 25, 25, -1, -1, 21, -1, -1, -1, -1, -1, -1, -1, 21, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 27, 27, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 27, 27, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 31, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 32, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 33, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 34, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], classes: 49}} } }, states: [:initial], stats: {initial: {states: 35, classes: 49, accepting_states: 15}}, diagnostics: [] } })
|
|
18
|
+
def scan_one
|
|
19
|
+
if @__flexr_generated_scan_state != @state
|
|
20
|
+
machine = self.class.__flexr_compiled.machines.fetch(@state)
|
|
21
|
+
@__flexr_generated_scan_state = @state
|
|
22
|
+
@__flexr_generated_scan_dfa = machine.dfa
|
|
23
|
+
@__flexr_generated_scan_fast = __flexr_generated_fast_path?(@__flexr_generated_scan_dfa)
|
|
24
|
+
@__flexr_generated_scan_simple = __flexr_generated_simple_fast_path?(@__flexr_generated_scan_dfa)
|
|
25
|
+
end
|
|
26
|
+
dfa = @__flexr_generated_scan_dfa
|
|
27
|
+
return Flexr::Runtime::Interpreter.new(self).scan unless @__flexr_generated_scan_fast
|
|
28
|
+
position = byte_pos
|
|
29
|
+
return nil unless valid_utf8_at?(position)
|
|
30
|
+
return __flexr_generated_scan_simple(dfa, position) if @__flexr_generated_scan_simple
|
|
31
|
+
cursor = position
|
|
32
|
+
current = dfa.start
|
|
33
|
+
direct = dfa.direct
|
|
34
|
+
direct_nxt = direct&.fetch(:nxt)
|
|
35
|
+
direct_classes = direct&.fetch(:classes)
|
|
36
|
+
ec = dfa.ec
|
|
37
|
+
source = buffer.stable_source
|
|
38
|
+
guarded_steps = scan_steps_guarded?
|
|
39
|
+
token_start = more_text_start || position
|
|
40
|
+
token_limit = max_token_size
|
|
41
|
+
check_token_limit = token_limit_required?
|
|
42
|
+
best = nil
|
|
43
|
+
accel_mode = @accel_mode
|
|
44
|
+
accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)
|
|
45
|
+
while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)
|
|
46
|
+
if accelerate
|
|
47
|
+
region = __flexr_generated_acceleration_region(dfa, current)
|
|
48
|
+
acceptance = dfa.accepts[current].first
|
|
49
|
+
token_rule = acceptance&.rule_index
|
|
50
|
+
accelerated_end = __flexr_generated_accelerate(region, cursor, token_rule, position) if region
|
|
51
|
+
if accelerated_end && accelerated_end > cursor
|
|
52
|
+
cursor = accelerated_end
|
|
53
|
+
best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
|
|
54
|
+
next
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)
|
|
58
|
+
consume_step! if guarded_steps
|
|
59
|
+
current = if direct_nxt
|
|
60
|
+
class_id = ec[byte]
|
|
61
|
+
value = direct_nxt[(current * direct_classes) + class_id]
|
|
62
|
+
value >= 0 ? value : nil
|
|
63
|
+
else
|
|
64
|
+
dfa.transition(current, byte)
|
|
65
|
+
end
|
|
66
|
+
break unless current
|
|
67
|
+
cursor += 1
|
|
68
|
+
resource_acceptance = dfa.accepts[current].first
|
|
69
|
+
resource_rule = resource_acceptance&.rule_index
|
|
70
|
+
if check_token_limit
|
|
71
|
+
size = cursor - token_start
|
|
72
|
+
defer_token_size_check!(size, rule: resource_rule) if size > token_limit
|
|
73
|
+
end
|
|
74
|
+
best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
|
|
75
|
+
end
|
|
76
|
+
best
|
|
77
|
+
end
|
|
78
|
+
def __flexr_generated_scan_simple(dfa, position)
|
|
79
|
+
current = dfa.start
|
|
80
|
+
direct = dfa.direct
|
|
81
|
+
direct_nxt = direct&.fetch(:nxt)
|
|
82
|
+
direct_classes = direct&.fetch(:classes)
|
|
83
|
+
ec = dfa.ec
|
|
84
|
+
accepts = dfa.accepts
|
|
85
|
+
rules = self.class.__flexr_rules
|
|
86
|
+
source = buffer.stable_source
|
|
87
|
+
guarded_steps = scan_steps_guarded?
|
|
88
|
+
token_start = more_text_start || position
|
|
89
|
+
token_limit = max_token_size
|
|
90
|
+
check_token_limit = token_limit_required?
|
|
91
|
+
cursor = position
|
|
92
|
+
best = nil
|
|
93
|
+
accel_mode = @accel_mode
|
|
94
|
+
accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)
|
|
95
|
+
if source && !guarded_steps && !check_token_limit && !accelerate
|
|
96
|
+
while cursor < source.bytesize
|
|
97
|
+
byte = source.getbyte(cursor)
|
|
98
|
+
current = if direct_nxt
|
|
99
|
+
class_id = ec[byte]
|
|
100
|
+
value = direct_nxt[(current * direct_classes) + class_id]
|
|
101
|
+
value >= 0 ? value : nil
|
|
102
|
+
else
|
|
103
|
+
dfa.transition(current, byte)
|
|
104
|
+
end
|
|
105
|
+
break unless current
|
|
106
|
+
cursor += 1
|
|
107
|
+
acceptance = accepts[current].first
|
|
108
|
+
next unless acceptance
|
|
109
|
+
rule = rules.fetch(acceptance.rule_index)
|
|
110
|
+
best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
|
|
111
|
+
best.rule = rule
|
|
112
|
+
best.start_pos = position
|
|
113
|
+
best.end_pos = cursor
|
|
114
|
+
best.total_end_pos = cursor
|
|
115
|
+
end
|
|
116
|
+
return best
|
|
117
|
+
end
|
|
118
|
+
while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)
|
|
119
|
+
if accelerate
|
|
120
|
+
region = __flexr_generated_acceleration_region(dfa, current)
|
|
121
|
+
acceptance = accepts[current].first
|
|
122
|
+
accelerated_end = __flexr_generated_accelerate(region, cursor, acceptance&.rule_index, position) if region
|
|
123
|
+
if accelerated_end && accelerated_end > cursor
|
|
124
|
+
cursor = accelerated_end
|
|
125
|
+
best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
|
|
126
|
+
next
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)
|
|
130
|
+
consume_step! if guarded_steps
|
|
131
|
+
current = if direct_nxt
|
|
132
|
+
class_id = ec[byte]
|
|
133
|
+
value = direct_nxt[(current * direct_classes) + class_id]
|
|
134
|
+
value >= 0 ? value : nil
|
|
135
|
+
else
|
|
136
|
+
dfa.transition(current, byte)
|
|
137
|
+
end
|
|
138
|
+
break unless current
|
|
139
|
+
cursor += 1
|
|
140
|
+
acceptance = accepts[current].first
|
|
141
|
+
if check_token_limit
|
|
142
|
+
size = cursor - token_start
|
|
143
|
+
defer_token_size_check!(size, rule: acceptance&.rule_index) if size > token_limit
|
|
144
|
+
end
|
|
145
|
+
next unless acceptance
|
|
146
|
+
rule = rules.fetch(acceptance.rule_index)
|
|
147
|
+
best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
|
|
148
|
+
best.rule = rule
|
|
149
|
+
best.start_pos = position
|
|
150
|
+
best.end_pos = cursor
|
|
151
|
+
best.total_end_pos = cursor
|
|
152
|
+
end
|
|
153
|
+
best
|
|
154
|
+
end
|
|
155
|
+
def __flexr_generated_acceleration_region(dfa, state)
|
|
156
|
+
@__flexr_generated_accel_regions ||= {}
|
|
157
|
+
regions = (@__flexr_generated_accel_regions[dfa] ||= Flexr::Automaton::Accel.extract(dfa).to_h { |region| [region.state, region] })
|
|
158
|
+
region = regions[state]
|
|
159
|
+
return unless region
|
|
160
|
+
return if @__flexr_generated_disabled_auto_accel&.key?(region)
|
|
161
|
+
return if utf8_input? && region.bytes.any? { |byte| byte >= 128 }
|
|
162
|
+
return if dfa.accepts[state].any? do |acceptance|
|
|
163
|
+
rule = self.class.__flexr_rules.fetch(acceptance.rule_index)
|
|
164
|
+
acceptance.bol_only || acceptance.end_anchor || rule.trailing
|
|
165
|
+
end
|
|
166
|
+
region
|
|
167
|
+
end
|
|
168
|
+
def __flexr_generated_accelerate(region, position, token_rule, token_start)
|
|
169
|
+
mode = @accel_mode
|
|
170
|
+
cursor = position
|
|
171
|
+
matched = false
|
|
172
|
+
loop do
|
|
173
|
+
if cursor >= buffer.bytesize
|
|
174
|
+
return matched ? cursor : nil if buffer.eof_loaded?
|
|
175
|
+
return matched ? cursor : nil unless buffer.ensure_available?(cursor + 1)
|
|
176
|
+
end
|
|
177
|
+
segment = buffer.source
|
|
178
|
+
segment_position = cursor - buffer.base_offset
|
|
179
|
+
if segment.encoding == ::Encoding::UTF_8 && region.utf8_regexp
|
|
180
|
+
regexp = region.utf8_regexp
|
|
181
|
+
else
|
|
182
|
+
segment = segment.b unless segment.encoding == ::Encoding::BINARY
|
|
183
|
+
regexp = region.regexp
|
|
184
|
+
end
|
|
185
|
+
length = begin
|
|
186
|
+
if %i[strscan auto].include?(mode) && defined?(::StringScanner)
|
|
187
|
+
@__flexr_generated_accel_scanner ||= ::StringScanner.new("".b)
|
|
188
|
+
@__flexr_generated_accel_scanner.string = segment
|
|
189
|
+
@__flexr_generated_accel_scanner.pos = segment_position
|
|
190
|
+
@__flexr_generated_accel_scanner.skip(regexp)
|
|
191
|
+
else
|
|
192
|
+
match = regexp.match(segment.byteslice(segment_position..).to_s, 0)
|
|
193
|
+
match && match.begin(0).zero? ? match[0].bytesize : nil
|
|
194
|
+
end
|
|
195
|
+
rescue ArgumentError
|
|
196
|
+
nil
|
|
197
|
+
end
|
|
198
|
+
__flexr_generated_record_auto_accel_miss(region) if mode == :auto && length.to_i < 8
|
|
199
|
+
return matched ? cursor : nil unless length&.positive?
|
|
200
|
+
matched = true
|
|
201
|
+
cursor += length
|
|
202
|
+
consume_step!(length)
|
|
203
|
+
defer_token_size_check!(cursor - (more_text_start || token_start), rule: token_rule)
|
|
204
|
+
return cursor if cursor < buffer.bytesize || buffer.eof_loaded?
|
|
205
|
+
end
|
|
206
|
+
end
|
|
207
|
+
def __flexr_generated_record_auto_accel_miss(region)
|
|
208
|
+
@__flexr_generated_auto_accel_misses ||= Hash.new(0)
|
|
209
|
+
@__flexr_generated_auto_accel_misses[region] += 1
|
|
210
|
+
@__flexr_generated_auto_accel_miss_total = @__flexr_generated_auto_accel_miss_total.to_i + 1
|
|
211
|
+
@__flexr_generated_auto_accel_off = true if @__flexr_generated_auto_accel_miss_total >= 9
|
|
212
|
+
return if @__flexr_generated_auto_accel_misses[region] < 3
|
|
213
|
+
@__flexr_generated_disabled_auto_accel ||= {}
|
|
214
|
+
@__flexr_generated_disabled_auto_accel[region] = true
|
|
215
|
+
end
|
|
216
|
+
def __flexr_generated_fast_path?(dfa)
|
|
217
|
+
@__flexr_generated_fast_paths ||= {}
|
|
218
|
+
return @__flexr_generated_fast_paths[dfa] if @__flexr_generated_fast_paths.key?(dfa)
|
|
219
|
+
rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }
|
|
220
|
+
@__flexr_generated_fast_paths[dfa] = self.class.__flexr_config.backend != :firstmatch &&
|
|
221
|
+
self.class.__flexr_config.options[:allow_empty_match] != true && rules.none?(&:trailing)
|
|
222
|
+
end
|
|
223
|
+
def __flexr_generated_simple_fast_path?(dfa)
|
|
224
|
+
@__flexr_generated_simple_fast_paths ||= {}
|
|
225
|
+
return @__flexr_generated_simple_fast_paths[dfa] if @__flexr_generated_simple_fast_paths.key?(dfa)
|
|
226
|
+
rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }
|
|
227
|
+
@__flexr_generated_simple_fast_paths[dfa] = rules.all? do |rule|
|
|
228
|
+
rule.pattern_conditions.all? { |condition| condition && !condition.bol_only && !condition.end_anchor }
|
|
229
|
+
end
|
|
230
|
+
end
|
|
231
|
+
def __flexr_generated_acceptance(dfa, state, start_position, cursor, best)
|
|
232
|
+
dfa.accepts[state].each do |acceptance|
|
|
233
|
+
next if acceptance.bol_only && !beginning_of_line?
|
|
234
|
+
next if acceptance.end_anchor && !(buffer.eof?(cursor) || buffer.getbyte(cursor) == 0x0a)
|
|
235
|
+
rule = self.class.__flexr_rules.fetch(acceptance.rule_index)
|
|
236
|
+
next if best && cursor < best.total_end_pos
|
|
237
|
+
next if best && cursor == best.total_end_pos &&
|
|
238
|
+
rule.index > best.rule.index
|
|
239
|
+
best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
|
|
240
|
+
best.rule = rule
|
|
241
|
+
best.start_pos = start_position
|
|
242
|
+
best.end_pos = cursor
|
|
243
|
+
best.total_end_pos = cursor
|
|
244
|
+
end
|
|
245
|
+
best
|
|
246
|
+
end
|
|
247
|
+
def __flexr_generated_execute(rule)
|
|
248
|
+
case rule.index
|
|
249
|
+
when 0
|
|
250
|
+
when 1
|
|
251
|
+
emit(:LBRACE, text)
|
|
252
|
+
when 2
|
|
253
|
+
emit(:RBRACE, text)
|
|
254
|
+
when 3
|
|
255
|
+
emit(:LBRACKET, text)
|
|
256
|
+
when 4
|
|
257
|
+
emit(:RBRACKET, text)
|
|
258
|
+
when 5
|
|
259
|
+
emit(:COLON, text)
|
|
260
|
+
when 6
|
|
261
|
+
emit(:COMMA, text)
|
|
262
|
+
when 7
|
|
263
|
+
emit :TRUE, true
|
|
264
|
+
when 8
|
|
265
|
+
emit :FALSE, false
|
|
266
|
+
when 9
|
|
267
|
+
emit :NULL, nil
|
|
268
|
+
when 10
|
|
269
|
+
emit :NUMBER, text.to_f
|
|
270
|
+
when 11
|
|
271
|
+
emit :STRING, text.byteslice(1...-1)
|
|
272
|
+
else
|
|
273
|
+
instance_exec(&rule.action)
|
|
274
|
+
end
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
end
|
|
292
|
+
end
|
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# Generated by flexr. DO NOT EDIT.
|
|
3
|
+
# source: lib/flexr/regexp/tokenizer.flexr.rb
|
|
4
|
+
# spec-digest: <normalized>
|
|
5
|
+
# artifact-schema: 2
|
|
6
|
+
# runtime-abi: 2
|
|
7
|
+
# unicode: 15.1.0
|
|
8
|
+
# backend: table
|
|
9
|
+
# compiled: true
|
|
10
|
+
# eval: false
|
|
11
|
+
# standalone: false
|
|
12
|
+
|
|
13
|
+
require "flexr"
|
|
14
|
+
|
|
15
|
+
module Flexr
|
|
16
|
+
module Regexp
|
|
17
|
+
# This specification is intentionally small: it is a dogfood target for
|
|
18
|
+
# the source lexer path, not the parser's implementation.
|
|
19
|
+
class SourceLexer < Flexr::Lexer
|
|
20
|
+
Flexr::Generated.install_compiled!(self, { artifact: {schema_version: 2, compiler_version: "<compiler-version>", runtime_abi_version: 2, unicode_version: "15.1.0"}, rules: [{ index: 0, patterns: [/[ \t\r\n]+/], pattern_conditions: [[0, 0, false, false]], trailing: nil, action: :skip, states: [:initial], bol_only: false, end_anchor: nil }, { index: 1, patterns: [/\\p\{[A-Za-z_][A-Za-z0-9_]*\}/], pattern_conditions: [[1, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 2, patterns: [/\\./], pattern_conditions: [[2, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 3, patterns: [/\[[^\]\n]*\]/], pattern_conditions: [[3, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 4, patterns: [/[?*+]|\{[0-9]+(?:,[0-9]*)?\}/], pattern_conditions: [[4, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 5, patterns: [/\|/], pattern_conditions: [[5, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 6, patterns: [/\(/], pattern_conditions: [[6, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 7, patterns: [/\)/], pattern_conditions: [[7, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 8, patterns: [/\^|\$/], pattern_conditions: [[8, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 9, patterns: [/\./], pattern_conditions: [[9, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }, { index: 10, patterns: [/[^\\\[\]().|?*+{}^$ \t\r\n]/], pattern_conditions: [[10, 0, false, false]], trailing: nil, action: :__flexr_generated_method, states: [:initial], bol_only: false, end_anchor: nil }], backend: :table, token_kind: :array, encoding: Encoding::BINARY, declared_tokens: [:PROPERTY, :ESCAPE, :CHAR_CLASS, :QUANTIFIER, :ALTERNATION, :GROUP_OPEN, :GROUP_CLOSE, :ANCHOR, :DOT, :LITERAL], options: {}, eof_rules: { }, states: [], inclusive_states: {initial: true}, compiled: { machines: { :initial => { state_name: :initial, dfa: {accepts: [[], [[10, 0, false, false]], [[0, 0, false, false]], [[8, 0, false, false]], [[6, 0, false, false]], [[7, 0, false, false]], [[4, 0, false, false]], [[9, 0, false, false]], [], [], [], [[5, 0, false, false]], [[3, 0, false, false]], [[2, 0, false, false]], [[2, 0, false, false]], [], [], [], [], [[1, 0, false, false]]], ec: [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 3, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 7, 7, 7, 8, 9, 9, 9, 10, 11, 12, 12, 13, 14, 15, 16, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, 19, 20, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 22, 23, 24, 25, 26, 27, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 29, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 31, 32, 33, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34], class_count: 35, state_count: 20, start: 0, rule_ids: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], packed: {base: [0, 0, 12, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 3, 22, 15, 4, 21, 0], default: [1, nil, nil, nil, nil, nil, nil, nil, 8, 13, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil], next: [nil, 2, 2, nil, 2, nil, 2, nil, 3, nil, 4, 5, 6, 2, 2, 7, 2, 15, 2, 6, nil, 17, 8, 9, nil, 3, nil, 12, nil, nil, 14, 10, 11, nil, 16, 17, 18, 6, 18, 15, nil, 18, 18, 18, 18, 18, nil, 18, nil, 18, 18, 18, nil, nil, 19, 6], check: [nil, 0, 0, 9, 0, 8, 0, nil, 0, nil, 0, 0, 0, 2, 2, 0, 2, 10, 2, 0, nil, 17, 0, 0, 0, 0, nil, 8, nil, nil, 9, 0, 0, 0, 14, 15, 16, 17, 18, 15, nil, 16, 18, 16, 16, 16, nil, 18, nil, 18, 18, 18, nil, nil, 18, 15]}} } }, states: [:initial], stats: {initial: {states: 20, classes: 35, accepting_states: 12}}, diagnostics: [] } })
|
|
21
|
+
def scan_one
|
|
22
|
+
if @__flexr_generated_scan_state != @state
|
|
23
|
+
machine = self.class.__flexr_compiled.machines.fetch(@state)
|
|
24
|
+
@__flexr_generated_scan_state = @state
|
|
25
|
+
@__flexr_generated_scan_dfa = machine.dfa
|
|
26
|
+
@__flexr_generated_scan_fast = __flexr_generated_fast_path?(@__flexr_generated_scan_dfa)
|
|
27
|
+
@__flexr_generated_scan_simple = __flexr_generated_simple_fast_path?(@__flexr_generated_scan_dfa)
|
|
28
|
+
end
|
|
29
|
+
dfa = @__flexr_generated_scan_dfa
|
|
30
|
+
return Flexr::Runtime::Interpreter.new(self).scan unless @__flexr_generated_scan_fast
|
|
31
|
+
position = byte_pos
|
|
32
|
+
return nil unless valid_utf8_at?(position)
|
|
33
|
+
return __flexr_generated_scan_simple(dfa, position) if @__flexr_generated_scan_simple
|
|
34
|
+
cursor = position
|
|
35
|
+
current = dfa.start
|
|
36
|
+
direct = dfa.direct
|
|
37
|
+
direct_nxt = direct&.fetch(:nxt)
|
|
38
|
+
direct_classes = direct&.fetch(:classes)
|
|
39
|
+
ec = dfa.ec
|
|
40
|
+
source = buffer.stable_source
|
|
41
|
+
guarded_steps = scan_steps_guarded?
|
|
42
|
+
token_start = more_text_start || position
|
|
43
|
+
token_limit = max_token_size
|
|
44
|
+
check_token_limit = token_limit_required?
|
|
45
|
+
best = nil
|
|
46
|
+
accel_mode = @accel_mode
|
|
47
|
+
accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)
|
|
48
|
+
while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)
|
|
49
|
+
if accelerate
|
|
50
|
+
region = __flexr_generated_acceleration_region(dfa, current)
|
|
51
|
+
acceptance = dfa.accepts[current].first
|
|
52
|
+
token_rule = acceptance&.rule_index
|
|
53
|
+
accelerated_end = __flexr_generated_accelerate(region, cursor, token_rule, position) if region
|
|
54
|
+
if accelerated_end && accelerated_end > cursor
|
|
55
|
+
cursor = accelerated_end
|
|
56
|
+
best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
|
|
57
|
+
next
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)
|
|
61
|
+
consume_step! if guarded_steps
|
|
62
|
+
current = if direct_nxt
|
|
63
|
+
class_id = ec[byte]
|
|
64
|
+
value = direct_nxt[(current * direct_classes) + class_id]
|
|
65
|
+
value >= 0 ? value : nil
|
|
66
|
+
else
|
|
67
|
+
dfa.transition(current, byte)
|
|
68
|
+
end
|
|
69
|
+
break unless current
|
|
70
|
+
cursor += 1
|
|
71
|
+
resource_acceptance = dfa.accepts[current].first
|
|
72
|
+
resource_rule = resource_acceptance&.rule_index
|
|
73
|
+
if check_token_limit
|
|
74
|
+
size = cursor - token_start
|
|
75
|
+
defer_token_size_check!(size, rule: resource_rule) if size > token_limit
|
|
76
|
+
end
|
|
77
|
+
best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
|
|
78
|
+
end
|
|
79
|
+
best
|
|
80
|
+
end
|
|
81
|
+
def __flexr_generated_scan_simple(dfa, position)
|
|
82
|
+
current = dfa.start
|
|
83
|
+
direct = dfa.direct
|
|
84
|
+
direct_nxt = direct&.fetch(:nxt)
|
|
85
|
+
direct_classes = direct&.fetch(:classes)
|
|
86
|
+
ec = dfa.ec
|
|
87
|
+
accepts = dfa.accepts
|
|
88
|
+
rules = self.class.__flexr_rules
|
|
89
|
+
source = buffer.stable_source
|
|
90
|
+
guarded_steps = scan_steps_guarded?
|
|
91
|
+
token_start = more_text_start || position
|
|
92
|
+
token_limit = max_token_size
|
|
93
|
+
check_token_limit = token_limit_required?
|
|
94
|
+
cursor = position
|
|
95
|
+
best = nil
|
|
96
|
+
accel_mode = @accel_mode
|
|
97
|
+
accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)
|
|
98
|
+
if source && !guarded_steps && !check_token_limit && !accelerate
|
|
99
|
+
while cursor < source.bytesize
|
|
100
|
+
byte = source.getbyte(cursor)
|
|
101
|
+
current = if direct_nxt
|
|
102
|
+
class_id = ec[byte]
|
|
103
|
+
value = direct_nxt[(current * direct_classes) + class_id]
|
|
104
|
+
value >= 0 ? value : nil
|
|
105
|
+
else
|
|
106
|
+
dfa.transition(current, byte)
|
|
107
|
+
end
|
|
108
|
+
break unless current
|
|
109
|
+
cursor += 1
|
|
110
|
+
acceptance = accepts[current].first
|
|
111
|
+
next unless acceptance
|
|
112
|
+
rule = rules.fetch(acceptance.rule_index)
|
|
113
|
+
best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
|
|
114
|
+
best.rule = rule
|
|
115
|
+
best.start_pos = position
|
|
116
|
+
best.end_pos = cursor
|
|
117
|
+
best.total_end_pos = cursor
|
|
118
|
+
end
|
|
119
|
+
return best
|
|
120
|
+
end
|
|
121
|
+
while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)
|
|
122
|
+
if accelerate
|
|
123
|
+
region = __flexr_generated_acceleration_region(dfa, current)
|
|
124
|
+
acceptance = accepts[current].first
|
|
125
|
+
accelerated_end = __flexr_generated_accelerate(region, cursor, acceptance&.rule_index, position) if region
|
|
126
|
+
if accelerated_end && accelerated_end > cursor
|
|
127
|
+
cursor = accelerated_end
|
|
128
|
+
best = __flexr_generated_acceptance(dfa, current, position, cursor, best)
|
|
129
|
+
next
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)
|
|
133
|
+
consume_step! if guarded_steps
|
|
134
|
+
current = if direct_nxt
|
|
135
|
+
class_id = ec[byte]
|
|
136
|
+
value = direct_nxt[(current * direct_classes) + class_id]
|
|
137
|
+
value >= 0 ? value : nil
|
|
138
|
+
else
|
|
139
|
+
dfa.transition(current, byte)
|
|
140
|
+
end
|
|
141
|
+
break unless current
|
|
142
|
+
cursor += 1
|
|
143
|
+
acceptance = accepts[current].first
|
|
144
|
+
if check_token_limit
|
|
145
|
+
size = cursor - token_start
|
|
146
|
+
defer_token_size_check!(size, rule: acceptance&.rule_index) if size > token_limit
|
|
147
|
+
end
|
|
148
|
+
next unless acceptance
|
|
149
|
+
rule = rules.fetch(acceptance.rule_index)
|
|
150
|
+
best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
|
|
151
|
+
best.rule = rule
|
|
152
|
+
best.start_pos = position
|
|
153
|
+
best.end_pos = cursor
|
|
154
|
+
best.total_end_pos = cursor
|
|
155
|
+
end
|
|
156
|
+
best
|
|
157
|
+
end
|
|
158
|
+
def __flexr_generated_acceleration_region(dfa, state)
|
|
159
|
+
@__flexr_generated_accel_regions ||= {}
|
|
160
|
+
regions = (@__flexr_generated_accel_regions[dfa] ||= Flexr::Automaton::Accel.extract(dfa).to_h { |region| [region.state, region] })
|
|
161
|
+
region = regions[state]
|
|
162
|
+
return unless region
|
|
163
|
+
return if @__flexr_generated_disabled_auto_accel&.key?(region)
|
|
164
|
+
return if utf8_input? && region.bytes.any? { |byte| byte >= 128 }
|
|
165
|
+
return if dfa.accepts[state].any? do |acceptance|
|
|
166
|
+
rule = self.class.__flexr_rules.fetch(acceptance.rule_index)
|
|
167
|
+
acceptance.bol_only || acceptance.end_anchor || rule.trailing
|
|
168
|
+
end
|
|
169
|
+
region
|
|
170
|
+
end
|
|
171
|
+
def __flexr_generated_accelerate(region, position, token_rule, token_start)
|
|
172
|
+
mode = @accel_mode
|
|
173
|
+
cursor = position
|
|
174
|
+
matched = false
|
|
175
|
+
loop do
|
|
176
|
+
if cursor >= buffer.bytesize
|
|
177
|
+
return matched ? cursor : nil if buffer.eof_loaded?
|
|
178
|
+
return matched ? cursor : nil unless buffer.ensure_available?(cursor + 1)
|
|
179
|
+
end
|
|
180
|
+
segment = buffer.source
|
|
181
|
+
segment_position = cursor - buffer.base_offset
|
|
182
|
+
if segment.encoding == ::Encoding::UTF_8 && region.utf8_regexp
|
|
183
|
+
regexp = region.utf8_regexp
|
|
184
|
+
else
|
|
185
|
+
segment = segment.b unless segment.encoding == ::Encoding::BINARY
|
|
186
|
+
regexp = region.regexp
|
|
187
|
+
end
|
|
188
|
+
length = begin
|
|
189
|
+
if %i[strscan auto].include?(mode) && defined?(::StringScanner)
|
|
190
|
+
@__flexr_generated_accel_scanner ||= ::StringScanner.new("".b)
|
|
191
|
+
@__flexr_generated_accel_scanner.string = segment
|
|
192
|
+
@__flexr_generated_accel_scanner.pos = segment_position
|
|
193
|
+
@__flexr_generated_accel_scanner.skip(regexp)
|
|
194
|
+
else
|
|
195
|
+
match = regexp.match(segment.byteslice(segment_position..).to_s, 0)
|
|
196
|
+
match && match.begin(0).zero? ? match[0].bytesize : nil
|
|
197
|
+
end
|
|
198
|
+
rescue ArgumentError
|
|
199
|
+
nil
|
|
200
|
+
end
|
|
201
|
+
__flexr_generated_record_auto_accel_miss(region) if mode == :auto && length.to_i < 8
|
|
202
|
+
return matched ? cursor : nil unless length&.positive?
|
|
203
|
+
matched = true
|
|
204
|
+
cursor += length
|
|
205
|
+
consume_step!(length)
|
|
206
|
+
defer_token_size_check!(cursor - (more_text_start || token_start), rule: token_rule)
|
|
207
|
+
return cursor if cursor < buffer.bytesize || buffer.eof_loaded?
|
|
208
|
+
end
|
|
209
|
+
end
|
|
210
|
+
def __flexr_generated_record_auto_accel_miss(region)
|
|
211
|
+
@__flexr_generated_auto_accel_misses ||= Hash.new(0)
|
|
212
|
+
@__flexr_generated_auto_accel_misses[region] += 1
|
|
213
|
+
@__flexr_generated_auto_accel_miss_total = @__flexr_generated_auto_accel_miss_total.to_i + 1
|
|
214
|
+
@__flexr_generated_auto_accel_off = true if @__flexr_generated_auto_accel_miss_total >= 9
|
|
215
|
+
return if @__flexr_generated_auto_accel_misses[region] < 3
|
|
216
|
+
@__flexr_generated_disabled_auto_accel ||= {}
|
|
217
|
+
@__flexr_generated_disabled_auto_accel[region] = true
|
|
218
|
+
end
|
|
219
|
+
def __flexr_generated_fast_path?(dfa)
|
|
220
|
+
@__flexr_generated_fast_paths ||= {}
|
|
221
|
+
return @__flexr_generated_fast_paths[dfa] if @__flexr_generated_fast_paths.key?(dfa)
|
|
222
|
+
rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }
|
|
223
|
+
@__flexr_generated_fast_paths[dfa] = self.class.__flexr_config.backend != :firstmatch &&
|
|
224
|
+
self.class.__flexr_config.options[:allow_empty_match] != true && rules.none?(&:trailing)
|
|
225
|
+
end
|
|
226
|
+
def __flexr_generated_simple_fast_path?(dfa)
|
|
227
|
+
@__flexr_generated_simple_fast_paths ||= {}
|
|
228
|
+
return @__flexr_generated_simple_fast_paths[dfa] if @__flexr_generated_simple_fast_paths.key?(dfa)
|
|
229
|
+
rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }
|
|
230
|
+
@__flexr_generated_simple_fast_paths[dfa] = rules.all? do |rule|
|
|
231
|
+
rule.pattern_conditions.all? { |condition| condition && !condition.bol_only && !condition.end_anchor }
|
|
232
|
+
end
|
|
233
|
+
end
|
|
234
|
+
def __flexr_generated_acceptance(dfa, state, start_position, cursor, best)
|
|
235
|
+
dfa.accepts[state].each do |acceptance|
|
|
236
|
+
next if acceptance.bol_only && !beginning_of_line?
|
|
237
|
+
next if acceptance.end_anchor && !(buffer.eof?(cursor) || buffer.getbyte(cursor) == 0x0a)
|
|
238
|
+
rule = self.class.__flexr_rules.fetch(acceptance.rule_index)
|
|
239
|
+
next if best && cursor < best.total_end_pos
|
|
240
|
+
next if best && cursor == best.total_end_pos &&
|
|
241
|
+
rule.index > best.rule.index
|
|
242
|
+
best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)
|
|
243
|
+
best.rule = rule
|
|
244
|
+
best.start_pos = start_position
|
|
245
|
+
best.end_pos = cursor
|
|
246
|
+
best.total_end_pos = cursor
|
|
247
|
+
end
|
|
248
|
+
best
|
|
249
|
+
end
|
|
250
|
+
def __flexr_generated_execute(rule)
|
|
251
|
+
case rule.index
|
|
252
|
+
when 0
|
|
253
|
+
when 1
|
|
254
|
+
emit :PROPERTY, text.byteslice(3...-1)
|
|
255
|
+
when 2
|
|
256
|
+
emit :ESCAPE, text
|
|
257
|
+
when 3
|
|
258
|
+
emit :CHAR_CLASS, text
|
|
259
|
+
when 4
|
|
260
|
+
emit :QUANTIFIER, text
|
|
261
|
+
when 5
|
|
262
|
+
emit :ALTERNATION, text
|
|
263
|
+
when 6
|
|
264
|
+
emit :GROUP_OPEN, text
|
|
265
|
+
when 7
|
|
266
|
+
emit :GROUP_CLOSE, text
|
|
267
|
+
when 8
|
|
268
|
+
emit :ANCHOR, text
|
|
269
|
+
when 9
|
|
270
|
+
emit :DOT, text
|
|
271
|
+
when 10
|
|
272
|
+
emit :LITERAL, text
|
|
273
|
+
else
|
|
274
|
+
instance_exec(&rule.action)
|
|
275
|
+
end
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
end
|
|
292
|
+
end
|
|
293
|
+
end
|