flexr 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/CHANGELOG.md +116 -0
- data/CONTRIBUTING.md +46 -3
- data/README.md +5 -4
- data/Rakefile +241 -72
- data/Steepfile +7 -0
- data/benchmark/baselines/json.json +2 -2
- data/benchmark/golden/calculator_lexer.rb +276 -0
- data/benchmark/golden/graphql_lexer.rb +310 -0
- data/benchmark/golden/json_lexer.rb +292 -0
- data/benchmark/golden/regexp_tokenizer.rb +293 -0
- data/benchmark/golden/ruby_subset_lexer.rb +280 -0
- data/benchmark/golden/sql_lexer.rb +293 -0
- data/benchmark/golden/toml_lexer.rb +292 -0
- data/benchmark/golden/toy_lang_lexer.rb +273 -0
- data/benchmark/golden/with_lrama_lexer.rb +264 -0
- data/benchmark/golden/with_racc_lexer.rb +264 -0
- data/benchmark/lrama_lexer.rb +165 -0
- data/benchmark/matrix.rb +79 -0
- data/benchmark/profile.rb +109 -0
- data/benchmark/run.rb +4 -1
- data/benchmark/state_dispatch.rb +78 -0
- data/docs/RELEASING.md +3 -2
- data/docs/explanation/backends.md +30 -10
- data/docs/explanation/matching-semantics.md +4 -2
- data/docs/explanation/security-model.md +10 -3
- data/docs/explanation/unicode-and-encoding.md +3 -1
- data/docs/how-to/deploy-a-standalone-lexer.md +6 -4
- data/docs/how-to/generate-a-lexer.md +3 -3
- data/docs/how-to/handle-errors.md +4 -1
- data/docs/how-to/run-a-lexer-at-runtime.md +5 -0
- data/docs/how-to/track-token-locations.md +2 -1
- data/docs/how-to/tune-performance.md +15 -3
- data/docs/migration-v2.md +74 -0
- data/docs/perf-log.md +212 -0
- data/docs/reference/README.md +2 -0
- data/docs/reference/actions.md +5 -2
- data/docs/reference/cli.md +71 -5
- data/docs/reference/compatibility.md +10 -4
- data/docs/reference/diagnostics.md +346 -0
- data/docs/reference/dsl.md +30 -9
- data/docs/reference/errors.md +34 -3
- data/docs/reference/generated-artifacts.md +71 -6
- data/docs/reference/public-api.md +37 -7
- data/docs/reference/regexp.md +27 -13
- data/docs/reference/ruby-lsp.md +34 -0
- data/docs/reference/runtime.md +21 -4
- data/docs/reference/testing.md +57 -0
- data/docs/reference/tokens-and-locations.md +14 -4
- data/examples/graphql/README.md +14 -0
- data/examples/graphql/lexer.flexr.rb +29 -0
- data/examples/json/lexer.flexr.rb +6 -6
- data/examples/sql/README.md +16 -0
- data/examples/sql/lexer.flexr.rb +23 -0
- data/examples/toml/README.md +14 -0
- data/examples/toml/lexer.flexr.rb +23 -0
- data/gemfiles/ruby_lsp.gemfile +4 -0
- data/gemfiles/types.gemfile +7 -0
- data/lib/flexr/action_resolver.rb +15 -0
- data/lib/flexr/artifact_writer.rb +61 -0
- data/lib/flexr/automaton/accel.rb +78 -5
- data/lib/flexr/automaton/analysis.rb +202 -7
- data/lib/flexr/automaton/backend_cost_model.rb +40 -0
- data/lib/flexr/automaton/compiler.rb +229 -96
- data/lib/flexr/automaton/dfa.rb +106 -19
- data/lib/flexr/automaton/minimizer.rb +79 -31
- data/lib/flexr/automaton/nfa.rb +45 -13
- data/lib/flexr/automaton/types.rb +14 -0
- data/lib/flexr/cli.rb +128 -22
- data/lib/flexr/codegen/direct.rb +5 -41
- data/lib/flexr/codegen/firstmatch.rb +1 -7
- data/lib/flexr/codegen/table.rb +181 -69
- data/lib/flexr/codegen.rb +8 -0
- data/lib/flexr/configuration.rb +45 -0
- data/lib/flexr/diagnostics.rb +39 -3
- data/lib/flexr/dsl.rb +116 -47
- data/lib/flexr/errors.rb +7 -6
- data/lib/flexr/generated.rb +120 -46
- data/lib/flexr/generator.rb +102 -109
- data/lib/flexr/grammar.rb +41 -0
- data/lib/flexr/importer.rb +4 -40
- data/lib/flexr/ir.rb +2 -2
- data/lib/flexr/migration.rb +31 -0
- data/lib/flexr/minitest.rb +24 -0
- data/lib/flexr/options.rb +9 -5
- data/lib/flexr/rake_task.rb +6 -1
- data/lib/flexr/regexp/ast.rb +5 -2
- data/lib/flexr/regexp/matcher.rb +26 -0
- data/lib/flexr/regexp/normalizer.rb +63 -21
- data/lib/flexr/regexp/parser.rb +241 -60
- data/lib/flexr/regexp/tokenizer.rb +172 -47
- data/lib/flexr/rspec.rb +51 -0
- data/lib/flexr/runtime/buffer.rb +115 -9
- data/lib/flexr/runtime/core.rb +248 -51
- data/lib/flexr/runtime/errors.rb +66 -4
- data/lib/flexr/runtime/interpreter.rb +213 -97
- data/lib/flexr/runtime/location.rb +35 -12
- data/lib/flexr/runtime/record.rb +14 -0
- data/lib/flexr/runtime/token.rb +7 -1
- data/lib/flexr/runtime.rb +75 -0
- data/lib/flexr/source/passthrough.rb +70 -14
- data/lib/flexr/source/prism_reader.rb +217 -51
- data/lib/flexr/source/static_eval.rb +40 -4
- data/lib/flexr/source.rb +6 -0
- data/lib/flexr/testing.rb +47 -0
- data/lib/flexr/trace.rb +197 -0
- data/lib/flexr/unicode/data/properties.rb +1 -1
- data/lib/flexr/unicode/data.rb +11 -0
- data/lib/flexr/unicode/property.rb +4 -8
- data/lib/flexr/unicode/reference_regexp.rb +11 -9
- data/lib/flexr/unicode/version.rb +7 -0
- data/lib/flexr/version.rb +1 -1
- data/lib/flexr.rb +10 -76
- data/lib/ruby_lsp/flexr/addon.rb +114 -0
- data/sig/flexr.rbs +191 -0
- data/site/README.md +54 -3
- data/site/astro.config.mjs +3 -1
- data/site/package.json +12 -5
- data/site/playwright.config.mjs +16 -0
- data/site/pnpm-lock.yaml +218 -88
- data/site/ruby/playground.rb +57 -0
- data/site/scripts/prepare-playground.mjs +35 -0
- data/site/scripts/verify-site.mjs +5 -3
- data/site/src/content/docs/concepts/matching-semantics.md +4 -2
- data/site/src/content/docs/concepts/security-model.md +9 -1
- data/site/src/content/docs/learn/generation.md +1 -1
- data/site/src/content/docs/reference/dsl.md +2 -2
- data/site/src/content/docs/reference/public-api.md +4 -2
- data/site/src/content/docs/reference/tokens-and-locations.md +3 -0
- data/site/src/layouts/SiteLayout.astro +11 -3
- data/site/src/pages/index.astro +69 -155
- data/site/src/pages/playground.astro +40 -38
- data/site/src/playground/client.js +211 -0
- data/site/src/playground/graph-worker.mjs +11 -0
- data/site/src/playground/presets.mjs +33 -0
- data/site/src/playground/runtime.mjs +34 -0
- data/site/src/playground/share.mjs +30 -0
- data/site/src/playground/worker.mjs +29 -0
- data/site/src/styles/custom.css +12 -287
- data/site/src/styles/landing.css +239 -0
- data/site/tests/browser/playground.spec.mjs +99 -0
- data/site/tests/playground-runtime.test.mjs +85 -0
- data/test/ruby_lsp_addon_spec.rb +135 -0
- data/tools/coverage.rb +6 -1
- data/tools/docs_verify.rb +3 -2
- data/tools/golden.rb +54 -0
- data/tools/verification_failure.rb +119 -0
- metadata +66 -12
- data/benchmark/golden/calculator_lexer.sha256 +0 -1
- data/benchmark/golden/json_lexer.sha256 +0 -1
- data/benchmark/golden/regexp_tokenizer.sha256 +0 -1
- data/benchmark/golden/ruby_subset_lexer.sha256 +0 -1
- data/benchmark/golden/toy_lang_lexer.sha256 +0 -1
- data/benchmark/golden/with_lrama_lexer.sha256 +0 -1
- data/benchmark/golden/with_racc_lexer.sha256 +0 -1
- data/site/public/playground.js +0 -189
data/lib/flexr/codegen/table.rb
CHANGED
|
@@ -27,57 +27,146 @@ module Flexr
|
|
|
27
27
|
|
|
28
28
|
def source(indent: " ")
|
|
29
29
|
lines = []
|
|
30
|
+
whole_tokens = compiled.machines.values.any?(&:whole_tokens)
|
|
30
31
|
lines << "#{indent}def scan_one"
|
|
31
|
-
lines << "#{indent}
|
|
32
|
-
lines << "#{indent}
|
|
33
|
-
lines << "#{indent}
|
|
32
|
+
lines << "#{indent} if @__flexr_generated_scan_state != @state"
|
|
33
|
+
lines << "#{indent} machine = self.class.__flexr_compiled.machines.fetch(@state)"
|
|
34
|
+
lines << "#{indent} @__flexr_generated_scan_state = @state"
|
|
35
|
+
lines << "#{indent} @__flexr_generated_scan_dfa = machine.dfa"
|
|
36
|
+
lines << "#{indent} @__flexr_generated_whole_tokens = machine.respond_to?(:whole_tokens) ? machine.whole_tokens : nil" if whole_tokens
|
|
37
|
+
lines << "#{indent} @__flexr_generated_scan_fast = __flexr_generated_fast_path?(@__flexr_generated_scan_dfa)"
|
|
38
|
+
lines << "#{indent} @__flexr_generated_scan_simple = __flexr_generated_simple_fast_path?(@__flexr_generated_scan_dfa)"
|
|
39
|
+
lines << "#{indent} end"
|
|
40
|
+
lines << "#{indent} dfa = @__flexr_generated_scan_dfa"
|
|
41
|
+
lines << "#{indent} return Flexr::Runtime::Interpreter.new(self).scan unless @__flexr_generated_scan_fast"
|
|
34
42
|
lines << "#{indent} position = byte_pos"
|
|
35
43
|
lines << "#{indent} return nil unless valid_utf8_at?(position)"
|
|
44
|
+
if whole_tokens
|
|
45
|
+
lines << "#{indent} match = __flexr_whole_token_match(@__flexr_generated_whole_tokens, position) if @__flexr_generated_whole_tokens"
|
|
46
|
+
lines << "#{indent} return match if match"
|
|
47
|
+
end
|
|
48
|
+
lines << "#{indent} return __flexr_generated_scan_simple(dfa, position) if @__flexr_generated_scan_simple"
|
|
36
49
|
lines << "#{indent} cursor = position"
|
|
37
50
|
lines << "#{indent} current = dfa.start"
|
|
38
51
|
lines << "#{indent} direct = dfa.direct"
|
|
39
|
-
lines << "#{indent}
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
lines << "#{indent}
|
|
48
|
-
lines << "#{indent}
|
|
52
|
+
lines << "#{indent} direct_nxt = direct&.fetch(:nxt)"
|
|
53
|
+
lines << "#{indent} direct_classes = direct&.fetch(:classes)"
|
|
54
|
+
lines << "#{indent} ec = dfa.ec"
|
|
55
|
+
lines << "#{indent} source = buffer.stable_source"
|
|
56
|
+
lines << "#{indent} guarded_steps = scan_steps_guarded?"
|
|
57
|
+
lines << "#{indent} token_start = more_text_start || position"
|
|
58
|
+
lines << "#{indent} token_limit = max_token_size"
|
|
59
|
+
lines << "#{indent} check_token_limit = token_limit_required?"
|
|
60
|
+
lines << "#{indent} best = nil"
|
|
61
|
+
lines << "#{indent} accel_mode = @accel_mode"
|
|
62
|
+
lines << "#{indent} accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)"
|
|
63
|
+
lines << "#{indent} while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)"
|
|
49
64
|
lines << "#{indent} if accelerate"
|
|
50
65
|
lines << "#{indent} region = __flexr_generated_acceleration_region(dfa, current)"
|
|
51
|
-
lines << "#{indent}
|
|
66
|
+
lines << "#{indent} acceptance = dfa.accepts[current].first"
|
|
67
|
+
lines << "#{indent} token_rule = acceptance&.rule_index"
|
|
68
|
+
lines << "#{indent} accelerated_end = __flexr_generated_accelerate(region, cursor, token_rule, position) if region"
|
|
52
69
|
lines << "#{indent} if accelerated_end && accelerated_end > cursor"
|
|
53
70
|
lines << "#{indent} cursor = accelerated_end"
|
|
54
71
|
lines << "#{indent} best = __flexr_generated_acceptance(dfa, current, position, cursor, best)"
|
|
55
72
|
lines << "#{indent} next"
|
|
56
73
|
lines << "#{indent} end"
|
|
57
74
|
lines << "#{indent} end"
|
|
58
|
-
lines << "#{indent} byte = source.getbyte(cursor)"
|
|
59
|
-
lines << "#{indent}
|
|
60
|
-
lines << "#{indent}
|
|
61
|
-
lines << "#{indent}
|
|
75
|
+
lines << "#{indent} byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)"
|
|
76
|
+
lines << "#{indent} consume_step! if guarded_steps"
|
|
77
|
+
lines << "#{indent} current = if direct_nxt"
|
|
78
|
+
lines << "#{indent} class_id = ec[byte]"
|
|
79
|
+
lines << "#{indent} value = direct_nxt[(current * direct_classes) + class_id]"
|
|
62
80
|
lines << "#{indent} value >= 0 ? value : nil"
|
|
63
81
|
lines << "#{indent} else"
|
|
64
82
|
lines << "#{indent} dfa.transition(current, byte)"
|
|
65
83
|
lines << "#{indent} end"
|
|
66
84
|
lines << "#{indent} break unless current"
|
|
67
85
|
lines << "#{indent} cursor += 1"
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
86
|
+
lines << "#{indent} resource_acceptance = dfa.accepts[current].first"
|
|
87
|
+
lines << "#{indent} resource_rule = resource_acceptance&.rule_index"
|
|
88
|
+
lines << "#{indent} if check_token_limit"
|
|
89
|
+
lines << "#{indent} size = cursor - token_start"
|
|
90
|
+
lines << "#{indent} defer_token_size_check!(size, rule: resource_rule) if size > token_limit"
|
|
91
|
+
lines << "#{indent} end"
|
|
92
|
+
lines << "#{indent} best = __flexr_generated_acceptance(dfa, current, position, cursor, best)"
|
|
93
|
+
lines << "#{indent} end"
|
|
94
|
+
lines << "#{indent} best"
|
|
95
|
+
lines << "#{indent}end"
|
|
96
|
+
lines << "#{indent}def __flexr_generated_scan_simple(dfa, position)"
|
|
97
|
+
lines << "#{indent} current = dfa.start"
|
|
98
|
+
lines << "#{indent} direct = dfa.direct"
|
|
99
|
+
lines << "#{indent} direct_nxt = direct&.fetch(:nxt)"
|
|
100
|
+
lines << "#{indent} direct_classes = direct&.fetch(:classes)"
|
|
101
|
+
lines << "#{indent} ec = dfa.ec"
|
|
102
|
+
lines << "#{indent} accepts = dfa.accepts"
|
|
103
|
+
lines << "#{indent} rules = self.class.__flexr_rules"
|
|
104
|
+
lines << "#{indent} source = buffer.stable_source"
|
|
105
|
+
lines << "#{indent} guarded_steps = scan_steps_guarded?"
|
|
106
|
+
lines << "#{indent} token_start = more_text_start || position"
|
|
107
|
+
lines << "#{indent} token_limit = max_token_size"
|
|
108
|
+
lines << "#{indent} check_token_limit = token_limit_required?"
|
|
109
|
+
lines << "#{indent} cursor = position"
|
|
110
|
+
lines << "#{indent} best = nil"
|
|
111
|
+
lines << "#{indent} accel_mode = @accel_mode"
|
|
112
|
+
lines << "#{indent} accelerate = accel_mode != :none && !(accel_mode == :auto && @__flexr_generated_auto_accel_off)"
|
|
113
|
+
lines << "#{indent} if source && !guarded_steps && !check_token_limit && !accelerate"
|
|
114
|
+
lines << "#{indent} while cursor < source.bytesize"
|
|
115
|
+
lines << "#{indent} byte = source.getbyte(cursor)"
|
|
116
|
+
lines << "#{indent} current = if direct_nxt"
|
|
117
|
+
lines << "#{indent} class_id = ec[byte]"
|
|
118
|
+
lines << "#{indent} value = direct_nxt[(current * direct_classes) + class_id]"
|
|
119
|
+
lines << "#{indent} value >= 0 ? value : nil"
|
|
120
|
+
lines << "#{indent} else"
|
|
121
|
+
lines << "#{indent} dfa.transition(current, byte)"
|
|
122
|
+
lines << "#{indent} end"
|
|
123
|
+
lines << "#{indent} break unless current"
|
|
124
|
+
lines << "#{indent} cursor += 1"
|
|
125
|
+
lines << "#{indent} acceptance = accepts[current].first"
|
|
126
|
+
lines << "#{indent} next unless acceptance"
|
|
127
|
+
lines << "#{indent} rule = rules.fetch(acceptance.rule_index)"
|
|
128
|
+
lines << "#{indent} best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)"
|
|
129
|
+
lines << "#{indent} best.rule = rule"
|
|
130
|
+
lines << "#{indent} best.start_pos = position"
|
|
131
|
+
lines << "#{indent} best.end_pos = cursor"
|
|
132
|
+
lines << "#{indent} best.total_end_pos = cursor"
|
|
133
|
+
lines << "#{indent} end"
|
|
134
|
+
lines << "#{indent} return best"
|
|
135
|
+
lines << "#{indent} end"
|
|
136
|
+
lines << "#{indent} while cursor < (source ? source.bytesize : buffer.bytesize) || buffer.ensure_available?(cursor + 1)"
|
|
137
|
+
lines << "#{indent} if accelerate"
|
|
138
|
+
lines << "#{indent} region = __flexr_generated_acceleration_region(dfa, current)"
|
|
139
|
+
lines << "#{indent} acceptance = accepts[current].first"
|
|
140
|
+
lines << "#{indent} accelerated_end = __flexr_generated_accelerate(region, cursor, acceptance&.rule_index, position) if region"
|
|
141
|
+
lines << "#{indent} if accelerated_end && accelerated_end > cursor"
|
|
142
|
+
lines << "#{indent} cursor = accelerated_end"
|
|
143
|
+
lines << "#{indent} best = __flexr_generated_acceptance(dfa, current, position, cursor, best)"
|
|
144
|
+
lines << "#{indent} next"
|
|
145
|
+
lines << "#{indent} end"
|
|
146
|
+
lines << "#{indent} end"
|
|
147
|
+
lines << "#{indent} byte = source ? source.getbyte(cursor) : buffer.getbyte(cursor)"
|
|
148
|
+
lines << "#{indent} consume_step! if guarded_steps"
|
|
149
|
+
lines << "#{indent} current = if direct_nxt"
|
|
150
|
+
lines << "#{indent} class_id = ec[byte]"
|
|
151
|
+
lines << "#{indent} value = direct_nxt[(current * direct_classes) + class_id]"
|
|
152
|
+
lines << "#{indent} value >= 0 ? value : nil"
|
|
153
|
+
lines << "#{indent} else"
|
|
154
|
+
lines << "#{indent} dfa.transition(current, byte)"
|
|
155
|
+
lines << "#{indent} end"
|
|
156
|
+
lines << "#{indent} break unless current"
|
|
157
|
+
lines << "#{indent} cursor += 1"
|
|
158
|
+
lines << "#{indent} acceptance = accepts[current].first"
|
|
159
|
+
lines << "#{indent} if check_token_limit"
|
|
160
|
+
lines << "#{indent} size = cursor - token_start"
|
|
161
|
+
lines << "#{indent} defer_token_size_check!(size, rule: acceptance&.rule_index) if size > token_limit"
|
|
162
|
+
lines << "#{indent} end"
|
|
163
|
+
lines << "#{indent} next unless acceptance"
|
|
164
|
+
lines << "#{indent} rule = rules.fetch(acceptance.rule_index)"
|
|
165
|
+
lines << "#{indent} best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)"
|
|
166
|
+
lines << "#{indent} best.rule = rule"
|
|
167
|
+
lines << "#{indent} best.start_pos = position"
|
|
168
|
+
lines << "#{indent} best.end_pos = cursor"
|
|
169
|
+
lines << "#{indent} best.total_end_pos = cursor"
|
|
81
170
|
lines << "#{indent} end"
|
|
82
171
|
lines << "#{indent} best"
|
|
83
172
|
lines << "#{indent}end"
|
|
@@ -86,44 +175,75 @@ module Flexr
|
|
|
86
175
|
lines << "#{indent} regions = (@__flexr_generated_accel_regions[dfa] ||= Flexr::Automaton::Accel.extract(dfa).to_h { |region| [region.state, region] })"
|
|
87
176
|
lines << "#{indent} region = regions[state]"
|
|
88
177
|
lines << "#{indent} return unless region"
|
|
178
|
+
lines << "#{indent} return if @__flexr_generated_disabled_auto_accel&.key?(region)"
|
|
179
|
+
lines << "#{indent} return if utf8_input? && region.bytes.any? { |byte| byte >= 128 }"
|
|
89
180
|
lines << "#{indent} return if dfa.accepts[state].any? do |acceptance|"
|
|
90
181
|
lines << "#{indent} rule = self.class.__flexr_rules.fetch(acceptance.rule_index)"
|
|
91
182
|
lines << "#{indent} acceptance.bol_only || acceptance.end_anchor || rule.trailing"
|
|
92
183
|
lines << "#{indent} end"
|
|
93
184
|
lines << "#{indent} region"
|
|
94
185
|
lines << "#{indent}end"
|
|
95
|
-
lines << "#{indent}def __flexr_generated_accelerate(region, position)"
|
|
96
|
-
lines << "#{indent}
|
|
97
|
-
lines << "#{indent}
|
|
98
|
-
lines << "#{indent}
|
|
99
|
-
lines << "#{indent}
|
|
100
|
-
lines << "#{indent}
|
|
101
|
-
lines << "#{indent}
|
|
102
|
-
lines << "#{indent}
|
|
103
|
-
lines << "#{indent}
|
|
104
|
-
lines << "#{indent}
|
|
105
|
-
lines << "#{indent}
|
|
106
|
-
lines << "#{indent}
|
|
107
|
-
lines << "#{indent}
|
|
108
|
-
lines << "#{indent}
|
|
109
|
-
lines << "#{indent}
|
|
110
|
-
lines << "#{indent}
|
|
111
|
-
lines << "#{indent}
|
|
112
|
-
lines << "#{indent}
|
|
113
|
-
lines << "#{indent}
|
|
114
|
-
lines << "#{indent}
|
|
115
|
-
lines << "#{indent}
|
|
116
|
-
lines << "#{indent}
|
|
117
|
-
lines << "#{indent}
|
|
118
|
-
lines << "#{indent}
|
|
119
|
-
lines << "#{indent}
|
|
120
|
-
lines << "#{indent}
|
|
121
|
-
lines << "#{indent} !rule.trailing.nil? || rule.patterns.any? do |pattern|"
|
|
122
|
-
lines << "#{indent} next false unless pattern.is_a?(Regexp)"
|
|
123
|
-
lines << "#{indent} next true if pattern.source.match?(/\\\\[pP]\\\\{/) || pattern.source.match?(/\\[:(?:\\^)?[a-z]+:\\]/)"
|
|
124
|
-
lines << "#{indent} self.class.__flexr_config.options[:unicode] == true && utf8_input? && pattern.source.match?(/\\\\[dDwWsS]/)"
|
|
186
|
+
lines << "#{indent}def __flexr_generated_accelerate(region, position, token_rule, token_start)"
|
|
187
|
+
lines << "#{indent} mode = @accel_mode"
|
|
188
|
+
lines << "#{indent} cursor = position"
|
|
189
|
+
lines << "#{indent} matched = false"
|
|
190
|
+
lines << "#{indent} loop do"
|
|
191
|
+
lines << "#{indent} if cursor >= buffer.bytesize"
|
|
192
|
+
lines << "#{indent} return matched ? cursor : nil if buffer.eof_loaded?"
|
|
193
|
+
lines << "#{indent} return matched ? cursor : nil unless buffer.ensure_available?(cursor + 1)"
|
|
194
|
+
lines << "#{indent} end"
|
|
195
|
+
lines << "#{indent} segment = buffer.source"
|
|
196
|
+
lines << "#{indent} segment_position = cursor - buffer.base_offset"
|
|
197
|
+
lines << "#{indent} if segment.encoding == ::Encoding::UTF_8 && region.utf8_regexp"
|
|
198
|
+
lines << "#{indent} regexp = region.utf8_regexp"
|
|
199
|
+
lines << "#{indent} else"
|
|
200
|
+
lines << "#{indent} segment = segment.b unless segment.encoding == ::Encoding::BINARY"
|
|
201
|
+
lines << "#{indent} regexp = region.regexp"
|
|
202
|
+
lines << "#{indent} end"
|
|
203
|
+
lines << "#{indent} length = begin"
|
|
204
|
+
lines << "#{indent} if %i[strscan auto].include?(mode) && defined?(::StringScanner)"
|
|
205
|
+
lines << "#{indent} @__flexr_generated_accel_scanner ||= ::StringScanner.new(\"\".b)"
|
|
206
|
+
lines << "#{indent} @__flexr_generated_accel_scanner.string = segment"
|
|
207
|
+
lines << "#{indent} @__flexr_generated_accel_scanner.pos = segment_position"
|
|
208
|
+
lines << "#{indent} @__flexr_generated_accel_scanner.skip(regexp)"
|
|
209
|
+
lines << "#{indent} else"
|
|
210
|
+
lines << "#{indent} match = regexp.match(segment.byteslice(segment_position..).to_s, 0)"
|
|
211
|
+
lines << "#{indent} match && match.begin(0).zero? ? match[0].bytesize : nil"
|
|
125
212
|
lines << "#{indent} end"
|
|
213
|
+
lines << "#{indent} rescue ArgumentError"
|
|
214
|
+
lines << "#{indent} nil"
|
|
126
215
|
lines << "#{indent} end"
|
|
216
|
+
lines << "#{indent} __flexr_generated_record_auto_accel_miss(region) if mode == :auto && length.to_i < 8"
|
|
217
|
+
lines << "#{indent} return matched ? cursor : nil unless length&.positive?"
|
|
218
|
+
lines << "#{indent} matched = true"
|
|
219
|
+
lines << "#{indent} cursor += length"
|
|
220
|
+
lines << "#{indent} consume_step!(length)"
|
|
221
|
+
lines << "#{indent} defer_token_size_check!(cursor - (more_text_start || token_start), rule: token_rule)"
|
|
222
|
+
lines << "#{indent} return cursor if cursor < buffer.bytesize || buffer.eof_loaded?"
|
|
223
|
+
lines << "#{indent} end"
|
|
224
|
+
lines << "#{indent}end"
|
|
225
|
+
lines << "#{indent}def __flexr_generated_record_auto_accel_miss(region)"
|
|
226
|
+
lines << "#{indent} @__flexr_generated_auto_accel_misses ||= Hash.new(0)"
|
|
227
|
+
lines << "#{indent} @__flexr_generated_auto_accel_misses[region] += 1"
|
|
228
|
+
lines << "#{indent} @__flexr_generated_auto_accel_miss_total = @__flexr_generated_auto_accel_miss_total.to_i + 1"
|
|
229
|
+
lines << "#{indent} @__flexr_generated_auto_accel_off = true if @__flexr_generated_auto_accel_miss_total >= 9"
|
|
230
|
+
lines << "#{indent} return if @__flexr_generated_auto_accel_misses[region] < 3"
|
|
231
|
+
lines << "#{indent} @__flexr_generated_disabled_auto_accel ||= {}"
|
|
232
|
+
lines << "#{indent} @__flexr_generated_disabled_auto_accel[region] = true"
|
|
233
|
+
lines << "#{indent}end"
|
|
234
|
+
lines << "#{indent}def __flexr_generated_fast_path?(dfa)"
|
|
235
|
+
lines << "#{indent} @__flexr_generated_fast_paths ||= {}"
|
|
236
|
+
lines << "#{indent} return @__flexr_generated_fast_paths[dfa] if @__flexr_generated_fast_paths.key?(dfa)"
|
|
237
|
+
lines << "#{indent} rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }"
|
|
238
|
+
lines << "#{indent} @__flexr_generated_fast_paths[dfa] = self.class.__flexr_config.backend != :firstmatch &&"
|
|
239
|
+
lines << "#{indent} self.class.__flexr_config.options[:allow_empty_match] != true && rules.none?(&:trailing)"
|
|
240
|
+
lines << "#{indent}end"
|
|
241
|
+
lines << "#{indent}def __flexr_generated_simple_fast_path?(dfa)"
|
|
242
|
+
lines << "#{indent} @__flexr_generated_simple_fast_paths ||= {}"
|
|
243
|
+
lines << "#{indent} return @__flexr_generated_simple_fast_paths[dfa] if @__flexr_generated_simple_fast_paths.key?(dfa)"
|
|
244
|
+
lines << "#{indent} rules = dfa.rule_ids.map { |rule_index| self.class.__flexr_rules.fetch(rule_index) }"
|
|
245
|
+
lines << "#{indent} @__flexr_generated_simple_fast_paths[dfa] = rules.all? do |rule|"
|
|
246
|
+
lines << "#{indent} rule.pattern_conditions.all? { |condition| condition && !condition.bol_only && !condition.end_anchor }"
|
|
127
247
|
lines << "#{indent} end"
|
|
128
248
|
lines << "#{indent}end"
|
|
129
249
|
lines << "#{indent}def __flexr_generated_acceptance(dfa, state, start_position, cursor, best)"
|
|
@@ -131,7 +251,6 @@ module Flexr
|
|
|
131
251
|
lines << "#{indent} next if acceptance.bol_only && !beginning_of_line?"
|
|
132
252
|
lines << "#{indent} next if acceptance.end_anchor && !(buffer.eof?(cursor) || buffer.getbyte(cursor) == 0x0a)"
|
|
133
253
|
lines << "#{indent} rule = self.class.__flexr_rules.fetch(acceptance.rule_index)"
|
|
134
|
-
lines << "#{indent} defer_token_size_check!(cursor - start_position)"
|
|
135
254
|
lines << "#{indent} next if best && cursor < best.total_end_pos"
|
|
136
255
|
lines << "#{indent} next if best && cursor == best.total_end_pos &&"
|
|
137
256
|
lines << "#{indent} rule.index > best.rule.index"
|
|
@@ -146,13 +265,6 @@ module Flexr
|
|
|
146
265
|
"#{lines.join("\n")}\n"
|
|
147
266
|
end
|
|
148
267
|
|
|
149
|
-
def simple_fast_path?
|
|
150
|
-
compiled.rules.all? do |rule|
|
|
151
|
-
rule.trailing.nil? && rule.pattern_conditions.all? do |condition|
|
|
152
|
-
condition && !condition.bol_only && !condition.end_anchor
|
|
153
|
-
end
|
|
154
|
-
end
|
|
155
|
-
end
|
|
156
268
|
end
|
|
157
269
|
end
|
|
158
270
|
end
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "runtime" unless defined?(Flexr::Lexer)
|
|
4
|
+
require_relative "codegen/base"
|
|
5
|
+
require_relative "codegen/table"
|
|
6
|
+
require_relative "codegen/direct"
|
|
7
|
+
require_relative "codegen/firstmatch"
|
|
8
|
+
require_relative "codegen/table_packer"
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Flexr
|
|
4
|
+
module Configuration
|
|
5
|
+
BACKENDS = %i[table direct firstmatch auto].freeze
|
|
6
|
+
TOKEN_KINDS = %i[array struct yield].freeze
|
|
7
|
+
ACCELERATORS = %i[auto strscan regexp none].freeze
|
|
8
|
+
BOOLEAN_OPTIONS = %i[unicode eager_columns allow_empty_match experimental standalone].freeze
|
|
9
|
+
|
|
10
|
+
module_function
|
|
11
|
+
|
|
12
|
+
def backend!(value)
|
|
13
|
+
validated_symbol!(value, BACKENDS, :backend)
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def token_kind!(value)
|
|
17
|
+
validated_symbol!(value, TOKEN_KINDS, :token_kind)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def accelerator!(value)
|
|
21
|
+
validated_symbol!(value, ACCELERATORS, :accel)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def encoding!(value)
|
|
25
|
+
encoding = value.is_a?(Encoding) ? value : Encoding.find(value.to_s)
|
|
26
|
+
return encoding if [Encoding::UTF_8, Encoding::BINARY].include?(encoding)
|
|
27
|
+
|
|
28
|
+
diagnostic = Diagnostics.error("FLEXR-E011", "flexr supports UTF-8 and BINARY only")
|
|
29
|
+
raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def option!(value)
|
|
33
|
+
validated_symbol!(value, BOOLEAN_OPTIONS, :option)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def validated_symbol!(value, allowed, name)
|
|
37
|
+
symbol = value.to_sym
|
|
38
|
+
return symbol if allowed.include?(symbol)
|
|
39
|
+
|
|
40
|
+
raise ArgumentError, "unsupported #{name}: #{value.inspect}"
|
|
41
|
+
rescue NoMethodError
|
|
42
|
+
raise ArgumentError, "unsupported #{name}: #{value.inspect}"
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
data/lib/flexr/diagnostics.rb
CHANGED
|
@@ -50,23 +50,46 @@ module Flexr
|
|
|
50
50
|
@items.dup
|
|
51
51
|
end
|
|
52
52
|
|
|
53
|
-
def render(format: :human, color: :auto)
|
|
53
|
+
def render(format: :human, color: :auto, source: nil, path: nil)
|
|
54
54
|
return JSON.generate(@items.map(&:to_h)) if format.to_sym == :json
|
|
55
55
|
|
|
56
|
-
@items.map { |item| render_one(item, color: color_enabled?(color)) }.join("\n")
|
|
56
|
+
@items.map { |item| render_one(item, color: color_enabled?(color), source: source, path: path) }.join("\n")
|
|
57
57
|
end
|
|
58
58
|
|
|
59
59
|
private
|
|
60
60
|
|
|
61
|
-
def render_one(item, color:)
|
|
61
|
+
def render_one(item, color:, source:, path:)
|
|
62
62
|
prefix = "#{item.severity}[#{item.code}]: #{item.message}"
|
|
63
63
|
prefix = "\e[31m#{prefix}\e[0m" if [true, :always].include?(color)
|
|
64
64
|
lines = [prefix]
|
|
65
|
+
lines.concat(render_source(item.location, source, path))
|
|
65
66
|
lines << " help: #{item.help}" if item.help
|
|
66
67
|
lines << " note: #{item.note}" if item.note
|
|
67
68
|
lines.join("\n")
|
|
68
69
|
end
|
|
69
70
|
|
|
71
|
+
def render_source(location, source, path)
|
|
72
|
+
return [] unless source.is_a?(String) && location.is_a?(Array) && location.length == 2 &&
|
|
73
|
+
location.all?(Integer)
|
|
74
|
+
|
|
75
|
+
first, last = location
|
|
76
|
+
return [] unless first.between?(0, source.bytesize) && last >= first
|
|
77
|
+
|
|
78
|
+
before = source.byteslice(0, first).b
|
|
79
|
+
line = before.count("\n") + 1
|
|
80
|
+
line_start = (before.rindex("\n") || -1) + 1
|
|
81
|
+
line_end = source.b.index("\n", first) || source.bytesize
|
|
82
|
+
text = source.byteslice(line_start...line_end).delete_suffix("\r")
|
|
83
|
+
prefix = source.byteslice(line_start...first).delete_suffix("\r").scrub
|
|
84
|
+
last = last.clamp(first, [line_start + text.bytesize, first].max)
|
|
85
|
+
width = [source.byteslice(first...last).scrub.length, 1].max
|
|
86
|
+
gutter = " " * line.to_s.length
|
|
87
|
+
# shortcut: carets count characters; use display-cell widths to align wide or combining glyphs.
|
|
88
|
+
indentation = prefix.gsub(/[^\t]/, " ")
|
|
89
|
+
[" --> #{path || '(source)'}:#{line}:#{prefix.length + 1}",
|
|
90
|
+
"#{gutter} |", "#{line} | #{text.scrub}", "#{gutter} | #{indentation}#{'^' * width}"]
|
|
91
|
+
end
|
|
92
|
+
|
|
70
93
|
def color_enabled?(color)
|
|
71
94
|
return false if ENV.key?("NO_COLOR") || color == :never || color == false
|
|
72
95
|
return true if [:always, true].include?(color)
|
|
@@ -78,6 +101,15 @@ module Flexr
|
|
|
78
101
|
module Diagnostics
|
|
79
102
|
module_function
|
|
80
103
|
|
|
104
|
+
def rule_label(rule)
|
|
105
|
+
return if rule.nil?
|
|
106
|
+
return "rule #{rule}" unless rule.is_a?(IR::Rule)
|
|
107
|
+
|
|
108
|
+
pattern = rule.patterns.one? ? rule.patterns.first : rule.patterns
|
|
109
|
+
label = "rule #{pattern.inspect}"
|
|
110
|
+
rule.trailing ? "#{label}, followed_by: #{rule.trailing.inspect}" : label
|
|
111
|
+
end
|
|
112
|
+
|
|
81
113
|
def error(code, message, location: nil, help: nil, note: nil)
|
|
82
114
|
Diagnostic.new(code: code, severity: :error, message: message, location: location, help: help, note: note)
|
|
83
115
|
end
|
|
@@ -86,6 +118,10 @@ module Flexr
|
|
|
86
118
|
Diagnostic.new(code: code, severity: :warning, message: message, location: location, help: help, note: note)
|
|
87
119
|
end
|
|
88
120
|
|
|
121
|
+
def info(code, message, location: nil, help: nil, note: nil)
|
|
122
|
+
Diagnostic.new(code: code, severity: :info, message: message, location: location, help: help, note: note)
|
|
123
|
+
end
|
|
124
|
+
|
|
89
125
|
def raise!(diagnostic)
|
|
90
126
|
klass = diagnostic.code == "FLEXR-E014" ? UnsupportedRegexpError : CompileError
|
|
91
127
|
raise klass.new(diagnostic.message, diagnostic: diagnostic)
|