ibex 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +247 -104
- data/docs/architecture.md +322 -28
- data/docs/cst-migration.md +107 -0
- data/docs/cst.md +213 -0
- data/docs/development.md +129 -0
- data/docs/editor-setup.md +25 -0
- data/docs/error-ux.md +60 -0
- data/docs/grammar-reference.md +458 -18
- data/docs/lexer-migration.md +51 -0
- data/docs/racc-migration.md +34 -6
- data/docs/release-readiness.md +163 -0
- data/docs/stability.md +124 -0
- data/examples/README.md +60 -0
- data/examples/calculator.y +44 -0
- data/examples/csv.y +32 -0
- data/examples/ini.y +70 -0
- data/examples/json.y +48 -0
- data/examples/tiny_language.y +75 -0
- data/lib/ibex/analysis/sets.rb +5 -4
- data/lib/ibex/artifact_set.rb +63 -0
- data/lib/ibex/cli/ambiguity.rb +66 -0
- data/lib/ibex/cli/counterexample_options.rb +6 -2
- data/lib/ibex/cli/coverage.rb +173 -0
- data/lib/ibex/cli/debug.rb +106 -0
- data/lib/ibex/cli/diagnostics.rb +116 -0
- data/lib/ibex/cli/documentation.rb +67 -0
- data/lib/ibex/cli/error_messages.rb +177 -0
- data/lib/ibex/cli/explain.rb +76 -0
- data/lib/ibex/cli/formatting.rb +386 -0
- data/lib/ibex/cli/generation_artifacts.rb +138 -0
- data/lib/ibex/cli/generation_error_messages.rb +39 -0
- data/lib/ibex/cli/grammar_tests.rb +122 -0
- data/lib/ibex/cli/ir_tools.rb +184 -0
- data/lib/ibex/cli/lsp.rb +33 -0
- data/lib/ibex/cli/outputs.rb +91 -8
- data/lib/ibex/cli/racc_migration.rb +87 -0
- data/lib/ibex/cli/samples.rb +92 -0
- data/lib/ibex/cli/watch.rb +103 -0
- data/lib/ibex/cli.rb +506 -37
- data/lib/ibex/codegen/action_locations.rb +103 -0
- data/lib/ibex/codegen/action_method_source.rb +210 -0
- data/lib/ibex/codegen/action_source.rb +118 -0
- data/lib/ibex/codegen/ambiguity.rb +173 -0
- data/lib/ibex/codegen/cst_metadata.rb +171 -0
- data/lib/ibex/codegen/documentation.rb +170 -0
- data/lib/ibex/codegen/explain.rb +312 -0
- data/lib/ibex/codegen/generated_action_abi.rb +288 -0
- data/lib/ibex/codegen/html.rb +94 -10
- data/lib/ibex/codegen/mermaid.rb +43 -0
- data/lib/ibex/codegen/railroad.rb +197 -0
- data/lib/ibex/codegen/railroad_documentation.rb +59 -0
- data/lib/ibex/codegen/rbs.rb +323 -2
- data/lib/ibex/codegen/report.rb +46 -6
- data/lib/ibex/codegen/ruby.rb +334 -59
- data/lib/ibex/codegen/ruby_actions.rb +143 -0
- data/lib/ibex/codegen/ruby_ast.rb +121 -0
- data/lib/ibex/codegen/ruby_error_messages.rb +28 -0
- data/lib/ibex/codegen/ruby_lexer.rb +83 -0
- data/lib/ibex/codegen/ruby_syntax.rb +90 -0
- data/lib/ibex/codegen/ruby_table_metadata.rb +57 -0
- data/lib/ibex/codegen/ruby_value_printers.rb +56 -0
- data/lib/ibex/codegen/symbol_labels.rb +1 -1
- data/lib/ibex/coverage/collector.rb +188 -0
- data/lib/ibex/coverage/event_stream.rb +97 -0
- data/lib/ibex/coverage/report.rb +259 -0
- data/lib/ibex/coverage/runtime_event_validator.rb +160 -0
- data/lib/ibex/coverage.rb +13 -0
- data/lib/ibex/error_messages/parser.rb +159 -0
- data/lib/ibex/error_messages/parser_v2.rb +198 -0
- data/lib/ibex/error_messages/renderer.rb +65 -0
- data/lib/ibex/error_messages/sentence_search.rb +196 -0
- data/lib/ibex/error_messages/update.rb +169 -0
- data/lib/ibex/error_messages.rb +165 -0
- data/lib/ibex/frontend/ast.rb +145 -6
- data/lib/ibex/frontend/bootstrap_parser.rb +47 -4
- data/lib/ibex/frontend/diagnostic.rb +81 -0
- data/lib/ibex/frontend/diagnostic_recovery.rb +268 -0
- data/lib/ibex/frontend/dsl.rb +66 -7
- data/lib/ibex/frontend/formatter.rb +407 -0
- data/lib/ibex/frontend/generated_parser.rb +614 -190
- data/lib/ibex/frontend/generated_parser_base.rb +164 -30
- data/lib/ibex/frontend/generated_parser_includes.rb +61 -0
- data/lib/ibex/frontend/generated_parser_metadata.rb +61 -0
- data/lib/ibex/frontend/generated_parser_parameters.rb +60 -0
- data/lib/ibex/frontend/generation.rb +33 -0
- data/lib/ibex/frontend/lexer.rb +162 -10
- data/lib/ibex/frontend/lexer_recovery.rb +84 -0
- data/lib/ibex/frontend/parser/declarations.rb +247 -11
- data/lib/ibex/frontend/parser/parameters.rb +82 -0
- data/lib/ibex/frontend/parser/rules.rb +43 -6
- data/lib/ibex/frontend/parser.rb +182 -4
- data/lib/ibex/frontend/regenerator.rb +26 -1
- data/lib/ibex/frontend/resolution.rb +69 -0
- data/lib/ibex/frontend/resolver.rb +217 -0
- data/lib/ibex/frontend/rule_documentation.rb +103 -0
- data/lib/ibex/frontend/source_cursor.rb +132 -8
- data/lib/ibex/frontend/source_document.rb +229 -0
- data/lib/ibex/frontend/source_loader.rb +150 -0
- data/lib/ibex/frontend/source_span.rb +81 -0
- data/lib/ibex/frontend/token_adapter/declaration_document_state.rb +47 -0
- data/lib/ibex/frontend/token_adapter/declaration_lexer_state.rb +83 -0
- data/lib/ibex/frontend/token_adapter/declaration_state.rb +216 -26
- data/lib/ibex/frontend/token_adapter/delimiter_tracker.rb +8 -2
- data/lib/ibex/frontend/token_adapter/rule_state.rb +60 -2
- data/lib/ibex/frontend/token_adapter.rb +8 -3
- data/lib/ibex/frontend.rb +13 -2
- data/lib/ibex/generation_input.rb +57 -0
- data/lib/ibex/generation_manifest.rb +200 -0
- data/lib/ibex/generation_transaction.rb +261 -0
- data/lib/ibex/generation_transaction_recovery.rb +109 -0
- data/lib/ibex/generation_transaction_validation.rb +196 -0
- data/lib/ibex/grammar_tests.rb +206 -0
- data/lib/ibex/ir/automaton_ir.rb +38 -5
- data/lib/ibex/ir/grammar_ir.rb +137 -24
- data/lib/ibex/ir/lexer_ir.rb +76 -0
- data/lib/ibex/ir/migration.rb +120 -0
- data/lib/ibex/ir/serialize.rb +110 -19
- data/lib/ibex/ir/validator/automaton.rb +345 -0
- data/lib/ibex/ir/validator/base.rb +129 -0
- data/lib/ibex/ir/validator/grammar.rb +604 -0
- data/lib/ibex/ir/validator/lexer.rb +113 -0
- data/lib/ibex/ir/validator.rb +62 -0
- data/lib/ibex/ir.rb +57 -4
- data/lib/ibex/lalr/build_metrics.rb +23 -0
- data/lib/ibex/lalr/builder.rb +379 -52
- data/lib/ibex/lalr/conflict.rb +1 -0
- data/lib/ibex/lalr/conflict_search.rb +11 -5
- data/lib/ibex/lalr/counterexample.rb +20 -5
- data/lib/ibex/lalr/direct_lookaheads.rb +236 -0
- data/lib/ibex/lalr/ielr_partition.rb +152 -0
- data/lib/ibex/lalr/on_error_reductions.rb +74 -0
- data/lib/ibex/lalr.rb +7 -0
- data/lib/ibex/location.rb +129 -0
- data/lib/ibex/lsp/document_handlers.rb +66 -0
- data/lib/ibex/lsp/document_store.rb +264 -0
- data/lib/ibex/lsp/document_store_diagnostics.rb +44 -0
- data/lib/ibex/lsp/document_store_validation.rb +61 -0
- data/lib/ibex/lsp/initialization_handlers.rb +78 -0
- data/lib/ibex/lsp/navigation_handlers.rb +56 -0
- data/lib/ibex/lsp/position_codec.rb +109 -0
- data/lib/ibex/lsp/protocol_error.rb +33 -0
- data/lib/ibex/lsp/request_handlers.rb +45 -0
- data/lib/ibex/lsp/request_support.rb +87 -0
- data/lib/ibex/lsp/server.rb +146 -0
- data/lib/ibex/lsp/symbol_index.rb +241 -0
- data/lib/ibex/lsp/symbol_index_builder.rb +267 -0
- data/lib/ibex/lsp/symbol_index_precedence_references.rb +44 -0
- data/lib/ibex/lsp/symbol_index_source_queries.rb +61 -0
- data/lib/ibex/lsp/symbol_occurrence.rb +17 -0
- data/lib/ibex/lsp/transport.rb +118 -0
- data/lib/ibex/lsp/workspace.rb +127 -0
- data/lib/ibex/lsp/workspace_analyzer.rb +199 -0
- data/lib/ibex/lsp.rb +32 -0
- data/lib/ibex/normalize/declarations.rb +140 -7
- data/lib/ibex/normalize/diagnostics.rb +50 -5
- data/lib/ibex/normalize/expander.rb +59 -55
- data/lib/ibex/normalize/expression.rb +60 -39
- data/lib/ibex/normalize/inline_expansion.rb +414 -0
- data/lib/ibex/normalize/inline_validation.rb +174 -0
- data/lib/ibex/normalize/lexer.rb +131 -0
- data/lib/ibex/normalize/named_references.rb +60 -0
- data/lib/ibex/normalize/nodes.rb +46 -0
- data/lib/ibex/normalize/parameter_ebnf_lowering.rb +69 -0
- data/lib/ibex/normalize/parameter_lowering.rb +126 -0
- data/lib/ibex/normalize/parameter_substitution.rb +125 -0
- data/lib/ibex/normalize/parameter_validation.rb +140 -0
- data/lib/ibex/normalize/parameters.rb +199 -0
- data/lib/ibex/normalize/recovery_declarations.rb +84 -0
- data/lib/ibex/normalize.rb +179 -14
- data/lib/ibex/racc_migration/checker.rb +122 -0
- data/lib/ibex/racc_migration/harness.rb +177 -0
- data/lib/ibex/racc_migration/report.rb +94 -0
- data/lib/ibex/racc_migration.rb +12 -0
- data/lib/ibex/rake_task.rb +116 -0
- data/lib/ibex/samples.rb +186 -0
- data/lib/ibex/table_simulation/result.rb +51 -0
- data/lib/ibex/table_simulation/simulator.rb +253 -0
- data/lib/ibex/table_simulation/step.rb +60 -0
- data/lib/ibex/table_simulation/text.rb +31 -0
- data/lib/ibex/table_simulation.rb +13 -0
- data/lib/ibex/tables.rb +9 -70
- data/lib/ibex/version.rb +1 -1
- data/lib/ibex/watch/runner.rb +172 -0
- data/lib/ibex/watch/source_snapshot.rb +93 -0
- data/lib/ibex/watch.rb +11 -0
- data/lib/ibex.rb +25 -1
- data/schema/automaton-ir-v1.schema.json +401 -0
- data/schema/automaton-ir-v2.schema.json +58 -0
- data/schema/benchmark-v1.schema.json +212 -0
- data/schema/benchmark-v2.schema.json +61 -0
- data/schema/cst-v1.json +170 -0
- data/schema/error-ux-v1.schema.json +258 -0
- data/schema/explain-v1.schema.json +433 -0
- data/schema/frontend-diagnostics-v1.schema.json +154 -0
- data/schema/generation-manifest-v1.schema.json +115 -0
- data/schema/grammar-ir-v1.schema.json +426 -0
- data/schema/grammar-ir-v2.schema.json +779 -0
- data/schema/lexer-ir-v1.schema.json +215 -0
- data/schema/migration-check-v1.schema.json +60 -0
- data/schema/performance-comparison-v1.schema.json +395 -0
- data/schema/public-performance-comparison-v1.schema.json +506 -0
- data/schema/public-performance-profile-v1.schema.json +360 -0
- data/schema/runtime-coverage-v1.schema.json +86 -0
- data/schema/runtime-event-v1.schema.json +308 -0
- data/schema/table-simulation-v1.schema.json +70 -0
- data/sig/ibex/artifact_set.rbs +37 -0
- data/sig/ibex/cli/ambiguity.rbs +22 -0
- data/sig/ibex/cli/counterexample_options.rbs +2 -0
- data/sig/ibex/cli/coverage.rbs +53 -0
- data/sig/ibex/cli/debug.rbs +28 -0
- data/sig/ibex/cli/diagnostics.rbs +38 -0
- data/sig/ibex/cli/documentation.rbs +25 -0
- data/sig/ibex/cli/error_messages.rbs +57 -0
- data/sig/ibex/cli/explain.rbs +25 -0
- data/sig/ibex/cli/formatting.rbs +103 -0
- data/sig/ibex/cli/generation_artifacts.rbs +50 -0
- data/sig/ibex/cli/generation_error_messages.rbs +19 -0
- data/sig/ibex/cli/grammar_tests.rbs +36 -0
- data/sig/ibex/cli/ir_tools.rbs +51 -0
- data/sig/ibex/cli/lsp.rbs +14 -0
- data/sig/ibex/cli/outputs.rbs +17 -0
- data/sig/ibex/cli/racc_migration.rbs +30 -0
- data/sig/ibex/cli/samples.rbs +30 -0
- data/sig/ibex/cli/watch.rbs +43 -0
- data/sig/ibex/cli.rbs +104 -5
- data/sig/ibex/codegen/action_locations.rbs +45 -0
- data/sig/ibex/codegen/action_method_source.rbs +65 -0
- data/sig/ibex/codegen/action_source.rbs +50 -0
- data/sig/ibex/codegen/ambiguity.rbs +60 -0
- data/sig/ibex/codegen/cst_metadata.rbs +59 -0
- data/sig/ibex/codegen/documentation.rbs +50 -0
- data/sig/ibex/codegen/explain.rbs +85 -0
- data/sig/ibex/codegen/generated_action_abi.rbs +101 -0
- data/sig/ibex/codegen/html.rbs +18 -2
- data/sig/ibex/codegen/mermaid.rbs +16 -0
- data/sig/ibex/codegen/railroad.rbs +82 -0
- data/sig/ibex/codegen/railroad_documentation.rbs +31 -0
- data/sig/ibex/codegen/rbs.rbs +84 -4
- data/sig/ibex/codegen/report.rbs +8 -0
- data/sig/ibex/codegen/ruby.rbs +97 -23
- data/sig/ibex/codegen/ruby_actions.rbs +54 -0
- data/sig/ibex/codegen/ruby_ast.rbs +34 -0
- data/sig/ibex/codegen/ruby_error_messages.rbs +16 -0
- data/sig/ibex/codegen/ruby_lexer.rbs +25 -0
- data/sig/ibex/codegen/ruby_syntax.rbs +25 -0
- data/sig/ibex/codegen/ruby_table_metadata.rbs +26 -0
- data/sig/ibex/codegen/ruby_value_printers.rbs +28 -0
- data/sig/ibex/coverage/collector.rbs +76 -0
- data/sig/ibex/coverage/event_stream.rbs +42 -0
- data/sig/ibex/coverage/report.rbs +100 -0
- data/sig/ibex/coverage/runtime_event_validator.rbs +68 -0
- data/sig/ibex/coverage.rbs +7 -0
- data/sig/ibex/error_messages/parser.rbs +58 -0
- data/sig/ibex/error_messages/parser_v2.rbs +67 -0
- data/sig/ibex/error_messages/renderer.rbs +23 -0
- data/sig/ibex/error_messages/sentence_search.rbs +80 -0
- data/sig/ibex/error_messages/update.rbs +43 -0
- data/sig/ibex/error_messages.rbs +85 -0
- data/sig/ibex/frontend/ast.rbs +208 -19
- data/sig/ibex/frontend/bootstrap_parser.rbs +11 -0
- data/sig/ibex/frontend/diagnostic.rbs +53 -0
- data/sig/ibex/frontend/diagnostic_recovery.rbs +98 -0
- data/sig/ibex/frontend/dsl.rbs +33 -4
- data/sig/ibex/frontend/formatter.rbs +135 -0
- data/sig/ibex/frontend/generated_parser.rbs +218 -68
- data/sig/ibex/frontend/generated_parser_base.rbs +61 -10
- data/sig/ibex/frontend/generated_parser_includes.rbs +23 -0
- data/sig/ibex/frontend/generated_parser_metadata.rbs +23 -0
- data/sig/ibex/frontend/generated_parser_parameters.rbs +24 -0
- data/sig/ibex/frontend/generation.rbs +6 -0
- data/sig/ibex/frontend/lexer.rbs +49 -2
- data/sig/ibex/frontend/lexer_recovery.rbs +25 -0
- data/sig/ibex/frontend/parser/declarations.rbs +54 -0
- data/sig/ibex/frontend/parser/parameters.rbs +28 -0
- data/sig/ibex/frontend/parser/rules.rbs +3 -0
- data/sig/ibex/frontend/parser.rbs +66 -0
- data/sig/ibex/frontend/regenerator.rbs +11 -0
- data/sig/ibex/frontend/resolution.rbs +33 -0
- data/sig/ibex/frontend/resolver.rbs +91 -0
- data/sig/ibex/frontend/rule_documentation.rbs +42 -0
- data/sig/ibex/frontend/source_cursor.rbs +36 -3
- data/sig/ibex/frontend/source_document.rbs +119 -0
- data/sig/ibex/frontend/source_loader.rbs +66 -0
- data/sig/ibex/frontend/source_span.rbs +53 -0
- data/sig/ibex/frontend/token_adapter/declaration_document_state.rbs +21 -0
- data/sig/ibex/frontend/token_adapter/declaration_lexer_state.rbs +27 -0
- data/sig/ibex/frontend/token_adapter/declaration_state.rbs +73 -5
- data/sig/ibex/frontend/token_adapter/rule_state.rbs +20 -0
- data/sig/ibex/frontend/token_adapter.rbs +5 -2
- data/sig/ibex/frontend.rbs +1 -1
- data/sig/ibex/generation_input.rbs +37 -0
- data/sig/ibex/generation_manifest.rbs +67 -0
- data/sig/ibex/generation_transaction.rbs +82 -0
- data/sig/ibex/generation_transaction_recovery.rbs +36 -0
- data/sig/ibex/generation_transaction_validation.rbs +65 -0
- data/sig/ibex/grammar_tests.rbs +93 -0
- data/sig/ibex/ir/automaton_ir.rbs +8 -2
- data/sig/ibex/ir/grammar_ir.rbs +75 -15
- data/sig/ibex/ir/lexer_ir.rbs +57 -0
- data/sig/ibex/ir/migration.rbs +34 -0
- data/sig/ibex/ir/serialize.rbs +23 -6
- data/sig/ibex/ir/validator/automaton.rbs +109 -0
- data/sig/ibex/ir/validator/base.rbs +65 -0
- data/sig/ibex/ir/validator/grammar.rbs +184 -0
- data/sig/ibex/ir/validator/lexer.rbs +37 -0
- data/sig/ibex/ir/validator.rbs +16 -0
- data/sig/ibex/ir.rbs +38 -4
- data/sig/ibex/lalr/build_metrics.rbs +20 -0
- data/sig/ibex/lalr/builder.rbs +95 -15
- data/sig/ibex/lalr/conflict_search.rbs +7 -3
- data/sig/ibex/lalr/counterexample.rbs +5 -2
- data/sig/ibex/lalr/direct_lookaheads.rbs +86 -0
- data/sig/ibex/lalr/ielr_partition.rbs +59 -0
- data/sig/ibex/lalr/on_error_reductions.rbs +22 -0
- data/sig/ibex/lalr.rbs +6 -0
- data/sig/ibex/location.rbs +67 -0
- data/sig/ibex/lsp/document_handlers.rbs +24 -0
- data/sig/ibex/lsp/document_store.rbs +94 -0
- data/sig/ibex/lsp/document_store_diagnostics.rbs +20 -0
- data/sig/ibex/lsp/document_store_validation.rbs +26 -0
- data/sig/ibex/lsp/initialization_handlers.rbs +30 -0
- data/sig/ibex/lsp/navigation_handlers.rbs +30 -0
- data/sig/ibex/lsp/position_codec.rbs +40 -0
- data/sig/ibex/lsp/protocol_error.rbs +32 -0
- data/sig/ibex/lsp/request_handlers.rbs +26 -0
- data/sig/ibex/lsp/request_support.rbs +40 -0
- data/sig/ibex/lsp/server.rbs +55 -0
- data/sig/ibex/lsp/symbol_index.rbs +78 -0
- data/sig/ibex/lsp/symbol_index_builder.rbs +87 -0
- data/sig/ibex/lsp/symbol_index_precedence_references.rbs +18 -0
- data/sig/ibex/lsp/symbol_index_source_queries.rbs +26 -0
- data/sig/ibex/lsp/symbol_occurrence.rbs +25 -0
- data/sig/ibex/lsp/transport.rbs +35 -0
- data/sig/ibex/lsp/workspace.rbs +41 -0
- data/sig/ibex/lsp/workspace_analyzer.rbs +69 -0
- data/sig/ibex/lsp.rbs +7 -0
- data/sig/ibex/normalize/declarations.rbs +31 -0
- data/sig/ibex/normalize/diagnostics.rbs +9 -0
- data/sig/ibex/normalize/expander.rbs +20 -14
- data/sig/ibex/normalize/expression.rbs +10 -10
- data/sig/ibex/normalize/inline_expansion.rbs +120 -0
- data/sig/ibex/normalize/inline_validation.rbs +40 -0
- data/sig/ibex/normalize/lexer.rbs +37 -0
- data/sig/ibex/normalize/named_references.rbs +20 -0
- data/sig/ibex/normalize/nodes.rbs +14 -0
- data/sig/ibex/normalize/parameter_ebnf_lowering.rbs +32 -0
- data/sig/ibex/normalize/parameter_lowering.rbs +35 -0
- data/sig/ibex/normalize/parameter_substitution.rbs +42 -0
- data/sig/ibex/normalize/parameter_validation.rbs +44 -0
- data/sig/ibex/normalize/parameters.rbs +46 -0
- data/sig/ibex/normalize/recovery_declarations.rbs +24 -0
- data/sig/ibex/normalize.rbs +123 -18
- data/sig/ibex/racc_migration/checker.rbs +36 -0
- data/sig/ibex/racc_migration/harness.rbs +16 -0
- data/sig/ibex/racc_migration/report.rbs +53 -0
- data/sig/ibex/racc_migration.rbs +8 -0
- data/sig/ibex/rake_task.rbs +51 -0
- data/sig/ibex/samples.rbs +51 -0
- data/sig/ibex/table_simulation/result.rbs +32 -0
- data/sig/ibex/table_simulation/simulator.rbs +98 -0
- data/sig/ibex/table_simulation/step.rbs +40 -0
- data/sig/ibex/table_simulation/text.rbs +14 -0
- data/sig/ibex/table_simulation.rbs +7 -0
- data/sig/ibex/tables.rbs +0 -26
- data/sig/ibex/watch/runner.rbs +48 -0
- data/sig/ibex/watch/source_snapshot.rbs +42 -0
- data/sig/ibex/watch.rbs +7 -0
- data/sig/ibex.rbs +2 -0
- metadata +301 -16
- data/.rubocop.yml +0 -43
- data/CHANGELOG.md +0 -30
- data/Rakefile +0 -25
- data/Steepfile +0 -10
- data/docs/compat-notes.md +0 -37
- data/docs/lexer-coverage.md +0 -14
- data/docs/phase10-extensions.md +0 -27
- data/gemfiles/Gemfile +0 -7
- data/gemfiles/Gemfile.lock +0 -98
- data/lib/ibex/frontend/grammar.y +0 -156
- data/lib/ibex/runtime/parser.rb +0 -360
- data/lib/ibex/runtime.rb +0 -8
- data/sig/ibex/runtime/parser.rbs +0 -167
- data/sig/ibex/runtime.rbs +0 -6
data/lib/ibex/normalize.rb
CHANGED
|
@@ -1,21 +1,52 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "normalize/declarations"
|
|
4
|
+
require_relative "normalize/lexer"
|
|
5
|
+
require_relative "normalize/recovery_declarations"
|
|
4
6
|
require_relative "normalize/expression"
|
|
7
|
+
require_relative "normalize/parameter_validation"
|
|
8
|
+
require_relative "normalize/inline_validation"
|
|
9
|
+
require_relative "normalize/parameter_substitution"
|
|
10
|
+
require_relative "normalize/parameter_ebnf_lowering"
|
|
11
|
+
require_relative "normalize/parameter_lowering"
|
|
12
|
+
require_relative "normalize/parameters"
|
|
13
|
+
require_relative "normalize/named_references"
|
|
14
|
+
require_relative "normalize/nodes"
|
|
5
15
|
require_relative "normalize/expander"
|
|
16
|
+
require_relative "normalize/inline_expansion"
|
|
6
17
|
require_relative "normalize/diagnostics"
|
|
18
|
+
require_relative "frontend/resolution"
|
|
7
19
|
|
|
8
20
|
module Ibex
|
|
9
21
|
# Converts a frontend AST into immutable Grammar IR.
|
|
10
22
|
class Normalizer
|
|
11
23
|
include NormalizeDeclarations
|
|
24
|
+
include NormalizeLexer
|
|
25
|
+
include NormalizeRecoveryDeclarations
|
|
26
|
+
include NormalizeParameterValidation
|
|
27
|
+
include NormalizeInlineValidation
|
|
28
|
+
include NormalizeParameterSubstitution
|
|
29
|
+
include NormalizeParameterEbnfLowering
|
|
30
|
+
include NormalizeParameterLowering
|
|
31
|
+
include NormalizeParameters
|
|
32
|
+
include NormalizeNamedReferences
|
|
33
|
+
include NormalizeNodes
|
|
12
34
|
include NormalizeExpander
|
|
35
|
+
include NormalizeInlineExpansion
|
|
13
36
|
include NormalizeDiagnostics
|
|
14
37
|
|
|
15
38
|
RESERVED_NAMES = %w[result val _values].freeze #: Array[String]
|
|
39
|
+
RUBY_KEYWORDS = %w[
|
|
40
|
+
__ENCODING__ __FILE__ __LINE__ alias and begin break case class def defined do else elsif end ensure false for if
|
|
41
|
+
in module next nil not or redo rescue retry return self super then true undef unless until when while yield
|
|
42
|
+
].freeze #: Array[String]
|
|
43
|
+
DEFAULT_MAX_PARAMETER_SPECIALIZATIONS = 1_000
|
|
44
|
+
DEFAULT_MAX_INLINE_EXPANSIONS = 10_000
|
|
16
45
|
|
|
17
46
|
# @rbs @ast: Frontend::AST::Root
|
|
18
|
-
# @rbs @
|
|
47
|
+
# @rbs @resolution: Frontend::Resolution?
|
|
48
|
+
# @rbs @current_include_chain: Array[IR::source_provenance]
|
|
49
|
+
# @rbs @mode: IR::grammar_mode
|
|
19
50
|
# @rbs @symbols: Array[IR::GrammarSymbol]
|
|
20
51
|
# @rbs @symbols_by_name: Hash[String, IR::GrammarSymbol]
|
|
21
52
|
# @rbs @productions: Array[IR::Production]
|
|
@@ -24,40 +55,115 @@ module Ibex
|
|
|
24
55
|
# @rbs @declared_tokens: Hash[String, IR::location]
|
|
25
56
|
# @rbs @precedence: Hash[String, IR::precedence]
|
|
26
57
|
# @rbs @precedence_locations: Hash[String, IR::location]
|
|
58
|
+
# @rbs @display_names: Hash[String, String]
|
|
59
|
+
# @rbs @display_name_locations: Hash[String, IR::location]
|
|
60
|
+
# @rbs @semantic_types: Hash[String, String]
|
|
61
|
+
# @rbs @semantic_type_locations: Hash[String, IR::location]
|
|
27
62
|
# @rbs @options: IR::grammar_options
|
|
28
63
|
# @rbs @expected_conflicts: Integer
|
|
64
|
+
# @rbs @expected_rr_conflicts: Integer?
|
|
29
65
|
# @rbs @conversions: Hash[String, String]
|
|
30
|
-
# @rbs @
|
|
66
|
+
# @rbs @parser_parameters: Array[IR::parser_parameter]
|
|
67
|
+
# @rbs @value_printers: Hash[String, IR::value_printer]
|
|
68
|
+
# @rbs @recovery_sync_tokens: Array[String]
|
|
69
|
+
# @rbs @on_error_reduce_groups: Array[Array[String]]
|
|
70
|
+
# @rbs @grammar_tests: Array[IR::grammar_test]
|
|
71
|
+
# @rbs @lexer_declaration: Frontend::AST::Lexer?
|
|
72
|
+
# @rbs @explicit_starts: Array[String]?
|
|
73
|
+
# @rbs @start_names: Array[String]
|
|
31
74
|
# @rbs @start_name: String
|
|
32
75
|
# @rbs @start_location: Frontend::Location?
|
|
76
|
+
# @rbs @parameter_templates: Hash[String, Array[Frontend::AST::Rule]]
|
|
77
|
+
# @rbs @parameter_formals: Hash[String, Array[String]]
|
|
78
|
+
# @rbs @parameter_specializations: Hash[[String, Array[String]], String]
|
|
79
|
+
# @rbs @parameter_worklist: Array[Hash[Symbol, untyped]]
|
|
80
|
+
# @rbs @parameter_worklist_active: bool
|
|
81
|
+
# @rbs @max_parameter_specializations: Integer
|
|
82
|
+
# @rbs @current_parameter_expansion: IR::parameter_expansion?
|
|
83
|
+
# @rbs @rule_documentation: Hash[String, String]
|
|
84
|
+
# @rbs @inline_rule_names: Set[String]
|
|
85
|
+
# @rbs @inline_symbol_ids: Set[Integer]
|
|
86
|
+
# @rbs @inline_rule_by_symbol: Hash[Integer, String]
|
|
87
|
+
# @rbs @max_inline_expansions: Integer
|
|
88
|
+
# @rbs @inline_expansion_count: Integer
|
|
89
|
+
# @rbs @node_shapes: Hash[String, Array[String]]
|
|
90
|
+
|
|
91
|
+
# @rbs (Frontend::AST::Root | Frontend::AST::Fragment | Frontend::Resolution input,
|
|
92
|
+
# ?mode: Symbol | String, ?max_parameter_specializations: Integer,
|
|
93
|
+
# ?max_inline_expansions: Integer) -> void
|
|
94
|
+
def initialize(input, mode: :default, max_parameter_specializations: DEFAULT_MAX_PARAMETER_SPECIALIZATIONS,
|
|
95
|
+
max_inline_expansions: DEFAULT_MAX_INLINE_EXPANSIONS)
|
|
96
|
+
validate_positive_limit!(:max_parameter_specializations, max_parameter_specializations)
|
|
97
|
+
validate_positive_limit!(:max_inline_expansions, max_inline_expansions)
|
|
98
|
+
normalized_mode = mode.to_sym
|
|
99
|
+
raise ArgumentError, "mode must be :default or :extended" unless %i[default extended].include?(normalized_mode)
|
|
100
|
+
|
|
101
|
+
@resolution = input if input.is_a?(Frontend::Resolution)
|
|
102
|
+
ast = input.is_a?(Frontend::Resolution) ? input.root : input
|
|
103
|
+
normalized_mode = :extended if ast.is_a?(Frontend::AST::Root) && ast.extended
|
|
104
|
+
fail_at(ast.loc, "fragments must be resolved before normalization") if ast.is_a?(Frontend::AST::Fragment)
|
|
105
|
+
unresolved = ast.declarations.find { |declaration| declaration.is_a?(Frontend::AST::Include) }
|
|
106
|
+
fail_at(unresolved.loc, "includes must be resolved before normalization") if unresolved
|
|
33
107
|
|
|
34
|
-
# @rbs (Frontend::AST::Root ast, ?mode: Symbol | String) -> void
|
|
35
|
-
def initialize(ast, mode: :racc)
|
|
36
108
|
@ast = ast
|
|
37
|
-
@mode =
|
|
109
|
+
@mode = normalized_mode #: IR::grammar_mode
|
|
38
110
|
@symbols = [] #: Array[IR::GrammarSymbol]
|
|
39
111
|
@symbols_by_name = {} #: Hash[String, IR::GrammarSymbol]
|
|
40
112
|
@productions = [] #: Array[IR::Production]
|
|
41
113
|
@warnings = [] #: Array[IR::grammar_warning]
|
|
42
114
|
@helper_sequence = 0
|
|
115
|
+
@current_include_chain = [] #: Array[IR::source_provenance]
|
|
116
|
+
@parameter_specializations = {} #: Hash[[String, Array[String]], String]
|
|
117
|
+
@parameter_worklist = [] #: Array[Hash[Symbol, untyped]]
|
|
118
|
+
@parameter_worklist_active = false
|
|
119
|
+
@max_parameter_specializations = max_parameter_specializations
|
|
120
|
+
@max_inline_expansions = max_inline_expansions
|
|
121
|
+
@inline_expansion_count = 0
|
|
122
|
+
@inline_symbol_ids = Set.new #: Set[Integer]
|
|
123
|
+
@inline_rule_by_symbol = {} #: Hash[Integer, String]
|
|
43
124
|
end
|
|
44
125
|
|
|
45
126
|
# @rbs () -> IR::Grammar
|
|
46
127
|
def normalize
|
|
47
128
|
read_declarations
|
|
129
|
+
gather_parameter_templates
|
|
130
|
+
gather_inline_rules
|
|
48
131
|
intern_reserved_symbols
|
|
49
132
|
intern_declared_terminals
|
|
50
133
|
intern_user_nonterminals
|
|
51
134
|
normalize_user_productions
|
|
135
|
+
expand_inline_rules
|
|
136
|
+
validate_value_printers
|
|
137
|
+
validate_recovery_declarations
|
|
52
138
|
validate_grammar
|
|
53
139
|
IR::Grammar.new(class_name: @ast.class_name, superclass: @ast.superclass, start: @start_name,
|
|
54
140
|
expect: @expected_conflicts, options: @options, symbols: @symbols,
|
|
141
|
+
mode: @mode, starts: @start_names,
|
|
142
|
+
expect_rr: @expected_rr_conflicts,
|
|
143
|
+
parser_parameters: @parser_parameters,
|
|
144
|
+
value_printers: @value_printers.values,
|
|
145
|
+
grammar_tests: @grammar_tests,
|
|
146
|
+
lexer: normalize_lexer,
|
|
147
|
+
recovery: {
|
|
148
|
+
sync_tokens: @recovery_sync_tokens,
|
|
149
|
+
on_error_reduce: @on_error_reduce_groups
|
|
150
|
+
},
|
|
55
151
|
productions: @productions, user_code: normalized_user_code,
|
|
56
|
-
conversions: @conversions, warnings: @warnings, user_code_chunks: normalized_user_code_chunks
|
|
152
|
+
conversions: @conversions, warnings: @warnings, user_code_chunks: normalized_user_code_chunks,
|
|
153
|
+
source_provenance: {
|
|
154
|
+
file: @ast.loc.file, root: @resolution&.root_directory, byte_span: nil
|
|
155
|
+
})
|
|
57
156
|
end
|
|
58
157
|
|
|
59
158
|
private
|
|
60
159
|
|
|
160
|
+
# @rbs (Symbol name, untyped value) -> void
|
|
161
|
+
def validate_positive_limit!(name, value)
|
|
162
|
+
return if value.is_a?(Integer) && value.positive?
|
|
163
|
+
|
|
164
|
+
raise ArgumentError, "#{name} must be a positive Integer"
|
|
165
|
+
end
|
|
166
|
+
|
|
61
167
|
# @rbs () -> void
|
|
62
168
|
def intern_reserved_symbols
|
|
63
169
|
intern("$eof", :terminal, reserved: true)
|
|
@@ -72,16 +178,62 @@ module Ibex
|
|
|
72
178
|
|
|
73
179
|
# @rbs () -> void
|
|
74
180
|
def intern_user_nonterminals
|
|
75
|
-
@ast.rules.
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
181
|
+
@ast.rules.reject { |rule| parameterized_rule?(rule) }.each do |rule|
|
|
182
|
+
intern_user_nonterminal(rule)
|
|
183
|
+
end
|
|
184
|
+
@start_names = normalized_start_names
|
|
185
|
+
fail_at(@ast.loc, "grammar has no start rule") if @start_names.empty?
|
|
186
|
+
@start_name = @start_names.fetch(0)
|
|
187
|
+
@start_names.each do |name|
|
|
188
|
+
next if symbol(name)&.nonterminal?
|
|
79
189
|
|
|
80
|
-
|
|
190
|
+
fail_at(@start_location || @ast.loc, "undefined start symbol #{name}")
|
|
191
|
+
end
|
|
81
192
|
end
|
|
82
193
|
|
|
83
|
-
# @rbs (
|
|
84
|
-
def
|
|
194
|
+
# @rbs () -> Array[String]
|
|
195
|
+
def normalized_start_names
|
|
196
|
+
explicit = @explicit_starts
|
|
197
|
+
return explicit if explicit
|
|
198
|
+
|
|
199
|
+
candidate = @ast.rules.find { |rule| start_rule_candidate?(rule) }&.lhs
|
|
200
|
+
candidate ? [candidate] : []
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
# @rbs (Frontend::AST::Rule rule) -> bool
|
|
204
|
+
def start_rule_candidate?(rule)
|
|
205
|
+
!parameterized_rule?(rule) && !rule.inline
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
# @rbs (Frontend::AST::Rule rule) -> void
|
|
209
|
+
def intern_user_nonterminal(rule)
|
|
210
|
+
definition = intern(
|
|
211
|
+
rule.lhs, :nonterminal, location: rule.loc.to_h, documentation: @rule_documentation[rule.lhs]
|
|
212
|
+
)
|
|
213
|
+
return unless rule.inline
|
|
214
|
+
|
|
215
|
+
@inline_symbol_ids << definition.id
|
|
216
|
+
@inline_rule_by_symbol[definition.id] = rule.lhs
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
# @rbs () -> Hash[String, String]
|
|
220
|
+
def validated_rule_documentation
|
|
221
|
+
documentation = {} #: Hash[String, String]
|
|
222
|
+
@ast.rules.each do |rule|
|
|
223
|
+
text = rule.documentation
|
|
224
|
+
next unless text
|
|
225
|
+
|
|
226
|
+
existing = documentation[rule.lhs]
|
|
227
|
+
fail_at(rule.loc, "conflicting documentation for rule #{rule.lhs}") if existing && existing != text
|
|
228
|
+
|
|
229
|
+
documentation[rule.lhs] ||= text
|
|
230
|
+
end
|
|
231
|
+
documentation
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
# @rbs (String name, Symbol kind, ?reserved: bool, ?location: IR::location?,
|
|
235
|
+
# ?documentation: String?, ?metadata_name: String?) -> IR::GrammarSymbol
|
|
236
|
+
def intern(name, kind, reserved: false, location: nil, documentation: nil, metadata_name: nil)
|
|
85
237
|
existing = symbol(name)
|
|
86
238
|
if existing
|
|
87
239
|
fail_hash(location, "symbol #{name} is both terminal and nonterminal") if existing.kind != kind
|
|
@@ -89,8 +241,11 @@ module Ibex
|
|
|
89
241
|
end
|
|
90
242
|
|
|
91
243
|
precedence = @precedence[name]
|
|
244
|
+
metadata_key = metadata_name || name
|
|
92
245
|
definition = IR::GrammarSymbol.new(id: @symbols.length, name: name, kind: kind, reserved: reserved,
|
|
93
|
-
precedence: precedence, location: location
|
|
246
|
+
precedence: precedence, location: location,
|
|
247
|
+
display_name: @display_names[metadata_key],
|
|
248
|
+
semantic_type: @semantic_types[metadata_key], documentation: documentation)
|
|
94
249
|
@symbols << definition
|
|
95
250
|
@symbols_by_name[name] = definition
|
|
96
251
|
definition
|
|
@@ -110,6 +265,9 @@ module Ibex
|
|
|
110
265
|
def symbol_for_reference(reference)
|
|
111
266
|
existing = symbol(reference.name)
|
|
112
267
|
return existing if existing
|
|
268
|
+
|
|
269
|
+
fail_at(reference.loc, "parameterized rule #{reference.name} requires arguments") if
|
|
270
|
+
parameter_template?(reference.name)
|
|
113
271
|
return undefined_nonterminal(reference) if nonterminal_name?(reference.name)
|
|
114
272
|
|
|
115
273
|
warn_undeclared_terminal(reference)
|
|
@@ -162,5 +320,12 @@ module Ibex
|
|
|
162
320
|
rendered = location ? "#{location[:file]}:#{location[:line]}:#{location[:column]}" : "(grammar):1:1"
|
|
163
321
|
raise Ibex::Error, "#{rendered}: #{message}"
|
|
164
322
|
end
|
|
323
|
+
|
|
324
|
+
# @rbs () -> IR::production_expansion?
|
|
325
|
+
def resolved_expansion
|
|
326
|
+
return unless @resolution || @current_parameter_expansion
|
|
327
|
+
|
|
328
|
+
{ parameter: @current_parameter_expansion, inline: nil, include_chain: @current_include_chain }
|
|
329
|
+
end
|
|
165
330
|
end
|
|
166
331
|
end
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# rbs_inline: enabled
|
|
3
|
+
|
|
4
|
+
module Ibex
|
|
5
|
+
module RaccMigration
|
|
6
|
+
# Checks syntax, normalization, superclass, and runtime-coupling hazards.
|
|
7
|
+
# Semantic action code is treated as opaque data and never executed.
|
|
8
|
+
class Checker
|
|
9
|
+
RACC_REQUIRE = %r{(?:require|require_relative)\s*\(?\s*["']racc(?:/parser)?["']} #: Regexp
|
|
10
|
+
RACC_CONSTANT = /\bRacc::[A-Za-z_]\w*/ #: Regexp
|
|
11
|
+
|
|
12
|
+
# @rbs (String source, file: String) -> Report
|
|
13
|
+
def check(source, file:)
|
|
14
|
+
result = Frontend::Parser.new(source, file: file, mode: :default).parse_with_diagnostics
|
|
15
|
+
findings = result.diagnostics.map { |diagnostic| syntax_finding(diagnostic) }
|
|
16
|
+
ast = result.ast
|
|
17
|
+
return Report.new(file: file, class_name: nil, findings: findings) unless ast
|
|
18
|
+
|
|
19
|
+
findings.concat(superclass_findings(ast))
|
|
20
|
+
findings.concat(user_code_findings(ast))
|
|
21
|
+
findings.concat(normalization_findings(ast))
|
|
22
|
+
Report.new(file: file, class_name: ast.class_name, findings: sort_findings(findings))
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
private
|
|
26
|
+
|
|
27
|
+
# @rbs (Frontend::Diagnostic diagnostic) -> Finding
|
|
28
|
+
def syntax_finding(diagnostic)
|
|
29
|
+
Finding.new(
|
|
30
|
+
code: "racc.syntax",
|
|
31
|
+
severity: :error,
|
|
32
|
+
message: diagnostic.message,
|
|
33
|
+
location: diagnostic.location,
|
|
34
|
+
suggestion: "keep the grammar in default mode, or adopt the reported extension explicitly " \
|
|
35
|
+
"with `pragma extended`"
|
|
36
|
+
)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# @rbs (Frontend::AST::Root ast) -> Array[Finding]
|
|
40
|
+
def superclass_findings(ast)
|
|
41
|
+
return [] unless ast.superclass&.start_with?("Racc::")
|
|
42
|
+
|
|
43
|
+
[
|
|
44
|
+
Finding.new(
|
|
45
|
+
code: "racc.runtime_superclass",
|
|
46
|
+
severity: :error,
|
|
47
|
+
message: "generated class inherits #{ast.superclass}",
|
|
48
|
+
location: ast.loc,
|
|
49
|
+
suggestion: "remove the explicit Racc superclass or generate with `--superclass=Ibex::Runtime::Parser`"
|
|
50
|
+
)
|
|
51
|
+
]
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# @rbs (Frontend::AST::Root ast) -> Array[Finding]
|
|
55
|
+
def user_code_findings(ast)
|
|
56
|
+
ast.user_code.values.flatten.flat_map do |chunk|
|
|
57
|
+
findings = [] #: Array[Finding]
|
|
58
|
+
if chunk.code.match?(RACC_REQUIRE)
|
|
59
|
+
findings << Finding.new(
|
|
60
|
+
code: "racc.runtime_require",
|
|
61
|
+
severity: :warning,
|
|
62
|
+
message: "user code requires the racc runtime",
|
|
63
|
+
location: chunk.loc,
|
|
64
|
+
suggestion: "remove this require after switching generated parsers to the Ibex runtime"
|
|
65
|
+
)
|
|
66
|
+
end
|
|
67
|
+
if chunk.code.match?(RACC_CONSTANT)
|
|
68
|
+
findings << Finding.new(
|
|
69
|
+
code: "racc.runtime_constant",
|
|
70
|
+
severity: :warning,
|
|
71
|
+
message: "user code references a Racc runtime constant",
|
|
72
|
+
location: chunk.loc,
|
|
73
|
+
suggestion: "replace the runtime-specific constant or keep an explicit compatibility adapter"
|
|
74
|
+
)
|
|
75
|
+
end
|
|
76
|
+
findings
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# @rbs (Frontend::AST::Root ast) -> Array[Finding]
|
|
81
|
+
def normalization_findings(ast)
|
|
82
|
+
Normalizer.new(ast, mode: :default).normalize
|
|
83
|
+
[]
|
|
84
|
+
rescue Ibex::Error => e
|
|
85
|
+
location, message = error_location(e.message, ast.loc)
|
|
86
|
+
[
|
|
87
|
+
Finding.new(
|
|
88
|
+
code: "racc.normalization",
|
|
89
|
+
severity: :error,
|
|
90
|
+
message: message,
|
|
91
|
+
location: location,
|
|
92
|
+
suggestion: "apply the diagnostic before generating either parser"
|
|
93
|
+
)
|
|
94
|
+
]
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# @rbs (String message, Frontend::Location fallback) -> [Frontend::Location, String]
|
|
98
|
+
def error_location(message, fallback)
|
|
99
|
+
match = message.match(/\A(.+):(\d+):(\d+): (.*)\z/m)
|
|
100
|
+
return [fallback, message] unless match
|
|
101
|
+
|
|
102
|
+
[
|
|
103
|
+
Frontend::Location.new(
|
|
104
|
+
file: match[1].to_s,
|
|
105
|
+
line: Integer(match[2].to_s, 10),
|
|
106
|
+
column: Integer(match[3].to_s, 10)
|
|
107
|
+
),
|
|
108
|
+
match[4].to_s
|
|
109
|
+
]
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# @rbs (Array[Finding] findings) -> Array[Finding]
|
|
113
|
+
def sort_findings(findings)
|
|
114
|
+
rank = { error: 0, warning: 1, info: 2 }
|
|
115
|
+
findings.sort_by do |finding|
|
|
116
|
+
location = finding.location
|
|
117
|
+
[location&.file || "", location&.line || 0, location&.column || 0, rank.fetch(finding.severity), finding.code]
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
end
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# rbs_inline: enabled
|
|
3
|
+
|
|
4
|
+
module Ibex
|
|
5
|
+
module RaccMigration
|
|
6
|
+
# Emits, but never executes, a subprocess-separated differential harness.
|
|
7
|
+
class Harness
|
|
8
|
+
CLASS_NAME = /\A[A-Z]\w*(?:::[A-Z]\w*)*\z/ #: Regexp
|
|
9
|
+
|
|
10
|
+
# @rbs (String class_name) -> String
|
|
11
|
+
def self.generate(class_name)
|
|
12
|
+
raise ArgumentError, "invalid parser class name #{class_name.inspect}" unless class_name.match?(CLASS_NAME)
|
|
13
|
+
|
|
14
|
+
template.sub("__PARSER_CLASS_NAME__", class_name.dump)
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# @rbs () -> String
|
|
18
|
+
def self.template
|
|
19
|
+
<<~'RUBY'
|
|
20
|
+
#!/usr/bin/env ruby
|
|
21
|
+
# frozen_string_literal: true
|
|
22
|
+
|
|
23
|
+
# Generated migration differential harness.
|
|
24
|
+
# Running this file executes both generated parsers and all grammar
|
|
25
|
+
# user code in child Ruby processes. Review the grammar and CASES,
|
|
26
|
+
# then run this only in an isolation boundary you trust.
|
|
27
|
+
|
|
28
|
+
require "rbconfig"
|
|
29
|
+
require "tempfile"
|
|
30
|
+
require "tmpdir"
|
|
31
|
+
|
|
32
|
+
PARSER_CLASS = __PARSER_CLASS_NAME__
|
|
33
|
+
TIMEOUT_SECONDS = 15
|
|
34
|
+
MAX_OUTPUT_BYTES = 1_048_576
|
|
35
|
+
CASES = [
|
|
36
|
+
# { name: "example", tokens: [[:TOKEN, "value"], ["+", nil]] }
|
|
37
|
+
].freeze
|
|
38
|
+
|
|
39
|
+
CHILD_RUNNER = <<~'CHILD'
|
|
40
|
+
begin
|
|
41
|
+
load ARGV.fetch(0)
|
|
42
|
+
parser_class = ARGV.fetch(1).split("::").inject(Object) { |scope, name| scope.const_get(name, false) }
|
|
43
|
+
tokens = Marshal.load(ARGV.fetch(2).unpack1("m0"))
|
|
44
|
+
parser = parser_class.new
|
|
45
|
+
parser.define_singleton_method(:next_token) { tokens.shift }
|
|
46
|
+
result = parser.do_parse
|
|
47
|
+
puts "IBEX_HARNESS_RESULT=#{[Marshal.dump([:ok, result])].pack("m0")}"
|
|
48
|
+
rescue StandardError => error
|
|
49
|
+
payload = [:error, error.class.name, error.message]
|
|
50
|
+
puts "IBEX_HARNESS_RESULT=#{[Marshal.dump(payload)].pack("m0")}"
|
|
51
|
+
end
|
|
52
|
+
CHILD
|
|
53
|
+
|
|
54
|
+
def terminate(pid)
|
|
55
|
+
Process.kill("TERM", pid)
|
|
56
|
+
rescue Errno::ESRCH
|
|
57
|
+
nil
|
|
58
|
+
ensure
|
|
59
|
+
deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 0.5
|
|
60
|
+
while Process.clock_gettime(Process::CLOCK_MONOTONIC) < deadline
|
|
61
|
+
waited = Process.waitpid2(pid, Process::WNOHANG)
|
|
62
|
+
return waited.last if waited
|
|
63
|
+
|
|
64
|
+
sleep 0.01
|
|
65
|
+
end
|
|
66
|
+
Process.kill("KILL", pid)
|
|
67
|
+
Process.waitpid2(pid).last
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def run_command(*command)
|
|
71
|
+
Tempfile.create("migration-stdout") do |stdout|
|
|
72
|
+
Tempfile.create("migration-stderr") do |stderr|
|
|
73
|
+
pid = Process.spawn(*command, out: stdout, err: stderr)
|
|
74
|
+
deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + TIMEOUT_SECONDS
|
|
75
|
+
status = nil
|
|
76
|
+
until status
|
|
77
|
+
waited = Process.waitpid2(pid, Process::WNOHANG)
|
|
78
|
+
status = waited.last if waited
|
|
79
|
+
break if status
|
|
80
|
+
if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
|
|
81
|
+
status = terminate(pid)
|
|
82
|
+
stdout.rewind
|
|
83
|
+
stderr.rewind
|
|
84
|
+
return [
|
|
85
|
+
stdout.read(MAX_OUTPUT_BYTES) || "", stderr.read(MAX_OUTPUT_BYTES) || "", status, true, false
|
|
86
|
+
]
|
|
87
|
+
end
|
|
88
|
+
if stdout.size > MAX_OUTPUT_BYTES || stderr.size > MAX_OUTPUT_BYTES
|
|
89
|
+
status = terminate(pid)
|
|
90
|
+
stdout.rewind
|
|
91
|
+
stderr.rewind
|
|
92
|
+
return [
|
|
93
|
+
stdout.read(MAX_OUTPUT_BYTES) || "", stderr.read(MAX_OUTPUT_BYTES) || "", status, false, true
|
|
94
|
+
]
|
|
95
|
+
end
|
|
96
|
+
sleep 0.01
|
|
97
|
+
end
|
|
98
|
+
stdout.rewind
|
|
99
|
+
stderr.rewind
|
|
100
|
+
[
|
|
101
|
+
stdout.read(MAX_OUTPUT_BYTES) || "", stderr.read(MAX_OUTPUT_BYTES) || "", status, false, false
|
|
102
|
+
]
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def compile!(label, *command)
|
|
108
|
+
stdout, stderr, status, timed_out, output_limited = run_command(*command)
|
|
109
|
+
abort "#{label} generation timed out" if timed_out
|
|
110
|
+
abort "#{label} generation exceeded #{MAX_OUTPUT_BYTES} output bytes" if output_limited
|
|
111
|
+
return if status.success?
|
|
112
|
+
|
|
113
|
+
abort "#{label} generation failed\n#{stdout}#{stderr}"
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def observe(parser_path, tokens)
|
|
117
|
+
payload = [Marshal.dump(tokens)].pack("m0")
|
|
118
|
+
stdout, stderr, status, timed_out, output_limited = run_command(
|
|
119
|
+
RbConfig.ruby, "-e", CHILD_RUNNER, parser_path, PARSER_CLASS, payload
|
|
120
|
+
)
|
|
121
|
+
return { timeout: true } if timed_out
|
|
122
|
+
return { output_limit: MAX_OUTPUT_BYTES } if output_limited
|
|
123
|
+
|
|
124
|
+
lines = stdout.lines
|
|
125
|
+
marker = lines.reverse.find { |line| line.start_with?("IBEX_HARNESS_RESULT=") }
|
|
126
|
+
unless status.success? && marker
|
|
127
|
+
return {
|
|
128
|
+
infrastructure_error: File.basename(parser_path),
|
|
129
|
+
status: status.exitstatus,
|
|
130
|
+
stdout: stdout,
|
|
131
|
+
stderr: stderr
|
|
132
|
+
}
|
|
133
|
+
end
|
|
134
|
+
outcome = marker && Marshal.load(marker.split("=", 2).last.strip.unpack1("m0"))
|
|
135
|
+
{
|
|
136
|
+
status: status.exitstatus,
|
|
137
|
+
outcome: outcome,
|
|
138
|
+
stdout: lines.reject { |line| line.equal?(marker) }.join,
|
|
139
|
+
stderr: stderr
|
|
140
|
+
}
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
abort "add at least one reviewed entry to CASES before running the harness" if CASES.empty?
|
|
144
|
+
grammar = File.expand_path(ARGV.fetch(0))
|
|
145
|
+
abort "grammar file does not exist: #{grammar}" unless File.file?(grammar)
|
|
146
|
+
racc = ENV.fetch("RACC", "racc")
|
|
147
|
+
ibex = ENV.fetch("IBEX", "ibex")
|
|
148
|
+
|
|
149
|
+
failures = 0
|
|
150
|
+
Dir.mktmpdir("ibex-migration-harness-") do |directory|
|
|
151
|
+
racc_parser = File.join(directory, "racc_parser.rb")
|
|
152
|
+
ibex_parser = File.join(directory, "ibex_parser.rb")
|
|
153
|
+
compile!("racc", racc, "-o", racc_parser, grammar)
|
|
154
|
+
compile!("ibex", ibex, "-E", "-o", ibex_parser, grammar)
|
|
155
|
+
|
|
156
|
+
CASES.each do |test_case|
|
|
157
|
+
name = test_case.fetch(:name)
|
|
158
|
+
tokens = test_case.fetch(:tokens)
|
|
159
|
+
expected = observe(racc_parser, tokens)
|
|
160
|
+
actual = observe(ibex_parser, tokens)
|
|
161
|
+
if expected == actual
|
|
162
|
+
puts "ok #{name}"
|
|
163
|
+
else
|
|
164
|
+
failures += 1
|
|
165
|
+
warn "difference #{name}"
|
|
166
|
+
warn " racc: #{expected.inspect}"
|
|
167
|
+
warn " ibex: #{actual.inspect}"
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
end
|
|
171
|
+
exit(failures.zero? ? 0 : 1)
|
|
172
|
+
RUBY
|
|
173
|
+
end
|
|
174
|
+
private_class_method :template
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
end
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# rbs_inline: enabled
|
|
3
|
+
|
|
4
|
+
require "json"
|
|
5
|
+
|
|
6
|
+
module Ibex
|
|
7
|
+
module RaccMigration
|
|
8
|
+
# One immutable migration compatibility observation.
|
|
9
|
+
class Finding
|
|
10
|
+
SEVERITIES = %i[error warning info].freeze #: Array[Symbol]
|
|
11
|
+
|
|
12
|
+
attr_reader :code #: String
|
|
13
|
+
attr_reader :severity #: Symbol
|
|
14
|
+
attr_reader :message #: String
|
|
15
|
+
attr_reader :location #: Frontend::Location?
|
|
16
|
+
attr_reader :suggestion #: String
|
|
17
|
+
|
|
18
|
+
# @rbs (code: String, severity: Symbol, message: String, location: Frontend::Location?,
|
|
19
|
+
# suggestion: String) -> void
|
|
20
|
+
def initialize(code:, severity:, message:, location:, suggestion:)
|
|
21
|
+
unless SEVERITIES.include?(severity)
|
|
22
|
+
raise ArgumentError, "unknown migration finding severity #{severity.inspect}"
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
@code = code.dup.freeze
|
|
26
|
+
@severity = severity
|
|
27
|
+
@message = message.dup.freeze
|
|
28
|
+
@location = location
|
|
29
|
+
@suggestion = suggestion.dup.freeze
|
|
30
|
+
freeze
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# @rbs () -> Hash[String, untyped]
|
|
34
|
+
def to_h
|
|
35
|
+
{
|
|
36
|
+
"code" => @code,
|
|
37
|
+
"severity" => @severity.to_s,
|
|
38
|
+
"message" => @message,
|
|
39
|
+
"location" => @location&.to_h&.transform_keys(&:to_s),
|
|
40
|
+
"suggestion" => @suggestion
|
|
41
|
+
}
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Versioned result of checking one grammar without executing its code.
|
|
46
|
+
class Report
|
|
47
|
+
SCHEMA_VERSION = 1 #: Integer
|
|
48
|
+
|
|
49
|
+
attr_reader :file #: String
|
|
50
|
+
attr_reader :class_name #: String?
|
|
51
|
+
attr_reader :findings #: Array[Finding]
|
|
52
|
+
|
|
53
|
+
# @rbs (file: String, class_name: String?, findings: Array[Finding]) -> void
|
|
54
|
+
def initialize(file:, class_name:, findings:)
|
|
55
|
+
@file = file.dup.freeze
|
|
56
|
+
@class_name = class_name&.dup&.freeze
|
|
57
|
+
@findings = findings.dup.freeze
|
|
58
|
+
freeze
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# @rbs () -> bool
|
|
62
|
+
def compatible? = @findings.none? { |finding| finding.severity == :error }
|
|
63
|
+
|
|
64
|
+
# @rbs () -> Hash[String, untyped]
|
|
65
|
+
def to_h
|
|
66
|
+
{
|
|
67
|
+
"ibex_migration_check" => "racc",
|
|
68
|
+
"schema_version" => SCHEMA_VERSION,
|
|
69
|
+
"compatible" => compatible?,
|
|
70
|
+
"file" => @file,
|
|
71
|
+
"class_name" => @class_name,
|
|
72
|
+
"findings" => @findings.map(&:to_h)
|
|
73
|
+
}
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# @rbs (*untyped) -> String
|
|
77
|
+
def to_json(*) = JSON.pretty_generate(to_h)
|
|
78
|
+
|
|
79
|
+
# @rbs () -> String
|
|
80
|
+
def to_text
|
|
81
|
+
return "#{@file}: compatible with the checked racc migration surface\n" if @findings.empty?
|
|
82
|
+
|
|
83
|
+
lines = @findings.flat_map do |finding|
|
|
84
|
+
prefix = finding.location ? finding.location.to_s : "#{@file}:1:1"
|
|
85
|
+
[
|
|
86
|
+
"#{prefix}: #{finding.severity}: #{finding.message} [#{finding.code}]",
|
|
87
|
+
" suggestion: #{finding.suggestion}"
|
|
88
|
+
]
|
|
89
|
+
end
|
|
90
|
+
"#{lines.join("\n")}\n"
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "racc_migration/report"
|
|
4
|
+
require_relative "racc_migration/checker"
|
|
5
|
+
require_relative "racc_migration/harness"
|
|
6
|
+
|
|
7
|
+
module Ibex
|
|
8
|
+
# Static checks and explicitly generated differential tooling for migration
|
|
9
|
+
# from racc grammar files.
|
|
10
|
+
module RaccMigration
|
|
11
|
+
end
|
|
12
|
+
end
|