ibex 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (603) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +89 -123
  3. data/docs/architecture.md +477 -31
  4. data/docs/bison-import.md +103 -0
  5. data/docs/comparison-policy.md +127 -0
  6. data/docs/configuration-model.md +59 -0
  7. data/docs/configuration-report.md +28 -0
  8. data/docs/conflict-explanation-reviews/v1/records/README.md +28 -0
  9. data/docs/conflict-explanation-study.md +62 -0
  10. data/docs/construction-profiling.md +157 -0
  11. data/docs/cst-migration.md +106 -0
  12. data/docs/cst.md +237 -0
  13. data/docs/decisions/0000-template.md +24 -0
  14. data/docs/decisions/0001-separate-ir-pipeline.md +34 -0
  15. data/docs/decisions/0002-opaque-user-code-boundary.md +32 -0
  16. data/docs/decisions/0003-self-hosted-grammar-frontend.md +32 -0
  17. data/docs/decisions/0004-shared-semantic-and-lossless-source-model.md +33 -0
  18. data/docs/decisions/0005-contained-grammar-composition.md +34 -0
  19. data/docs/decisions/0006-bounded-structural-grammar-lowering.md +32 -0
  20. data/docs/decisions/0007-shared-parser-construction-pipeline.md +33 -0
  21. data/docs/decisions/0008-versioned-runtime-package-boundary.md +36 -0
  22. data/docs/decisions/0009-isolated-parser-sessions.md +34 -0
  23. data/docs/decisions/0010-committed-runtime-observation.md +34 -0
  24. data/docs/decisions/0011-versioned-semantic-action-boundary.md +37 -0
  25. data/docs/decisions/0012-bounded-nonexecuting-analysis.md +37 -0
  26. data/docs/decisions/0013-transactional-generation-publication.md +37 -0
  27. data/docs/decisions/0014-versioned-generated-lexer.md +35 -0
  28. data/docs/decisions/0015-worker-isolated-browser-analysis.md +32 -0
  29. data/docs/decisions/0016-red-green-concrete-syntax.md +34 -0
  30. data/docs/decisions/0017-persistent-syntax-artifacts.md +35 -0
  31. data/docs/decisions/0018-conservative-incremental-syntax-reuse.md +38 -0
  32. data/docs/decisions/0019-runtime-syntax-session-boundary.md +51 -0
  33. data/docs/decisions/0020-grammar-ir-parser-contract.md +65 -0
  34. data/docs/decisions/0021-data-only-parser-table-sidecar.md +50 -0
  35. data/docs/decisions/0022-manifest-bound-verification-report.md +77 -0
  36. data/docs/decisions/0023-syntax-only-repair-results.md +48 -0
  37. data/docs/decisions/0024-direct-ielr-construction.md +42 -0
  38. data/docs/decisions/README.md +74 -0
  39. data/docs/declarative-configuration.md +262 -0
  40. data/docs/development.md +306 -0
  41. data/docs/direct-ielr-decision.md +120 -0
  42. data/docs/direct-multi-entry-decision.md +82 -0
  43. data/docs/editor-setup.md +26 -0
  44. data/docs/error-ux-review-rubric-v1.md +117 -0
  45. data/docs/error-ux-reviews/v1/records/README.md +22 -0
  46. data/docs/error-ux-round2-review-status-v1.json +27 -0
  47. data/docs/error-ux-round2-reviews/v1/records/README.md +30 -0
  48. data/docs/error-ux-round2-v1.json +1581 -0
  49. data/docs/error-ux-round2.md +111 -0
  50. data/docs/error-ux.md +114 -0
  51. data/docs/getting-started.md +80 -0
  52. data/docs/grammar-reference.md +629 -18
  53. data/docs/ielr-design.md +43 -0
  54. data/docs/ielr.md +92 -0
  55. data/docs/ja/bison-import.md +51 -0
  56. data/docs/ja/stable-api.md +53 -0
  57. data/docs/lexer-construction-profile.md +109 -0
  58. data/docs/lexer-migration.md +56 -0
  59. data/docs/maturity.md +128 -0
  60. data/docs/project-site-strategy.md +54 -0
  61. data/docs/racc-migration-evidence.md +45 -0
  62. data/docs/racc-migration.md +45 -6
  63. data/docs/release-readiness.md +221 -0
  64. data/docs/repair-semantics.md +123 -0
  65. data/docs/runtime-abi-evolution.md +307 -0
  66. data/docs/stability.md +234 -0
  67. data/docs/status.md +43 -0
  68. data/docs/syntax-sessions.md +183 -0
  69. data/docs/table-artifact.md +125 -0
  70. data/docs/test-interactions.md +219 -0
  71. data/docs/verification-report.md +137 -0
  72. data/docs/verifier-trust-boundary.md +199 -0
  73. data/docs/workloads.md +132 -0
  74. data/examples/README.md +60 -0
  75. data/examples/calculator.y +44 -0
  76. data/examples/csv.y +32 -0
  77. data/examples/ini.y +70 -0
  78. data/examples/json.y +48 -0
  79. data/examples/tiny_language.y +75 -0
  80. data/lib/ibex/analysis/digraph.rb +128 -0
  81. data/lib/ibex/analysis/sets.rb +5 -4
  82. data/lib/ibex/analysis.rb +1 -0
  83. data/lib/ibex/artifact_set.rb +63 -0
  84. data/lib/ibex/bison_import/importer.rb +516 -0
  85. data/lib/ibex/bison_import/tokenizer.rb +260 -0
  86. data/lib/ibex/bison_import.rb +200 -0
  87. data/lib/ibex/bounded_subprocess.rb +171 -0
  88. data/lib/ibex/cli/ambiguity.rb +75 -0
  89. data/lib/ibex/cli/analysis.rb +134 -0
  90. data/lib/ibex/cli/bison_import.rb +121 -0
  91. data/lib/ibex/cli/config.rb +128 -0
  92. data/lib/ibex/cli/counterexample_options.rb +6 -2
  93. data/lib/ibex/cli/coverage.rb +173 -0
  94. data/lib/ibex/cli/debug.rb +106 -0
  95. data/lib/ibex/cli/diagnostics.rb +136 -0
  96. data/lib/ibex/cli/documentation.rb +70 -0
  97. data/lib/ibex/cli/equiv.rb +169 -0
  98. data/lib/ibex/cli/error_messages.rb +184 -0
  99. data/lib/ibex/cli/explain.rb +85 -0
  100. data/lib/ibex/cli/fix.rb +203 -0
  101. data/lib/ibex/cli/formatting.rb +387 -0
  102. data/lib/ibex/cli/fuzz.rb +200 -0
  103. data/lib/ibex/cli/fuzz_regressions.rb +145 -0
  104. data/lib/ibex/cli/generation_artifacts.rb +189 -0
  105. data/lib/ibex/cli/generation_error_messages.rb +39 -0
  106. data/lib/ibex/cli/grammar_tests.rb +140 -0
  107. data/lib/ibex/cli/ir_tools.rb +128 -0
  108. data/lib/ibex/cli/lsp.rb +33 -0
  109. data/lib/ibex/cli/outputs.rb +133 -23
  110. data/lib/ibex/cli/racc_migration.rb +87 -0
  111. data/lib/ibex/cli/reduce.rb +202 -0
  112. data/lib/ibex/cli/reduce_reporting.rb +71 -0
  113. data/lib/ibex/cli/samples.rb +118 -0
  114. data/lib/ibex/cli/verify.rb +116 -0
  115. data/lib/ibex/cli/watch.rb +104 -0
  116. data/lib/ibex/cli.rb +825 -55
  117. data/lib/ibex/codegen/action_locations.rb +103 -0
  118. data/lib/ibex/codegen/action_method_source.rb +209 -0
  119. data/lib/ibex/codegen/action_source.rb +119 -0
  120. data/lib/ibex/codegen/ambiguity.rb +174 -0
  121. data/lib/ibex/codegen/cst_metadata.rb +171 -0
  122. data/lib/ibex/codegen/documentation.rb +170 -0
  123. data/lib/ibex/codegen/explain.rb +380 -0
  124. data/lib/ibex/codegen/generated_action_abi.rb +293 -0
  125. data/lib/ibex/codegen/html.rb +94 -10
  126. data/lib/ibex/codegen/mermaid.rb +43 -0
  127. data/lib/ibex/codegen/railroad.rb +197 -0
  128. data/lib/ibex/codegen/railroad_documentation.rb +59 -0
  129. data/lib/ibex/codegen/rbs.rb +331 -2
  130. data/lib/ibex/codegen/report.rb +67 -15
  131. data/lib/ibex/codegen/ruby.rb +305 -66
  132. data/lib/ibex/codegen/ruby_actions.rb +146 -0
  133. data/lib/ibex/codegen/ruby_ast.rb +121 -0
  134. data/lib/ibex/codegen/ruby_error_messages.rb +28 -0
  135. data/lib/ibex/codegen/ruby_lexer.rb +83 -0
  136. data/lib/ibex/codegen/ruby_syntax.rb +96 -0
  137. data/lib/ibex/codegen/ruby_table_metadata.rb +57 -0
  138. data/lib/ibex/codegen/ruby_value_printers.rb +56 -0
  139. data/lib/ibex/codegen/symbol_labels.rb +1 -1
  140. data/lib/ibex/configuration/analysis_grammar.rb +35 -0
  141. data/lib/ibex/configuration/explanation.rb +421 -0
  142. data/lib/ibex/configuration/inspector.rb +232 -0
  143. data/lib/ibex/configuration.rb +580 -0
  144. data/lib/ibex/coverage/collector.rb +198 -0
  145. data/lib/ibex/coverage/event_stream.rb +103 -0
  146. data/lib/ibex/coverage/report.rb +267 -0
  147. data/lib/ibex/coverage/runtime_event_validator.rb +169 -0
  148. data/lib/ibex/coverage.rb +13 -0
  149. data/lib/ibex/delta_reducer.rb +99 -0
  150. data/lib/ibex/diff.rb +140 -0
  151. data/lib/ibex/equiv/machine.rb +135 -0
  152. data/lib/ibex/equiv.rb +373 -0
  153. data/lib/ibex/error_messages/parser.rb +159 -0
  154. data/lib/ibex/error_messages/parser_v2.rb +198 -0
  155. data/lib/ibex/error_messages/renderer.rb +65 -0
  156. data/lib/ibex/error_messages/sentence_search.rb +196 -0
  157. data/lib/ibex/error_messages/update.rb +169 -0
  158. data/lib/ibex/error_messages.rb +165 -0
  159. data/lib/ibex/fix.rb +557 -0
  160. data/lib/ibex/frontend/ast.rb +166 -7
  161. data/lib/ibex/frontend/bootstrap_parser.rb +40 -4
  162. data/lib/ibex/frontend/diagnostic.rb +81 -0
  163. data/lib/ibex/frontend/diagnostic_recovery.rb +268 -0
  164. data/lib/ibex/frontend/dsl.rb +66 -7
  165. data/lib/ibex/frontend/formatter.rb +453 -0
  166. data/lib/ibex/frontend/generated_parser.rb +644 -190
  167. data/lib/ibex/frontend/generated_parser_base.rb +170 -32
  168. data/lib/ibex/frontend/generated_parser_includes.rb +51 -0
  169. data/lib/ibex/frontend/generated_parser_metadata.rb +61 -0
  170. data/lib/ibex/frontend/generated_parser_parameters.rb +60 -0
  171. data/lib/ibex/frontend/generation.rb +33 -0
  172. data/lib/ibex/frontend/lexer.rb +162 -10
  173. data/lib/ibex/frontend/lexer_recovery.rb +84 -0
  174. data/lib/ibex/frontend/parser/declarations.rb +262 -11
  175. data/lib/ibex/frontend/parser/parameters.rb +82 -0
  176. data/lib/ibex/frontend/parser/rules.rb +43 -6
  177. data/lib/ibex/frontend/parser.rb +182 -4
  178. data/lib/ibex/frontend/parser_configuration_support.rb +56 -0
  179. data/lib/ibex/frontend/regenerator.rb +27 -1
  180. data/lib/ibex/frontend/resolution.rb +70 -0
  181. data/lib/ibex/frontend/resolver.rb +217 -0
  182. data/lib/ibex/frontend/rule_documentation.rb +104 -0
  183. data/lib/ibex/frontend/source_cursor.rb +133 -9
  184. data/lib/ibex/frontend/source_document.rb +229 -0
  185. data/lib/ibex/frontend/source_loader.rb +150 -0
  186. data/lib/ibex/frontend/source_span.rb +89 -0
  187. data/lib/ibex/frontend/token_adapter/declaration_document_state.rb +47 -0
  188. data/lib/ibex/frontend/token_adapter/declaration_lexer_state.rb +83 -0
  189. data/lib/ibex/frontend/token_adapter/declaration_state.rb +244 -22
  190. data/lib/ibex/frontend/token_adapter/delimiter_tracker.rb +8 -2
  191. data/lib/ibex/frontend/token_adapter/rule_state.rb +60 -2
  192. data/lib/ibex/frontend/token_adapter.rb +18 -3
  193. data/lib/ibex/frontend.rb +14 -2
  194. data/lib/ibex/fuzz.rb +219 -0
  195. data/lib/ibex/generation_input.rb +57 -0
  196. data/lib/ibex/generation_manifest.rb +211 -0
  197. data/lib/ibex/generation_transaction.rb +261 -0
  198. data/lib/ibex/generation_transaction_recovery.rb +109 -0
  199. data/lib/ibex/generation_transaction_validation.rb +196 -0
  200. data/lib/ibex/grammar_tests.rb +206 -0
  201. data/lib/ibex/ir/automaton_ir.rb +86 -11
  202. data/lib/ibex/ir/grammar_ir.rb +170 -26
  203. data/lib/ibex/ir/lexer_ir.rb +76 -0
  204. data/lib/ibex/ir/parser_contract.rb +135 -0
  205. data/lib/ibex/ir/serialize.rb +268 -50
  206. data/lib/ibex/ir/validator/automaton.rb +375 -0
  207. data/lib/ibex/ir/validator/base.rb +147 -0
  208. data/lib/ibex/ir/validator/grammar.rb +634 -0
  209. data/lib/ibex/ir/validator/lexer.rb +113 -0
  210. data/lib/ibex/ir/validator.rb +63 -0
  211. data/lib/ibex/ir.rb +71 -5
  212. data/lib/ibex/lalr/build_metrics.rb +71 -0
  213. data/lib/ibex/lalr/builder.rb +514 -52
  214. data/lib/ibex/lalr/conflict.rb +1 -0
  215. data/lib/ibex/lalr/conflict_search.rb +26 -9
  216. data/lib/ibex/lalr/counterexample.rb +38 -18
  217. data/lib/ibex/lalr/direct_lookaheads.rb +289 -0
  218. data/lib/ibex/lalr/goto_follows.rb +229 -0
  219. data/lib/ibex/lalr/ielr/annotator.rb +214 -0
  220. data/lib/ibex/lalr/ielr/bits.rb +28 -0
  221. data/lib/ibex/lalr/ielr/inadequacy.rb +47 -0
  222. data/lib/ibex/lalr/ielr/item_lookaheads.rb +78 -0
  223. data/lib/ibex/lalr/ielr/pipeline.rb +75 -0
  224. data/lib/ibex/lalr/ielr/split_stability.rb +78 -0
  225. data/lib/ibex/lalr/ielr/split_state.rb +20 -0
  226. data/lib/ibex/lalr/ielr/state_splitter.rb +258 -0
  227. data/lib/ibex/lalr/ielr_partition.rb +166 -0
  228. data/lib/ibex/lalr/inadequacy_report.rb +50 -0
  229. data/lib/ibex/lalr/lookahead_propagation.rb +111 -0
  230. data/lib/ibex/lalr/lr0_collection.rb +121 -0
  231. data/lib/ibex/lalr/on_error_reductions.rb +74 -0
  232. data/lib/ibex/lalr/unreachable_states.rb +80 -0
  233. data/lib/ibex/lalr.rb +59 -1
  234. data/lib/ibex/location.rb +129 -0
  235. data/lib/ibex/lsp/document_handlers.rb +68 -0
  236. data/lib/ibex/lsp/document_store.rb +264 -0
  237. data/lib/ibex/lsp/document_store_diagnostics.rb +44 -0
  238. data/lib/ibex/lsp/document_store_validation.rb +61 -0
  239. data/lib/ibex/lsp/initialization_handlers.rb +80 -0
  240. data/lib/ibex/lsp/navigation_handlers.rb +70 -0
  241. data/lib/ibex/lsp/parser_configuration_assistance.rb +149 -0
  242. data/lib/ibex/lsp/position_codec.rb +113 -0
  243. data/lib/ibex/lsp/protocol_error.rb +33 -0
  244. data/lib/ibex/lsp/request_handlers.rb +46 -0
  245. data/lib/ibex/lsp/request_support.rb +94 -0
  246. data/lib/ibex/lsp/server.rb +148 -0
  247. data/lib/ibex/lsp/symbol_index.rb +241 -0
  248. data/lib/ibex/lsp/symbol_index_builder.rb +277 -0
  249. data/lib/ibex/lsp/symbol_index_precedence_references.rb +44 -0
  250. data/lib/ibex/lsp/symbol_index_source_queries.rb +61 -0
  251. data/lib/ibex/lsp/symbol_occurrence.rb +23 -0
  252. data/lib/ibex/lsp/transport.rb +128 -0
  253. data/lib/ibex/lsp/workspace.rb +128 -0
  254. data/lib/ibex/lsp/workspace_analyzer.rb +199 -0
  255. data/lib/ibex/lsp.rb +33 -0
  256. data/lib/ibex/messages/en.yml +28 -0
  257. data/lib/ibex/messages/ja.yml +28 -0
  258. data/lib/ibex/messages.rb +68 -0
  259. data/lib/ibex/metrics.rb +175 -0
  260. data/lib/ibex/normalize/declarations.rb +142 -7
  261. data/lib/ibex/normalize/diagnostics.rb +94 -24
  262. data/lib/ibex/normalize/expander.rb +64 -59
  263. data/lib/ibex/normalize/expression.rb +63 -40
  264. data/lib/ibex/normalize/grammar_builder.rb +26 -0
  265. data/lib/ibex/normalize/inline_expansion.rb +465 -0
  266. data/lib/ibex/normalize/inline_validation.rb +174 -0
  267. data/lib/ibex/normalize/lexer.rb +131 -0
  268. data/lib/ibex/normalize/named_references.rb +60 -0
  269. data/lib/ibex/normalize/nodes.rb +46 -0
  270. data/lib/ibex/normalize/parameter_ebnf_lowering.rb +70 -0
  271. data/lib/ibex/normalize/parameter_lowering.rb +143 -0
  272. data/lib/ibex/normalize/parameter_substitution.rb +125 -0
  273. data/lib/ibex/normalize/parameter_validation.rb +140 -0
  274. data/lib/ibex/normalize/parameters.rb +226 -0
  275. data/lib/ibex/normalize/parser_configuration.rb +70 -0
  276. data/lib/ibex/normalize/recovery_declarations.rb +84 -0
  277. data/lib/ibex/normalize.rb +172 -17
  278. data/lib/ibex/racc_migration/checker.rb +122 -0
  279. data/lib/ibex/racc_migration/harness.rb +177 -0
  280. data/lib/ibex/racc_migration/report.rb +97 -0
  281. data/lib/ibex/racc_migration.rb +12 -0
  282. data/lib/ibex/rake_task.rb +116 -0
  283. data/lib/ibex/samples.rb +216 -0
  284. data/lib/ibex/table_artifact/builder.rb +266 -0
  285. data/lib/ibex/table_artifact/cst_projection.rb +70 -0
  286. data/lib/ibex/table_artifact/document.rb +39 -0
  287. data/lib/ibex/table_artifact/executor.rb +187 -0
  288. data/lib/ibex/table_artifact/serializer.rb +57 -0
  289. data/lib/ibex/table_artifact/validator/metadata.rb +179 -0
  290. data/lib/ibex/table_artifact/validator/support.rb +86 -0
  291. data/lib/ibex/table_artifact/validator/tables.rb +238 -0
  292. data/lib/ibex/table_artifact/validator.rb +253 -0
  293. data/lib/ibex/table_artifact.rb +88 -0
  294. data/lib/ibex/table_simulation/result.rb +55 -0
  295. data/lib/ibex/table_simulation/simulator.rb +253 -0
  296. data/lib/ibex/table_simulation/step.rb +62 -0
  297. data/lib/ibex/table_simulation/text.rb +31 -0
  298. data/lib/ibex/table_simulation.rb +13 -0
  299. data/lib/ibex/tables.rb +22 -69
  300. data/lib/ibex/verifiable_generation_bundle.rb +86 -0
  301. data/lib/ibex/verification_report/builder.rb +159 -0
  302. data/lib/ibex/verification_report/canonical_ir.rb +123 -0
  303. data/lib/ibex/verification_report/logical_path.rb +68 -0
  304. data/lib/ibex/verification_report/validator.rb +329 -0
  305. data/lib/ibex/verification_report.rb +69 -0
  306. data/lib/ibex/verify/action_correspondence.rb +138 -0
  307. data/lib/ibex/verify/language_witness.rb +295 -0
  308. data/lib/ibex/verify/reference_collection.rb +190 -0
  309. data/lib/ibex/verify/result.rb +73 -0
  310. data/lib/ibex/verify/verifier.rb +598 -0
  311. data/lib/ibex/verify.rb +15 -0
  312. data/lib/ibex/version.rb +1 -1
  313. data/lib/ibex/watch/runner.rb +181 -0
  314. data/lib/ibex/watch/source_snapshot.rb +98 -0
  315. data/lib/ibex/watch.rb +11 -0
  316. data/lib/ibex.rb +35 -1
  317. data/schema/automaton-ir-definitions.schema.json +401 -0
  318. data/schema/automaton-ir.schema.json +65 -0
  319. data/schema/benchmark-v1.schema.json +212 -0
  320. data/schema/benchmark-v2.schema.json +61 -0
  321. data/schema/bison-import-v1.schema.json +106 -0
  322. data/schema/conflict-explanation-review-v1.schema.json +196 -0
  323. data/schema/conflict-explanation-study-v1.schema.json +528 -0
  324. data/schema/construction-profile-v1.schema.json +779 -0
  325. data/schema/cst-v1.json +170 -0
  326. data/schema/diff-v1.schema.json +41 -0
  327. data/schema/direct-ielr-decision-v1.schema.json +200 -0
  328. data/schema/equiv-v1.schema.json +64 -0
  329. data/schema/error-ux-review-v1.schema.json +609 -0
  330. data/schema/error-ux-round2-review-v1.schema.json +191 -0
  331. data/schema/error-ux-round2-v1.schema.json +519 -0
  332. data/schema/error-ux-v1.schema.json +258 -0
  333. data/schema/explain-v1.schema.json +495 -0
  334. data/schema/fix-v1.schema.json +64 -0
  335. data/schema/fix-v2.schema.json +66 -0
  336. data/schema/fix-v3.schema.json +101 -0
  337. data/schema/frontend-diagnostics-v1.schema.json +154 -0
  338. data/schema/fuzz-regression-v1.schema.json +79 -0
  339. data/schema/fuzz-v1.schema.json +114 -0
  340. data/schema/generation-manifest-v1.schema.json +115 -0
  341. data/schema/grammar-ir-extensions.schema.json +734 -0
  342. data/schema/grammar-ir-foundation.schema.json +426 -0
  343. data/schema/grammar-ir.schema.json +271 -0
  344. data/schema/ielr-benchmark-v1.schema.json +85 -0
  345. data/schema/lexer-ir-v1.schema.json +215 -0
  346. data/schema/lexer-profile-v1.schema.json +491 -0
  347. data/schema/metrics-v1.schema.json +60 -0
  348. data/schema/migration-check-v1.schema.json +60 -0
  349. data/schema/performance-comparison-v1.schema.json +395 -0
  350. data/schema/public-performance-comparison-v1.schema.json +506 -0
  351. data/schema/public-performance-profile-v1.schema.json +360 -0
  352. data/schema/reduce-v1.schema.json +22 -0
  353. data/schema/reduce-v2.schema.json +63 -0
  354. data/schema/runtime-coverage-v1.schema.json +86 -0
  355. data/schema/runtime-event-v1.schema.json +308 -0
  356. data/schema/table-artifact-v1.schema.json +377 -0
  357. data/schema/table-simulation-v1.schema.json +70 -0
  358. data/schema/verification-report-v1.schema.json +222 -0
  359. data/schema/verify-v1.schema.json +42 -0
  360. data/sig/ibex/analysis/digraph.rbs +19 -0
  361. data/sig/ibex/artifact_set.rbs +37 -0
  362. data/sig/ibex/bison_import/importer.rbs +140 -0
  363. data/sig/ibex/bison_import/tokenizer.rbs +91 -0
  364. data/sig/ibex/bison_import.rbs +95 -0
  365. data/sig/ibex/bounded_subprocess.rbs +76 -0
  366. data/sig/ibex/cli/ambiguity.rbs +28 -0
  367. data/sig/ibex/cli/analysis.rbs +43 -0
  368. data/sig/ibex/cli/bison_import.rbs +32 -0
  369. data/sig/ibex/cli/config.rbs +35 -0
  370. data/sig/ibex/cli/counterexample_options.rbs +2 -0
  371. data/sig/ibex/cli/coverage.rbs +53 -0
  372. data/sig/ibex/cli/debug.rbs +28 -0
  373. data/sig/ibex/cli/diagnostics.rbs +51 -0
  374. data/sig/ibex/cli/documentation.rbs +29 -0
  375. data/sig/ibex/cli/equiv.rbs +49 -0
  376. data/sig/ibex/cli/error_messages.rbs +63 -0
  377. data/sig/ibex/cli/explain.rbs +31 -0
  378. data/sig/ibex/cli/fix.rbs +53 -0
  379. data/sig/ibex/cli/formatting.rbs +107 -0
  380. data/sig/ibex/cli/fuzz.rbs +47 -0
  381. data/sig/ibex/cli/fuzz_regressions.rbs +40 -0
  382. data/sig/ibex/cli/generation_artifacts.rbs +69 -0
  383. data/sig/ibex/cli/generation_error_messages.rbs +19 -0
  384. data/sig/ibex/cli/grammar_tests.rbs +48 -0
  385. data/sig/ibex/cli/ir_tools.rbs +39 -0
  386. data/sig/ibex/cli/lsp.rbs +14 -0
  387. data/sig/ibex/cli/outputs.rbs +26 -3
  388. data/sig/ibex/cli/racc_migration.rbs +30 -0
  389. data/sig/ibex/cli/reduce.rbs +60 -0
  390. data/sig/ibex/cli/reduce_reporting.rbs +27 -0
  391. data/sig/ibex/cli/samples.rbs +38 -0
  392. data/sig/ibex/cli/verify.rbs +28 -0
  393. data/sig/ibex/cli/watch.rbs +43 -0
  394. data/sig/ibex/cli.rbs +182 -9
  395. data/sig/ibex/codegen/action_locations.rbs +45 -0
  396. data/sig/ibex/codegen/action_method_source.rbs +65 -0
  397. data/sig/ibex/codegen/action_source.rbs +50 -0
  398. data/sig/ibex/codegen/ambiguity.rbs +60 -0
  399. data/sig/ibex/codegen/cst_metadata.rbs +59 -0
  400. data/sig/ibex/codegen/documentation.rbs +50 -0
  401. data/sig/ibex/codegen/explain.rbs +105 -0
  402. data/sig/ibex/codegen/generated_action_abi.rbs +105 -0
  403. data/sig/ibex/codegen/html.rbs +18 -2
  404. data/sig/ibex/codegen/mermaid.rbs +16 -0
  405. data/sig/ibex/codegen/railroad.rbs +82 -0
  406. data/sig/ibex/codegen/railroad_documentation.rbs +31 -0
  407. data/sig/ibex/codegen/rbs.rbs +86 -4
  408. data/sig/ibex/codegen/report.rbs +16 -8
  409. data/sig/ibex/codegen/ruby.rbs +105 -28
  410. data/sig/ibex/codegen/ruby_actions.rbs +54 -0
  411. data/sig/ibex/codegen/ruby_ast.rbs +34 -0
  412. data/sig/ibex/codegen/ruby_error_messages.rbs +16 -0
  413. data/sig/ibex/codegen/ruby_lexer.rbs +25 -0
  414. data/sig/ibex/codegen/ruby_syntax.rbs +27 -0
  415. data/sig/ibex/codegen/ruby_table_metadata.rbs +26 -0
  416. data/sig/ibex/codegen/ruby_value_printers.rbs +28 -0
  417. data/sig/ibex/configuration/analysis_grammar.rbs +13 -0
  418. data/sig/ibex/configuration/explanation.rbs +195 -0
  419. data/sig/ibex/configuration/inspector.rbs +53 -0
  420. data/sig/ibex/configuration.rbs +242 -0
  421. data/sig/ibex/coverage/collector.rbs +78 -0
  422. data/sig/ibex/coverage/event_stream.rbs +44 -0
  423. data/sig/ibex/coverage/report.rbs +102 -0
  424. data/sig/ibex/coverage/runtime_event_validator.rbs +70 -0
  425. data/sig/ibex/coverage.rbs +7 -0
  426. data/sig/ibex/delta_reducer.rbs +38 -0
  427. data/sig/ibex/diff.rbs +47 -0
  428. data/sig/ibex/equiv/machine.rbs +54 -0
  429. data/sig/ibex/equiv.rbs +129 -0
  430. data/sig/ibex/error_messages/parser.rbs +58 -0
  431. data/sig/ibex/error_messages/parser_v2.rbs +67 -0
  432. data/sig/ibex/error_messages/renderer.rbs +23 -0
  433. data/sig/ibex/error_messages/sentence_search.rbs +80 -0
  434. data/sig/ibex/error_messages/update.rbs +43 -0
  435. data/sig/ibex/error_messages.rbs +85 -0
  436. data/sig/ibex/fix.rbs +138 -0
  437. data/sig/ibex/frontend/ast.rbs +237 -22
  438. data/sig/ibex/frontend/bootstrap_parser.rbs +13 -0
  439. data/sig/ibex/frontend/diagnostic.rbs +53 -0
  440. data/sig/ibex/frontend/diagnostic_recovery.rbs +98 -0
  441. data/sig/ibex/frontend/dsl.rbs +33 -4
  442. data/sig/ibex/frontend/formatter.rbs +143 -0
  443. data/sig/ibex/frontend/generated_parser.rbs +228 -68
  444. data/sig/ibex/frontend/generated_parser_base.rbs +65 -12
  445. data/sig/ibex/frontend/generated_parser_includes.rbs +23 -0
  446. data/sig/ibex/frontend/generated_parser_metadata.rbs +23 -0
  447. data/sig/ibex/frontend/generated_parser_parameters.rbs +24 -0
  448. data/sig/ibex/frontend/generation.rbs +6 -0
  449. data/sig/ibex/frontend/lexer.rbs +49 -2
  450. data/sig/ibex/frontend/lexer_recovery.rbs +25 -0
  451. data/sig/ibex/frontend/parser/declarations.rbs +57 -0
  452. data/sig/ibex/frontend/parser/parameters.rbs +28 -0
  453. data/sig/ibex/frontend/parser/rules.rbs +3 -0
  454. data/sig/ibex/frontend/parser.rbs +66 -0
  455. data/sig/ibex/frontend/parser_configuration_support.rbs +27 -0
  456. data/sig/ibex/frontend/regenerator.rbs +11 -0
  457. data/sig/ibex/frontend/resolution.rbs +34 -0
  458. data/sig/ibex/frontend/resolver.rbs +91 -0
  459. data/sig/ibex/frontend/rule_documentation.rbs +42 -0
  460. data/sig/ibex/frontend/source_cursor.rbs +38 -5
  461. data/sig/ibex/frontend/source_document.rbs +119 -0
  462. data/sig/ibex/frontend/source_loader.rbs +66 -0
  463. data/sig/ibex/frontend/source_span.rbs +56 -0
  464. data/sig/ibex/frontend/token_adapter/declaration_document_state.rbs +21 -0
  465. data/sig/ibex/frontend/token_adapter/declaration_lexer_state.rbs +27 -0
  466. data/sig/ibex/frontend/token_adapter/declaration_state.rbs +83 -3
  467. data/sig/ibex/frontend/token_adapter/rule_state.rbs +20 -0
  468. data/sig/ibex/frontend/token_adapter.rbs +11 -2
  469. data/sig/ibex/frontend.rbs +1 -1
  470. data/sig/ibex/fuzz.rbs +71 -0
  471. data/sig/ibex/generation_input.rbs +37 -0
  472. data/sig/ibex/generation_manifest.rbs +70 -0
  473. data/sig/ibex/generation_transaction.rbs +82 -0
  474. data/sig/ibex/generation_transaction_recovery.rbs +36 -0
  475. data/sig/ibex/generation_transaction_validation.rbs +65 -0
  476. data/sig/ibex/grammar_tests.rbs +93 -0
  477. data/sig/ibex/ir/automaton_ir.rbs +35 -10
  478. data/sig/ibex/ir/grammar_ir.rbs +96 -22
  479. data/sig/ibex/ir/lexer_ir.rbs +57 -0
  480. data/sig/ibex/ir/parser_contract.rbs +69 -0
  481. data/sig/ibex/ir/serialize.rbs +75 -23
  482. data/sig/ibex/ir/validator/automaton.rbs +112 -0
  483. data/sig/ibex/ir/validator/base.rbs +69 -0
  484. data/sig/ibex/ir/validator/grammar.rbs +195 -0
  485. data/sig/ibex/ir/validator/lexer.rbs +37 -0
  486. data/sig/ibex/ir/validator.rbs +16 -0
  487. data/sig/ibex/ir.rbs +45 -5
  488. data/sig/ibex/lalr/build_metrics.rbs +60 -0
  489. data/sig/ibex/lalr/builder.rbs +150 -19
  490. data/sig/ibex/lalr/conflict_search.rbs +12 -5
  491. data/sig/ibex/lalr/counterexample.rbs +20 -14
  492. data/sig/ibex/lalr/direct_lookaheads.rbs +93 -0
  493. data/sig/ibex/lalr/goto_follows.rbs +89 -0
  494. data/sig/ibex/lalr/ielr/annotator.rbs +67 -0
  495. data/sig/ibex/lalr/ielr/bits.rbs +14 -0
  496. data/sig/ibex/lalr/ielr/inadequacy.rbs +24 -0
  497. data/sig/ibex/lalr/ielr/item_lookaheads.rbs +35 -0
  498. data/sig/ibex/lalr/ielr/pipeline.rbs +17 -0
  499. data/sig/ibex/lalr/ielr/split_stability.rbs +27 -0
  500. data/sig/ibex/lalr/ielr/split_state.rbs +14 -0
  501. data/sig/ibex/lalr/ielr/state_splitter.rbs +76 -0
  502. data/sig/ibex/lalr/ielr_partition.rbs +71 -0
  503. data/sig/ibex/lalr/inadequacy_report.rbs +21 -0
  504. data/sig/ibex/lalr/lookahead_propagation.rbs +38 -0
  505. data/sig/ibex/lalr/lr0_collection.rbs +50 -0
  506. data/sig/ibex/lalr/on_error_reductions.rbs +22 -0
  507. data/sig/ibex/lalr/unreachable_states.rbs +25 -0
  508. data/sig/ibex/lalr.rbs +17 -1
  509. data/sig/ibex/location.rbs +67 -0
  510. data/sig/ibex/lsp/document_handlers.rbs +24 -0
  511. data/sig/ibex/lsp/document_store.rbs +94 -0
  512. data/sig/ibex/lsp/document_store_diagnostics.rbs +20 -0
  513. data/sig/ibex/lsp/document_store_validation.rbs +26 -0
  514. data/sig/ibex/lsp/initialization_handlers.rbs +30 -0
  515. data/sig/ibex/lsp/navigation_handlers.rbs +33 -0
  516. data/sig/ibex/lsp/parser_configuration_assistance.rbs +49 -0
  517. data/sig/ibex/lsp/position_codec.rbs +44 -0
  518. data/sig/ibex/lsp/protocol_error.rbs +32 -0
  519. data/sig/ibex/lsp/request_handlers.rbs +26 -0
  520. data/sig/ibex/lsp/request_support.rbs +40 -0
  521. data/sig/ibex/lsp/server.rbs +55 -0
  522. data/sig/ibex/lsp/symbol_index.rbs +77 -0
  523. data/sig/ibex/lsp/symbol_index_builder.rbs +88 -0
  524. data/sig/ibex/lsp/symbol_index_precedence_references.rbs +18 -0
  525. data/sig/ibex/lsp/symbol_index_source_queries.rbs +26 -0
  526. data/sig/ibex/lsp/symbol_occurrence.rbs +31 -0
  527. data/sig/ibex/lsp/transport.rbs +47 -0
  528. data/sig/ibex/lsp/workspace.rbs +42 -0
  529. data/sig/ibex/lsp/workspace_analyzer.rbs +69 -0
  530. data/sig/ibex/lsp.rbs +7 -0
  531. data/sig/ibex/messages.rbs +26 -0
  532. data/sig/ibex/metrics.rbs +50 -0
  533. data/sig/ibex/normalize/declarations.rbs +31 -0
  534. data/sig/ibex/normalize/diagnostics.rbs +13 -4
  535. data/sig/ibex/normalize/expander.rbs +22 -16
  536. data/sig/ibex/normalize/expression.rbs +11 -9
  537. data/sig/ibex/normalize/grammar_builder.rbs +11 -0
  538. data/sig/ibex/normalize/inline_expansion.rbs +129 -0
  539. data/sig/ibex/normalize/inline_validation.rbs +40 -0
  540. data/sig/ibex/normalize/lexer.rbs +37 -0
  541. data/sig/ibex/normalize/named_references.rbs +20 -0
  542. data/sig/ibex/normalize/nodes.rbs +14 -0
  543. data/sig/ibex/normalize/parameter_ebnf_lowering.rbs +33 -0
  544. data/sig/ibex/normalize/parameter_lowering.rbs +35 -0
  545. data/sig/ibex/normalize/parameter_substitution.rbs +42 -0
  546. data/sig/ibex/normalize/parameter_validation.rbs +44 -0
  547. data/sig/ibex/normalize/parameters.rbs +51 -0
  548. data/sig/ibex/normalize/parser_configuration.rbs +23 -0
  549. data/sig/ibex/normalize/recovery_declarations.rbs +24 -0
  550. data/sig/ibex/normalize.rbs +129 -18
  551. data/sig/ibex/racc_migration/checker.rbs +36 -0
  552. data/sig/ibex/racc_migration/harness.rbs +16 -0
  553. data/sig/ibex/racc_migration/report.rbs +55 -0
  554. data/sig/ibex/racc_migration.rbs +8 -0
  555. data/sig/ibex/rake_task.rbs +51 -0
  556. data/sig/ibex/samples.rbs +60 -0
  557. data/sig/ibex/table_artifact/builder.rbs +93 -0
  558. data/sig/ibex/table_artifact/cst_projection.rbs +20 -0
  559. data/sig/ibex/table_artifact/document.rbs +27 -0
  560. data/sig/ibex/table_artifact/executor.rbs +66 -0
  561. data/sig/ibex/table_artifact/serializer.rbs +25 -0
  562. data/sig/ibex/table_artifact/validator/metadata.rbs +52 -0
  563. data/sig/ibex/table_artifact/validator/support.rbs +46 -0
  564. data/sig/ibex/table_artifact/validator/tables.rbs +63 -0
  565. data/sig/ibex/table_artifact/validator.rbs +66 -0
  566. data/sig/ibex/table_artifact.rbs +32 -0
  567. data/sig/ibex/table_simulation/result.rbs +36 -0
  568. data/sig/ibex/table_simulation/simulator.rbs +98 -0
  569. data/sig/ibex/table_simulation/step.rbs +42 -0
  570. data/sig/ibex/table_simulation/text.rbs +14 -0
  571. data/sig/ibex/table_simulation.rbs +7 -0
  572. data/sig/ibex/tables.rbs +10 -35
  573. data/sig/ibex/verifiable_generation_bundle.rbs +24 -0
  574. data/sig/ibex/verification_report/builder.rbs +53 -0
  575. data/sig/ibex/verification_report/canonical_ir.rbs +41 -0
  576. data/sig/ibex/verification_report/logical_path.rbs +34 -0
  577. data/sig/ibex/verification_report/validator.rbs +114 -0
  578. data/sig/ibex/verification_report.rbs +44 -0
  579. data/sig/ibex/verify/action_correspondence.rbs +50 -0
  580. data/sig/ibex/verify/language_witness.rbs +112 -0
  581. data/sig/ibex/verify/reference_collection.rbs +63 -0
  582. data/sig/ibex/verify/result.rbs +52 -0
  583. data/sig/ibex/verify/verifier.rbs +170 -0
  584. data/sig/ibex/verify.rbs +6 -0
  585. data/sig/ibex/watch/runner.rbs +58 -0
  586. data/sig/ibex/watch/source_snapshot.rbs +46 -0
  587. data/sig/ibex/watch.rbs +7 -0
  588. data/sig/ibex.rbs +2 -0
  589. metadata +524 -19
  590. data/.rubocop.yml +0 -43
  591. data/CHANGELOG.md +0 -30
  592. data/Rakefile +0 -25
  593. data/Steepfile +0 -10
  594. data/docs/compat-notes.md +0 -37
  595. data/docs/lexer-coverage.md +0 -14
  596. data/docs/phase10-extensions.md +0 -27
  597. data/gemfiles/Gemfile +0 -7
  598. data/gemfiles/Gemfile.lock +0 -98
  599. data/lib/ibex/frontend/grammar.y +0 -156
  600. data/lib/ibex/runtime/parser.rb +0 -360
  601. data/lib/ibex/runtime.rb +0 -8
  602. data/sig/ibex/runtime/parser.rbs +0 -167
  603. data/sig/ibex/runtime.rbs +0 -6
@@ -1,21 +1,56 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative "normalize/declarations"
4
+ require_relative "normalize/parser_configuration"
5
+ require_relative "normalize/grammar_builder"
6
+ require_relative "normalize/lexer"
7
+ require_relative "normalize/recovery_declarations"
4
8
  require_relative "normalize/expression"
9
+ require_relative "normalize/parameter_validation"
10
+ require_relative "normalize/inline_validation"
11
+ require_relative "normalize/parameter_substitution"
12
+ require_relative "normalize/parameter_ebnf_lowering"
13
+ require_relative "normalize/parameter_lowering"
14
+ require_relative "normalize/parameters"
15
+ require_relative "normalize/named_references"
16
+ require_relative "normalize/nodes"
5
17
  require_relative "normalize/expander"
18
+ require_relative "normalize/inline_expansion"
6
19
  require_relative "normalize/diagnostics"
20
+ require_relative "frontend/resolution"
7
21
 
8
22
  module Ibex
9
23
  # Converts a frontend AST into immutable Grammar IR.
10
24
  class Normalizer
11
25
  include NormalizeDeclarations
26
+ include NormalizeParserConfiguration
27
+ include NormalizeGrammarBuilder
28
+ include NormalizeLexer
29
+ include NormalizeRecoveryDeclarations
30
+ include NormalizeParameterValidation
31
+ include NormalizeInlineValidation
32
+ include NormalizeParameterSubstitution
33
+ include NormalizeParameterEbnfLowering
34
+ include NormalizeParameterLowering
35
+ include NormalizeParameters
36
+ include NormalizeNamedReferences
37
+ include NormalizeNodes
12
38
  include NormalizeExpander
39
+ include NormalizeInlineExpansion
13
40
  include NormalizeDiagnostics
14
41
 
15
42
  RESERVED_NAMES = %w[result val _values].freeze #: Array[String]
43
+ RUBY_KEYWORDS = %w[
44
+ __ENCODING__ __FILE__ __LINE__ alias and begin break case class def defined do else elsif end ensure false for if
45
+ in module next nil not or redo rescue retry return self super then true undef unless until when while yield
46
+ ].freeze #: Array[String]
47
+ DEFAULT_MAX_PARAMETER_SPECIALIZATIONS = 1_000
48
+ DEFAULT_MAX_INLINE_EXPANSIONS = 10_000
16
49
 
17
50
  # @rbs @ast: Frontend::AST::Root
18
- # @rbs @mode: Symbol
51
+ # @rbs @resolution: Frontend::Resolution?
52
+ # @rbs @current_include_chain: Array[IR::source_provenance]
53
+ # @rbs @mode: IR::grammar_mode
19
54
  # @rbs @symbols: Array[IR::GrammarSymbol]
20
55
  # @rbs @symbols_by_name: Hash[String, IR::GrammarSymbol]
21
56
  # @rbs @productions: Array[IR::Production]
@@ -24,40 +59,101 @@ module Ibex
24
59
  # @rbs @declared_tokens: Hash[String, IR::location]
25
60
  # @rbs @precedence: Hash[String, IR::precedence]
26
61
  # @rbs @precedence_locations: Hash[String, IR::location]
62
+ # @rbs @display_names: Hash[String, String]
63
+ # @rbs @display_name_locations: Hash[String, IR::location]
64
+ # @rbs @semantic_types: Hash[String, String]
65
+ # @rbs @semantic_type_locations: Hash[String, IR::location]
27
66
  # @rbs @options: IR::grammar_options
28
67
  # @rbs @expected_conflicts: Integer
68
+ # @rbs @expected_rr_conflicts: Integer?
29
69
  # @rbs @conversions: Hash[String, String]
30
- # @rbs @explicit_start: String?
70
+ # @rbs @parser_parameters: Array[IR::parser_parameter]
71
+ # @rbs @value_printers: Hash[String, IR::value_printer]
72
+ # @rbs @recovery_sync_tokens: Array[String]
73
+ # @rbs @on_error_reduce_groups: Array[Array[String]]
74
+ # @rbs @grammar_tests: Array[IR::grammar_test]
75
+ # @rbs @lexer_declaration: Frontend::AST::Lexer?
76
+ # @rbs @parser_configuration: Frontend::AST::ParserConfiguration?
77
+ # @rbs @explicit_starts: Array[String]?
78
+ # @rbs @start_names: Array[String]
31
79
  # @rbs @start_name: String
32
80
  # @rbs @start_location: Frontend::Location?
81
+ # @rbs @parameter_templates: Hash[String, Array[Frontend::AST::Rule]]
82
+ # @rbs @parameter_formals: Hash[String, Array[String]]
83
+ # @rbs @parameter_specializations: Hash[[String, Array[String]], String]
84
+ # @rbs @parameter_worklist: Array[NormalizeParameters::parameter_frame]
85
+ # @rbs @parameter_worklist_active: bool
86
+ # @rbs @max_parameter_specializations: Integer
87
+ # @rbs @current_parameter_expansion: IR::parameter_expansion?
88
+ # @rbs @rule_documentation: Hash[String, String]
89
+ # @rbs @inline_rule_names: Set[String]
90
+ # @rbs @inline_symbol_ids: Set[Integer]
91
+ # @rbs @inline_rule_by_symbol: Hash[Integer, String]
92
+ # @rbs @max_inline_expansions: Integer
93
+ # @rbs @inline_expansion_count: Integer
94
+ # @rbs @node_shapes: Hash[String, Array[String]]
95
+
96
+ # @rbs (Frontend::AST::Root | Frontend::AST::Fragment | Frontend::Resolution input,
97
+ # ?mode: Symbol | String, ?max_parameter_specializations: Integer,
98
+ # ?max_inline_expansions: Integer) -> void
99
+ def initialize(input, mode: :default, max_parameter_specializations: DEFAULT_MAX_PARAMETER_SPECIALIZATIONS,
100
+ max_inline_expansions: DEFAULT_MAX_INLINE_EXPANSIONS)
101
+ validate_positive_limit!(:max_parameter_specializations, max_parameter_specializations)
102
+ validate_positive_limit!(:max_inline_expansions, max_inline_expansions)
103
+ normalized_mode = mode.to_sym
104
+ raise ArgumentError, "mode must be :default or :extended" unless %i[default extended].include?(normalized_mode)
105
+
106
+ @resolution = input if input.is_a?(Frontend::Resolution)
107
+ ast = input.is_a?(Frontend::Resolution) ? input.root : input
108
+ normalized_mode = :extended if ast.is_a?(Frontend::AST::Root) && ast.extended
109
+ fail_at(ast.loc, "fragments must be resolved before normalization") if ast.is_a?(Frontend::AST::Fragment)
110
+ unresolved = ast.declarations.find { |declaration| declaration.is_a?(Frontend::AST::Include) }
111
+ fail_at(unresolved.loc, "includes must be resolved before normalization") if unresolved
33
112
 
34
- # @rbs (Frontend::AST::Root ast, ?mode: Symbol | String) -> void
35
- def initialize(ast, mode: :racc)
36
113
  @ast = ast
37
- @mode = mode
114
+ @mode = normalized_mode #: IR::grammar_mode
38
115
  @symbols = [] #: Array[IR::GrammarSymbol]
39
116
  @symbols_by_name = {} #: Hash[String, IR::GrammarSymbol]
40
117
  @productions = [] #: Array[IR::Production]
41
118
  @warnings = [] #: Array[IR::grammar_warning]
42
119
  @helper_sequence = 0
120
+ @current_include_chain = [] #: Array[IR::source_provenance]
121
+ @parameter_specializations = {} #: Hash[[String, Array[String]], String]
122
+ @parameter_worklist = [] #: Array[NormalizeParameters::parameter_frame]
123
+ @parameter_worklist_active = false
124
+ @max_parameter_specializations = max_parameter_specializations
125
+ @max_inline_expansions = max_inline_expansions
126
+ @inline_expansion_count = 0
127
+ @inline_symbol_ids = Set.new #: Set[Integer]
128
+ @inline_rule_by_symbol = {} #: Hash[Integer, String]
43
129
  end
44
130
 
45
131
  # @rbs () -> IR::Grammar
46
132
  def normalize
47
133
  read_declarations
134
+ gather_parameter_templates
135
+ gather_inline_rules
48
136
  intern_reserved_symbols
49
137
  intern_declared_terminals
50
138
  intern_user_nonterminals
139
+ parser_contract = normalized_parser_contract
51
140
  normalize_user_productions
141
+ expand_inline_rules
142
+ validate_value_printers
143
+ validate_recovery_declarations
52
144
  validate_grammar
53
- IR::Grammar.new(class_name: @ast.class_name, superclass: @ast.superclass, start: @start_name,
54
- expect: @expected_conflicts, options: @options, symbols: @symbols,
55
- productions: @productions, user_code: normalized_user_code,
56
- conversions: @conversions, warnings: @warnings, user_code_chunks: normalized_user_code_chunks)
145
+ build_grammar(parser_contract)
57
146
  end
58
147
 
59
148
  private
60
149
 
150
+ # @rbs (Symbol name, Object value) -> void
151
+ def validate_positive_limit!(name, value)
152
+ return if value.is_a?(Integer) && value.positive?
153
+
154
+ raise ArgumentError, "#{name} must be a positive Integer"
155
+ end
156
+
61
157
  # @rbs () -> void
62
158
  def intern_reserved_symbols
63
159
  intern("$eof", :terminal, reserved: true)
@@ -72,16 +168,62 @@ module Ibex
72
168
 
73
169
  # @rbs () -> void
74
170
  def intern_user_nonterminals
75
- @ast.rules.each { |rule| intern(rule.lhs, :nonterminal, location: rule.loc.to_h) }
76
- @start_name = @explicit_start || @ast.rules.first&.lhs
77
- fail_at(@ast.loc, "grammar has no start rule") unless @start_name
78
- return if symbol(@start_name)&.nonterminal?
171
+ @ast.rules.reject { |rule| parameterized_rule?(rule) }.each do |rule|
172
+ intern_user_nonterminal(rule)
173
+ end
174
+ @start_names = normalized_start_names
175
+ fail_at(@ast.loc, "grammar has no start rule") if @start_names.empty?
176
+ @start_name = @start_names.fetch(0)
177
+ @start_names.each do |name|
178
+ next if symbol(name)&.nonterminal?
79
179
 
80
- fail_at(@start_location || @ast.loc, "undefined start symbol #{@start_name}")
180
+ fail_at(@start_location || @ast.loc, "undefined start symbol #{name}")
181
+ end
81
182
  end
82
183
 
83
- # @rbs (String name, Symbol kind, ?reserved: bool, ?location: IR::location?) -> IR::GrammarSymbol
84
- def intern(name, kind, reserved: false, location: nil)
184
+ # @rbs () -> Array[String]
185
+ def normalized_start_names
186
+ explicit = @explicit_starts
187
+ return explicit if explicit
188
+
189
+ candidate = @ast.rules.find { |rule| start_rule_candidate?(rule) }&.lhs
190
+ candidate ? [candidate] : []
191
+ end
192
+
193
+ # @rbs (Frontend::AST::Rule rule) -> bool
194
+ def start_rule_candidate?(rule)
195
+ !parameterized_rule?(rule) && !rule.inline
196
+ end
197
+
198
+ # @rbs (Frontend::AST::Rule rule) -> void
199
+ def intern_user_nonterminal(rule)
200
+ definition = intern(
201
+ rule.lhs, :nonterminal, location: rule.loc.to_h, documentation: @rule_documentation[rule.lhs]
202
+ )
203
+ return unless rule.inline
204
+
205
+ @inline_symbol_ids << definition.id
206
+ @inline_rule_by_symbol[definition.id] = rule.lhs
207
+ end
208
+
209
+ # @rbs () -> Hash[String, String]
210
+ def validated_rule_documentation
211
+ documentation = {} #: Hash[String, String]
212
+ @ast.rules.each do |rule|
213
+ text = rule.documentation
214
+ next unless text
215
+
216
+ existing = documentation[rule.lhs]
217
+ fail_at(rule.loc, "conflicting documentation for rule #{rule.lhs}") if existing && existing != text
218
+
219
+ documentation[rule.lhs] ||= text
220
+ end
221
+ documentation
222
+ end
223
+
224
+ # @rbs (String name, Symbol kind, ?reserved: bool, ?location: IR::location?,
225
+ # ?documentation: String?, ?metadata_name: String?) -> IR::GrammarSymbol
226
+ def intern(name, kind, reserved: false, location: nil, documentation: nil, metadata_name: nil)
85
227
  existing = symbol(name)
86
228
  if existing
87
229
  fail_hash(location, "symbol #{name} is both terminal and nonterminal") if existing.kind != kind
@@ -89,8 +231,11 @@ module Ibex
89
231
  end
90
232
 
91
233
  precedence = @precedence[name]
234
+ metadata_key = metadata_name || name
92
235
  definition = IR::GrammarSymbol.new(id: @symbols.length, name: name, kind: kind, reserved: reserved,
93
- precedence: precedence, location: location)
236
+ precedence: precedence, location: location,
237
+ display_name: @display_names[metadata_key],
238
+ semantic_type: @semantic_types[metadata_key], documentation: documentation)
94
239
  @symbols << definition
95
240
  @symbols_by_name[name] = definition
96
241
  definition
@@ -110,6 +255,9 @@ module Ibex
110
255
  def symbol_for_reference(reference)
111
256
  existing = symbol(reference.name)
112
257
  return existing if existing
258
+
259
+ fail_at(reference.loc, "parameterized rule #{reference.name} requires arguments") if
260
+ parameter_template?(reference.name)
113
261
  return undefined_nonterminal(reference) if nonterminal_name?(reference.name)
114
262
 
115
263
  warn_undeclared_terminal(reference)
@@ -162,5 +310,12 @@ module Ibex
162
310
  rendered = location ? "#{location[:file]}:#{location[:line]}:#{location[:column]}" : "(grammar):1:1"
163
311
  raise Ibex::Error, "#{rendered}: #{message}"
164
312
  end
313
+
314
+ # @rbs () -> IR::production_expansion?
315
+ def resolved_expansion
316
+ return unless @resolution || @current_parameter_expansion
317
+
318
+ { parameter: @current_parameter_expansion, inline: nil, include_chain: @current_include_chain }
319
+ end
165
320
  end
166
321
  end
@@ -0,0 +1,122 @@
1
+ # frozen_string_literal: true
2
+ # rbs_inline: enabled
3
+
4
+ module Ibex
5
+ module RaccMigration
6
+ # Checks syntax, normalization, superclass, and runtime-coupling hazards.
7
+ # Semantic action code is treated as opaque data and never executed.
8
+ class Checker
9
+ RACC_REQUIRE = %r{(?:require|require_relative)\s*\(?\s*["']racc(?:/parser)?["']} #: Regexp
10
+ RACC_CONSTANT = /\bRacc::[A-Za-z_]\w*/ #: Regexp
11
+
12
+ # @rbs (String source, file: String) -> Report
13
+ def check(source, file:)
14
+ result = Frontend::Parser.new(source, file: file, mode: :default).parse_with_diagnostics
15
+ findings = result.diagnostics.map { |diagnostic| syntax_finding(diagnostic) }
16
+ ast = result.ast
17
+ return Report.new(file: file, class_name: nil, findings: findings) unless ast
18
+
19
+ findings.concat(superclass_findings(ast))
20
+ findings.concat(user_code_findings(ast))
21
+ findings.concat(normalization_findings(ast))
22
+ Report.new(file: file, class_name: ast.class_name, findings: sort_findings(findings))
23
+ end
24
+
25
+ private
26
+
27
+ # @rbs (Frontend::Diagnostic diagnostic) -> Finding
28
+ def syntax_finding(diagnostic)
29
+ Finding.new(
30
+ code: "racc.syntax",
31
+ severity: :error,
32
+ message: diagnostic.message,
33
+ location: diagnostic.location,
34
+ suggestion: "keep the grammar in default mode, or adopt the reported extension explicitly " \
35
+ "with `pragma extended`"
36
+ )
37
+ end
38
+
39
+ # @rbs (Frontend::AST::Root ast) -> Array[Finding]
40
+ def superclass_findings(ast)
41
+ return [] unless ast.superclass&.start_with?("Racc::")
42
+
43
+ [
44
+ Finding.new(
45
+ code: "racc.runtime_superclass",
46
+ severity: :error,
47
+ message: "generated class inherits #{ast.superclass}",
48
+ location: ast.loc,
49
+ suggestion: "remove the explicit Racc superclass or generate with `--superclass=Ibex::Runtime::Parser`"
50
+ )
51
+ ]
52
+ end
53
+
54
+ # @rbs (Frontend::AST::Root ast) -> Array[Finding]
55
+ def user_code_findings(ast)
56
+ ast.user_code.values.flatten.flat_map do |chunk|
57
+ findings = [] #: Array[Finding]
58
+ if chunk.code.match?(RACC_REQUIRE)
59
+ findings << Finding.new(
60
+ code: "racc.runtime_require",
61
+ severity: :warning,
62
+ message: "user code requires the racc runtime",
63
+ location: chunk.loc,
64
+ suggestion: "remove this require after switching generated parsers to the Ibex runtime"
65
+ )
66
+ end
67
+ if chunk.code.match?(RACC_CONSTANT)
68
+ findings << Finding.new(
69
+ code: "racc.runtime_constant",
70
+ severity: :warning,
71
+ message: "user code references a Racc runtime constant",
72
+ location: chunk.loc,
73
+ suggestion: "replace the runtime-specific constant or keep an explicit compatibility adapter"
74
+ )
75
+ end
76
+ findings
77
+ end
78
+ end
79
+
80
+ # @rbs (Frontend::AST::Root ast) -> Array[Finding]
81
+ def normalization_findings(ast)
82
+ Normalizer.new(ast, mode: :default).normalize
83
+ []
84
+ rescue Ibex::Error => e
85
+ location, message = error_location(e.message, ast.loc)
86
+ [
87
+ Finding.new(
88
+ code: "racc.normalization",
89
+ severity: :error,
90
+ message: message,
91
+ location: location,
92
+ suggestion: "apply the diagnostic before generating either parser"
93
+ )
94
+ ]
95
+ end
96
+
97
+ # @rbs (String message, Frontend::Location fallback) -> [Frontend::Location, String]
98
+ def error_location(message, fallback)
99
+ match = message.match(/\A(.+):(\d+):(\d+): (.*)\z/m)
100
+ return [fallback, message] unless match
101
+
102
+ [
103
+ Frontend::Location.new(
104
+ file: match[1].to_s,
105
+ line: Integer(match[2].to_s, 10),
106
+ column: Integer(match[3].to_s, 10)
107
+ ),
108
+ match[4].to_s
109
+ ]
110
+ end
111
+
112
+ # @rbs (Array[Finding] findings) -> Array[Finding]
113
+ def sort_findings(findings)
114
+ rank = { error: 0, warning: 1, info: 2 }
115
+ findings.sort_by do |finding|
116
+ location = finding.location
117
+ [location&.file || "", location&.line || 0, location&.column || 0, rank.fetch(finding.severity), finding.code]
118
+ end
119
+ end
120
+ end
121
+ end
122
+ end
@@ -0,0 +1,177 @@
1
+ # frozen_string_literal: true
2
+ # rbs_inline: enabled
3
+
4
+ module Ibex
5
+ module RaccMigration
6
+ # Emits, but never executes, a subprocess-separated differential harness.
7
+ class Harness
8
+ CLASS_NAME = /\A[A-Z]\w*(?:::[A-Z]\w*)*\z/ #: Regexp
9
+
10
+ # @rbs (String class_name) -> String
11
+ def self.generate(class_name)
12
+ raise ArgumentError, "invalid parser class name #{class_name.inspect}" unless class_name.match?(CLASS_NAME)
13
+
14
+ template.sub("__PARSER_CLASS_NAME__", class_name.dump)
15
+ end
16
+
17
+ # @rbs () -> String
18
+ def self.template
19
+ <<~'RUBY'
20
+ #!/usr/bin/env ruby
21
+ # frozen_string_literal: true
22
+
23
+ # Generated migration differential harness.
24
+ # Running this file executes both generated parsers and all grammar
25
+ # user code in child Ruby processes. Review the grammar and CASES,
26
+ # then run this only in an isolation boundary you trust.
27
+
28
+ require "rbconfig"
29
+ require "tempfile"
30
+ require "tmpdir"
31
+
32
+ PARSER_CLASS = __PARSER_CLASS_NAME__
33
+ TIMEOUT_SECONDS = 15
34
+ MAX_OUTPUT_BYTES = 1_048_576
35
+ CASES = [
36
+ # { name: "example", tokens: [[:TOKEN, "value"], ["+", nil]] }
37
+ ].freeze
38
+
39
+ CHILD_RUNNER = <<~'CHILD'
40
+ begin
41
+ load ARGV.fetch(0)
42
+ parser_class = ARGV.fetch(1).split("::").inject(Object) { |scope, name| scope.const_get(name, false) }
43
+ tokens = Marshal.load(ARGV.fetch(2).unpack1("m0"))
44
+ parser = parser_class.new
45
+ parser.define_singleton_method(:next_token) { tokens.shift }
46
+ result = parser.do_parse
47
+ puts "IBEX_HARNESS_RESULT=#{[Marshal.dump([:ok, result])].pack("m0")}"
48
+ rescue StandardError => error
49
+ payload = [:error, error.class.name, error.message]
50
+ puts "IBEX_HARNESS_RESULT=#{[Marshal.dump(payload)].pack("m0")}"
51
+ end
52
+ CHILD
53
+
54
+ def terminate(pid)
55
+ Process.kill("TERM", pid)
56
+ rescue Errno::ESRCH
57
+ nil
58
+ ensure
59
+ deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 0.5
60
+ while Process.clock_gettime(Process::CLOCK_MONOTONIC) < deadline
61
+ waited = Process.waitpid2(pid, Process::WNOHANG)
62
+ return waited.last if waited
63
+
64
+ sleep 0.01
65
+ end
66
+ Process.kill("KILL", pid)
67
+ Process.waitpid2(pid).last
68
+ end
69
+
70
+ def run_command(*command)
71
+ Tempfile.create("migration-stdout") do |stdout|
72
+ Tempfile.create("migration-stderr") do |stderr|
73
+ pid = Process.spawn(*command, out: stdout, err: stderr)
74
+ deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + TIMEOUT_SECONDS
75
+ status = nil
76
+ until status
77
+ waited = Process.waitpid2(pid, Process::WNOHANG)
78
+ status = waited.last if waited
79
+ break if status
80
+ if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
81
+ status = terminate(pid)
82
+ stdout.rewind
83
+ stderr.rewind
84
+ return [
85
+ stdout.read(MAX_OUTPUT_BYTES) || "", stderr.read(MAX_OUTPUT_BYTES) || "", status, true, false
86
+ ]
87
+ end
88
+ if stdout.size > MAX_OUTPUT_BYTES || stderr.size > MAX_OUTPUT_BYTES
89
+ status = terminate(pid)
90
+ stdout.rewind
91
+ stderr.rewind
92
+ return [
93
+ stdout.read(MAX_OUTPUT_BYTES) || "", stderr.read(MAX_OUTPUT_BYTES) || "", status, false, true
94
+ ]
95
+ end
96
+ sleep 0.01
97
+ end
98
+ stdout.rewind
99
+ stderr.rewind
100
+ [
101
+ stdout.read(MAX_OUTPUT_BYTES) || "", stderr.read(MAX_OUTPUT_BYTES) || "", status, false, false
102
+ ]
103
+ end
104
+ end
105
+ end
106
+
107
+ def compile!(label, *command)
108
+ stdout, stderr, status, timed_out, output_limited = run_command(*command)
109
+ abort "#{label} generation timed out" if timed_out
110
+ abort "#{label} generation exceeded #{MAX_OUTPUT_BYTES} output bytes" if output_limited
111
+ return if status.success?
112
+
113
+ abort "#{label} generation failed\n#{stdout}#{stderr}"
114
+ end
115
+
116
+ def observe(parser_path, tokens)
117
+ payload = [Marshal.dump(tokens)].pack("m0")
118
+ stdout, stderr, status, timed_out, output_limited = run_command(
119
+ RbConfig.ruby, "-e", CHILD_RUNNER, parser_path, PARSER_CLASS, payload
120
+ )
121
+ return { timeout: true } if timed_out
122
+ return { output_limit: MAX_OUTPUT_BYTES } if output_limited
123
+
124
+ lines = stdout.lines
125
+ marker = lines.reverse.find { |line| line.start_with?("IBEX_HARNESS_RESULT=") }
126
+ unless status.success? && marker
127
+ return {
128
+ infrastructure_error: File.basename(parser_path),
129
+ status: status.exitstatus,
130
+ stdout: stdout,
131
+ stderr: stderr
132
+ }
133
+ end
134
+ outcome = marker && Marshal.load(marker.split("=", 2).last.strip.unpack1("m0"))
135
+ {
136
+ status: status.exitstatus,
137
+ outcome: outcome,
138
+ stdout: lines.reject { |line| line.equal?(marker) }.join,
139
+ stderr: stderr
140
+ }
141
+ end
142
+
143
+ abort "add at least one reviewed entry to CASES before running the harness" if CASES.empty?
144
+ grammar = File.expand_path(ARGV.fetch(0))
145
+ abort "grammar file does not exist: #{grammar}" unless File.file?(grammar)
146
+ racc = ENV.fetch("RACC", "racc")
147
+ ibex = ENV.fetch("IBEX", "ibex")
148
+
149
+ failures = 0
150
+ Dir.mktmpdir("ibex-migration-harness-") do |directory|
151
+ racc_parser = File.join(directory, "racc_parser.rb")
152
+ ibex_parser = File.join(directory, "ibex_parser.rb")
153
+ compile!("racc", racc, "-o", racc_parser, grammar)
154
+ compile!("ibex", ibex, "-E", "-o", ibex_parser, grammar)
155
+
156
+ CASES.each do |test_case|
157
+ name = test_case.fetch(:name)
158
+ tokens = test_case.fetch(:tokens)
159
+ expected = observe(racc_parser, tokens)
160
+ actual = observe(ibex_parser, tokens)
161
+ if expected == actual
162
+ puts "ok #{name}"
163
+ else
164
+ failures += 1
165
+ warn "difference #{name}"
166
+ warn " racc: #{expected.inspect}"
167
+ warn " ibex: #{actual.inspect}"
168
+ end
169
+ end
170
+ end
171
+ exit(failures.zero? ? 0 : 1)
172
+ RUBY
173
+ end
174
+ private_class_method :template
175
+ end
176
+ end
177
+ end
@@ -0,0 +1,97 @@
1
+ # frozen_string_literal: true
2
+ # rbs_inline: enabled
3
+
4
+ require "json"
5
+
6
+ module Ibex
7
+ module RaccMigration
8
+ # @rbs!
9
+ # type json_value = String | Integer | Float | bool | nil | Array[json_value] | Hash[String, json_value]
10
+
11
+ # One immutable migration compatibility observation.
12
+ class Finding
13
+ SEVERITIES = %i[error warning info].freeze #: Array[Symbol]
14
+
15
+ attr_reader :code #: String
16
+ attr_reader :severity #: Symbol
17
+ attr_reader :message #: String
18
+ attr_reader :location #: Frontend::Location?
19
+ attr_reader :suggestion #: String
20
+
21
+ # @rbs (code: String, severity: Symbol, message: String, location: Frontend::Location?,
22
+ # suggestion: String) -> void
23
+ def initialize(code:, severity:, message:, location:, suggestion:)
24
+ unless SEVERITIES.include?(severity)
25
+ raise ArgumentError, "unknown migration finding severity #{severity.inspect}"
26
+ end
27
+
28
+ @code = code.dup.freeze
29
+ @severity = severity
30
+ @message = message.dup.freeze
31
+ @location = location
32
+ @suggestion = suggestion.dup.freeze
33
+ freeze
34
+ end
35
+
36
+ # @rbs () -> Hash[String, json_value]
37
+ def to_h
38
+ {
39
+ "code" => @code,
40
+ "severity" => @severity.to_s,
41
+ "message" => @message,
42
+ "location" => @location&.to_h&.transform_keys(&:to_s),
43
+ "suggestion" => @suggestion
44
+ }
45
+ end
46
+ end
47
+
48
+ # Versioned result of checking one grammar without executing its code.
49
+ class Report
50
+ SCHEMA_VERSION = 1 #: Integer
51
+
52
+ attr_reader :file #: String
53
+ attr_reader :class_name #: String?
54
+ attr_reader :findings #: Array[Finding]
55
+
56
+ # @rbs (file: String, class_name: String?, findings: Array[Finding]) -> void
57
+ def initialize(file:, class_name:, findings:)
58
+ @file = file.dup.freeze
59
+ @class_name = class_name&.dup&.freeze
60
+ @findings = findings.dup.freeze
61
+ freeze
62
+ end
63
+
64
+ # @rbs () -> bool
65
+ def compatible? = @findings.none? { |finding| finding.severity == :error }
66
+
67
+ # @rbs () -> Hash[String, json_value]
68
+ def to_h
69
+ {
70
+ "ibex_migration_check" => "racc",
71
+ "schema_version" => SCHEMA_VERSION,
72
+ "compatible" => compatible?,
73
+ "file" => @file,
74
+ "class_name" => @class_name,
75
+ "findings" => @findings.map(&:to_h)
76
+ }
77
+ end
78
+
79
+ # @rbs (*Object?) -> String
80
+ def to_json(*) = JSON.pretty_generate(to_h)
81
+
82
+ # @rbs () -> String
83
+ def to_text
84
+ return "#{@file}: compatible with the checked racc migration surface\n" if @findings.empty?
85
+
86
+ lines = @findings.flat_map do |finding|
87
+ prefix = finding.location ? finding.location.to_s : "#{@file}:1:1"
88
+ [
89
+ "#{prefix}: #{finding.severity}: #{finding.message} [#{finding.code}]",
90
+ " suggestion: #{finding.suggestion}"
91
+ ]
92
+ end
93
+ "#{lines.join("\n")}\n"
94
+ end
95
+ end
96
+ end
97
+ end
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "racc_migration/report"
4
+ require_relative "racc_migration/checker"
5
+ require_relative "racc_migration/harness"
6
+
7
+ module Ibex
8
+ # Static checks and explicitly generated differential tooling for migration
9
+ # from racc grammar files.
10
+ module RaccMigration
11
+ end
12
+ end