ibex 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (384) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +247 -104
  3. data/docs/architecture.md +322 -28
  4. data/docs/cst-migration.md +107 -0
  5. data/docs/cst.md +213 -0
  6. data/docs/development.md +129 -0
  7. data/docs/editor-setup.md +25 -0
  8. data/docs/error-ux.md +60 -0
  9. data/docs/grammar-reference.md +458 -18
  10. data/docs/lexer-migration.md +51 -0
  11. data/docs/racc-migration.md +34 -6
  12. data/docs/release-readiness.md +163 -0
  13. data/docs/stability.md +124 -0
  14. data/examples/README.md +60 -0
  15. data/examples/calculator.y +44 -0
  16. data/examples/csv.y +32 -0
  17. data/examples/ini.y +70 -0
  18. data/examples/json.y +48 -0
  19. data/examples/tiny_language.y +75 -0
  20. data/lib/ibex/analysis/sets.rb +5 -4
  21. data/lib/ibex/artifact_set.rb +63 -0
  22. data/lib/ibex/cli/ambiguity.rb +66 -0
  23. data/lib/ibex/cli/counterexample_options.rb +6 -2
  24. data/lib/ibex/cli/coverage.rb +173 -0
  25. data/lib/ibex/cli/debug.rb +106 -0
  26. data/lib/ibex/cli/diagnostics.rb +116 -0
  27. data/lib/ibex/cli/documentation.rb +67 -0
  28. data/lib/ibex/cli/error_messages.rb +177 -0
  29. data/lib/ibex/cli/explain.rb +76 -0
  30. data/lib/ibex/cli/formatting.rb +386 -0
  31. data/lib/ibex/cli/generation_artifacts.rb +138 -0
  32. data/lib/ibex/cli/generation_error_messages.rb +39 -0
  33. data/lib/ibex/cli/grammar_tests.rb +122 -0
  34. data/lib/ibex/cli/ir_tools.rb +184 -0
  35. data/lib/ibex/cli/lsp.rb +33 -0
  36. data/lib/ibex/cli/outputs.rb +91 -8
  37. data/lib/ibex/cli/racc_migration.rb +87 -0
  38. data/lib/ibex/cli/samples.rb +92 -0
  39. data/lib/ibex/cli/watch.rb +103 -0
  40. data/lib/ibex/cli.rb +506 -37
  41. data/lib/ibex/codegen/action_locations.rb +103 -0
  42. data/lib/ibex/codegen/action_method_source.rb +210 -0
  43. data/lib/ibex/codegen/action_source.rb +118 -0
  44. data/lib/ibex/codegen/ambiguity.rb +173 -0
  45. data/lib/ibex/codegen/cst_metadata.rb +171 -0
  46. data/lib/ibex/codegen/documentation.rb +170 -0
  47. data/lib/ibex/codegen/explain.rb +312 -0
  48. data/lib/ibex/codegen/generated_action_abi.rb +288 -0
  49. data/lib/ibex/codegen/html.rb +94 -10
  50. data/lib/ibex/codegen/mermaid.rb +43 -0
  51. data/lib/ibex/codegen/railroad.rb +197 -0
  52. data/lib/ibex/codegen/railroad_documentation.rb +59 -0
  53. data/lib/ibex/codegen/rbs.rb +323 -2
  54. data/lib/ibex/codegen/report.rb +46 -6
  55. data/lib/ibex/codegen/ruby.rb +334 -59
  56. data/lib/ibex/codegen/ruby_actions.rb +143 -0
  57. data/lib/ibex/codegen/ruby_ast.rb +121 -0
  58. data/lib/ibex/codegen/ruby_error_messages.rb +28 -0
  59. data/lib/ibex/codegen/ruby_lexer.rb +83 -0
  60. data/lib/ibex/codegen/ruby_syntax.rb +90 -0
  61. data/lib/ibex/codegen/ruby_table_metadata.rb +57 -0
  62. data/lib/ibex/codegen/ruby_value_printers.rb +56 -0
  63. data/lib/ibex/codegen/symbol_labels.rb +1 -1
  64. data/lib/ibex/coverage/collector.rb +188 -0
  65. data/lib/ibex/coverage/event_stream.rb +97 -0
  66. data/lib/ibex/coverage/report.rb +259 -0
  67. data/lib/ibex/coverage/runtime_event_validator.rb +160 -0
  68. data/lib/ibex/coverage.rb +13 -0
  69. data/lib/ibex/error_messages/parser.rb +159 -0
  70. data/lib/ibex/error_messages/parser_v2.rb +198 -0
  71. data/lib/ibex/error_messages/renderer.rb +65 -0
  72. data/lib/ibex/error_messages/sentence_search.rb +196 -0
  73. data/lib/ibex/error_messages/update.rb +169 -0
  74. data/lib/ibex/error_messages.rb +165 -0
  75. data/lib/ibex/frontend/ast.rb +145 -6
  76. data/lib/ibex/frontend/bootstrap_parser.rb +47 -4
  77. data/lib/ibex/frontend/diagnostic.rb +81 -0
  78. data/lib/ibex/frontend/diagnostic_recovery.rb +268 -0
  79. data/lib/ibex/frontend/dsl.rb +66 -7
  80. data/lib/ibex/frontend/formatter.rb +407 -0
  81. data/lib/ibex/frontend/generated_parser.rb +614 -190
  82. data/lib/ibex/frontend/generated_parser_base.rb +164 -30
  83. data/lib/ibex/frontend/generated_parser_includes.rb +61 -0
  84. data/lib/ibex/frontend/generated_parser_metadata.rb +61 -0
  85. data/lib/ibex/frontend/generated_parser_parameters.rb +60 -0
  86. data/lib/ibex/frontend/generation.rb +33 -0
  87. data/lib/ibex/frontend/lexer.rb +162 -10
  88. data/lib/ibex/frontend/lexer_recovery.rb +84 -0
  89. data/lib/ibex/frontend/parser/declarations.rb +247 -11
  90. data/lib/ibex/frontend/parser/parameters.rb +82 -0
  91. data/lib/ibex/frontend/parser/rules.rb +43 -6
  92. data/lib/ibex/frontend/parser.rb +182 -4
  93. data/lib/ibex/frontend/regenerator.rb +26 -1
  94. data/lib/ibex/frontend/resolution.rb +69 -0
  95. data/lib/ibex/frontend/resolver.rb +217 -0
  96. data/lib/ibex/frontend/rule_documentation.rb +103 -0
  97. data/lib/ibex/frontend/source_cursor.rb +132 -8
  98. data/lib/ibex/frontend/source_document.rb +229 -0
  99. data/lib/ibex/frontend/source_loader.rb +150 -0
  100. data/lib/ibex/frontend/source_span.rb +81 -0
  101. data/lib/ibex/frontend/token_adapter/declaration_document_state.rb +47 -0
  102. data/lib/ibex/frontend/token_adapter/declaration_lexer_state.rb +83 -0
  103. data/lib/ibex/frontend/token_adapter/declaration_state.rb +216 -26
  104. data/lib/ibex/frontend/token_adapter/delimiter_tracker.rb +8 -2
  105. data/lib/ibex/frontend/token_adapter/rule_state.rb +60 -2
  106. data/lib/ibex/frontend/token_adapter.rb +8 -3
  107. data/lib/ibex/frontend.rb +13 -2
  108. data/lib/ibex/generation_input.rb +57 -0
  109. data/lib/ibex/generation_manifest.rb +200 -0
  110. data/lib/ibex/generation_transaction.rb +261 -0
  111. data/lib/ibex/generation_transaction_recovery.rb +109 -0
  112. data/lib/ibex/generation_transaction_validation.rb +196 -0
  113. data/lib/ibex/grammar_tests.rb +206 -0
  114. data/lib/ibex/ir/automaton_ir.rb +38 -5
  115. data/lib/ibex/ir/grammar_ir.rb +137 -24
  116. data/lib/ibex/ir/lexer_ir.rb +76 -0
  117. data/lib/ibex/ir/migration.rb +120 -0
  118. data/lib/ibex/ir/serialize.rb +110 -19
  119. data/lib/ibex/ir/validator/automaton.rb +345 -0
  120. data/lib/ibex/ir/validator/base.rb +129 -0
  121. data/lib/ibex/ir/validator/grammar.rb +604 -0
  122. data/lib/ibex/ir/validator/lexer.rb +113 -0
  123. data/lib/ibex/ir/validator.rb +62 -0
  124. data/lib/ibex/ir.rb +57 -4
  125. data/lib/ibex/lalr/build_metrics.rb +23 -0
  126. data/lib/ibex/lalr/builder.rb +379 -52
  127. data/lib/ibex/lalr/conflict.rb +1 -0
  128. data/lib/ibex/lalr/conflict_search.rb +11 -5
  129. data/lib/ibex/lalr/counterexample.rb +20 -5
  130. data/lib/ibex/lalr/direct_lookaheads.rb +236 -0
  131. data/lib/ibex/lalr/ielr_partition.rb +152 -0
  132. data/lib/ibex/lalr/on_error_reductions.rb +74 -0
  133. data/lib/ibex/lalr.rb +7 -0
  134. data/lib/ibex/location.rb +129 -0
  135. data/lib/ibex/lsp/document_handlers.rb +66 -0
  136. data/lib/ibex/lsp/document_store.rb +264 -0
  137. data/lib/ibex/lsp/document_store_diagnostics.rb +44 -0
  138. data/lib/ibex/lsp/document_store_validation.rb +61 -0
  139. data/lib/ibex/lsp/initialization_handlers.rb +78 -0
  140. data/lib/ibex/lsp/navigation_handlers.rb +56 -0
  141. data/lib/ibex/lsp/position_codec.rb +109 -0
  142. data/lib/ibex/lsp/protocol_error.rb +33 -0
  143. data/lib/ibex/lsp/request_handlers.rb +45 -0
  144. data/lib/ibex/lsp/request_support.rb +87 -0
  145. data/lib/ibex/lsp/server.rb +146 -0
  146. data/lib/ibex/lsp/symbol_index.rb +241 -0
  147. data/lib/ibex/lsp/symbol_index_builder.rb +267 -0
  148. data/lib/ibex/lsp/symbol_index_precedence_references.rb +44 -0
  149. data/lib/ibex/lsp/symbol_index_source_queries.rb +61 -0
  150. data/lib/ibex/lsp/symbol_occurrence.rb +17 -0
  151. data/lib/ibex/lsp/transport.rb +118 -0
  152. data/lib/ibex/lsp/workspace.rb +127 -0
  153. data/lib/ibex/lsp/workspace_analyzer.rb +199 -0
  154. data/lib/ibex/lsp.rb +32 -0
  155. data/lib/ibex/normalize/declarations.rb +140 -7
  156. data/lib/ibex/normalize/diagnostics.rb +50 -5
  157. data/lib/ibex/normalize/expander.rb +59 -55
  158. data/lib/ibex/normalize/expression.rb +60 -39
  159. data/lib/ibex/normalize/inline_expansion.rb +414 -0
  160. data/lib/ibex/normalize/inline_validation.rb +174 -0
  161. data/lib/ibex/normalize/lexer.rb +131 -0
  162. data/lib/ibex/normalize/named_references.rb +60 -0
  163. data/lib/ibex/normalize/nodes.rb +46 -0
  164. data/lib/ibex/normalize/parameter_ebnf_lowering.rb +69 -0
  165. data/lib/ibex/normalize/parameter_lowering.rb +126 -0
  166. data/lib/ibex/normalize/parameter_substitution.rb +125 -0
  167. data/lib/ibex/normalize/parameter_validation.rb +140 -0
  168. data/lib/ibex/normalize/parameters.rb +199 -0
  169. data/lib/ibex/normalize/recovery_declarations.rb +84 -0
  170. data/lib/ibex/normalize.rb +179 -14
  171. data/lib/ibex/racc_migration/checker.rb +122 -0
  172. data/lib/ibex/racc_migration/harness.rb +177 -0
  173. data/lib/ibex/racc_migration/report.rb +94 -0
  174. data/lib/ibex/racc_migration.rb +12 -0
  175. data/lib/ibex/rake_task.rb +116 -0
  176. data/lib/ibex/samples.rb +186 -0
  177. data/lib/ibex/table_simulation/result.rb +51 -0
  178. data/lib/ibex/table_simulation/simulator.rb +253 -0
  179. data/lib/ibex/table_simulation/step.rb +60 -0
  180. data/lib/ibex/table_simulation/text.rb +31 -0
  181. data/lib/ibex/table_simulation.rb +13 -0
  182. data/lib/ibex/tables.rb +9 -70
  183. data/lib/ibex/version.rb +1 -1
  184. data/lib/ibex/watch/runner.rb +172 -0
  185. data/lib/ibex/watch/source_snapshot.rb +93 -0
  186. data/lib/ibex/watch.rb +11 -0
  187. data/lib/ibex.rb +25 -1
  188. data/schema/automaton-ir-v1.schema.json +401 -0
  189. data/schema/automaton-ir-v2.schema.json +58 -0
  190. data/schema/benchmark-v1.schema.json +212 -0
  191. data/schema/benchmark-v2.schema.json +61 -0
  192. data/schema/cst-v1.json +170 -0
  193. data/schema/error-ux-v1.schema.json +258 -0
  194. data/schema/explain-v1.schema.json +433 -0
  195. data/schema/frontend-diagnostics-v1.schema.json +154 -0
  196. data/schema/generation-manifest-v1.schema.json +115 -0
  197. data/schema/grammar-ir-v1.schema.json +426 -0
  198. data/schema/grammar-ir-v2.schema.json +779 -0
  199. data/schema/lexer-ir-v1.schema.json +215 -0
  200. data/schema/migration-check-v1.schema.json +60 -0
  201. data/schema/performance-comparison-v1.schema.json +395 -0
  202. data/schema/public-performance-comparison-v1.schema.json +506 -0
  203. data/schema/public-performance-profile-v1.schema.json +360 -0
  204. data/schema/runtime-coverage-v1.schema.json +86 -0
  205. data/schema/runtime-event-v1.schema.json +308 -0
  206. data/schema/table-simulation-v1.schema.json +70 -0
  207. data/sig/ibex/artifact_set.rbs +37 -0
  208. data/sig/ibex/cli/ambiguity.rbs +22 -0
  209. data/sig/ibex/cli/counterexample_options.rbs +2 -0
  210. data/sig/ibex/cli/coverage.rbs +53 -0
  211. data/sig/ibex/cli/debug.rbs +28 -0
  212. data/sig/ibex/cli/diagnostics.rbs +38 -0
  213. data/sig/ibex/cli/documentation.rbs +25 -0
  214. data/sig/ibex/cli/error_messages.rbs +57 -0
  215. data/sig/ibex/cli/explain.rbs +25 -0
  216. data/sig/ibex/cli/formatting.rbs +103 -0
  217. data/sig/ibex/cli/generation_artifacts.rbs +50 -0
  218. data/sig/ibex/cli/generation_error_messages.rbs +19 -0
  219. data/sig/ibex/cli/grammar_tests.rbs +36 -0
  220. data/sig/ibex/cli/ir_tools.rbs +51 -0
  221. data/sig/ibex/cli/lsp.rbs +14 -0
  222. data/sig/ibex/cli/outputs.rbs +17 -0
  223. data/sig/ibex/cli/racc_migration.rbs +30 -0
  224. data/sig/ibex/cli/samples.rbs +30 -0
  225. data/sig/ibex/cli/watch.rbs +43 -0
  226. data/sig/ibex/cli.rbs +104 -5
  227. data/sig/ibex/codegen/action_locations.rbs +45 -0
  228. data/sig/ibex/codegen/action_method_source.rbs +65 -0
  229. data/sig/ibex/codegen/action_source.rbs +50 -0
  230. data/sig/ibex/codegen/ambiguity.rbs +60 -0
  231. data/sig/ibex/codegen/cst_metadata.rbs +59 -0
  232. data/sig/ibex/codegen/documentation.rbs +50 -0
  233. data/sig/ibex/codegen/explain.rbs +85 -0
  234. data/sig/ibex/codegen/generated_action_abi.rbs +101 -0
  235. data/sig/ibex/codegen/html.rbs +18 -2
  236. data/sig/ibex/codegen/mermaid.rbs +16 -0
  237. data/sig/ibex/codegen/railroad.rbs +82 -0
  238. data/sig/ibex/codegen/railroad_documentation.rbs +31 -0
  239. data/sig/ibex/codegen/rbs.rbs +84 -4
  240. data/sig/ibex/codegen/report.rbs +8 -0
  241. data/sig/ibex/codegen/ruby.rbs +97 -23
  242. data/sig/ibex/codegen/ruby_actions.rbs +54 -0
  243. data/sig/ibex/codegen/ruby_ast.rbs +34 -0
  244. data/sig/ibex/codegen/ruby_error_messages.rbs +16 -0
  245. data/sig/ibex/codegen/ruby_lexer.rbs +25 -0
  246. data/sig/ibex/codegen/ruby_syntax.rbs +25 -0
  247. data/sig/ibex/codegen/ruby_table_metadata.rbs +26 -0
  248. data/sig/ibex/codegen/ruby_value_printers.rbs +28 -0
  249. data/sig/ibex/coverage/collector.rbs +76 -0
  250. data/sig/ibex/coverage/event_stream.rbs +42 -0
  251. data/sig/ibex/coverage/report.rbs +100 -0
  252. data/sig/ibex/coverage/runtime_event_validator.rbs +68 -0
  253. data/sig/ibex/coverage.rbs +7 -0
  254. data/sig/ibex/error_messages/parser.rbs +58 -0
  255. data/sig/ibex/error_messages/parser_v2.rbs +67 -0
  256. data/sig/ibex/error_messages/renderer.rbs +23 -0
  257. data/sig/ibex/error_messages/sentence_search.rbs +80 -0
  258. data/sig/ibex/error_messages/update.rbs +43 -0
  259. data/sig/ibex/error_messages.rbs +85 -0
  260. data/sig/ibex/frontend/ast.rbs +208 -19
  261. data/sig/ibex/frontend/bootstrap_parser.rbs +11 -0
  262. data/sig/ibex/frontend/diagnostic.rbs +53 -0
  263. data/sig/ibex/frontend/diagnostic_recovery.rbs +98 -0
  264. data/sig/ibex/frontend/dsl.rbs +33 -4
  265. data/sig/ibex/frontend/formatter.rbs +135 -0
  266. data/sig/ibex/frontend/generated_parser.rbs +218 -68
  267. data/sig/ibex/frontend/generated_parser_base.rbs +61 -10
  268. data/sig/ibex/frontend/generated_parser_includes.rbs +23 -0
  269. data/sig/ibex/frontend/generated_parser_metadata.rbs +23 -0
  270. data/sig/ibex/frontend/generated_parser_parameters.rbs +24 -0
  271. data/sig/ibex/frontend/generation.rbs +6 -0
  272. data/sig/ibex/frontend/lexer.rbs +49 -2
  273. data/sig/ibex/frontend/lexer_recovery.rbs +25 -0
  274. data/sig/ibex/frontend/parser/declarations.rbs +54 -0
  275. data/sig/ibex/frontend/parser/parameters.rbs +28 -0
  276. data/sig/ibex/frontend/parser/rules.rbs +3 -0
  277. data/sig/ibex/frontend/parser.rbs +66 -0
  278. data/sig/ibex/frontend/regenerator.rbs +11 -0
  279. data/sig/ibex/frontend/resolution.rbs +33 -0
  280. data/sig/ibex/frontend/resolver.rbs +91 -0
  281. data/sig/ibex/frontend/rule_documentation.rbs +42 -0
  282. data/sig/ibex/frontend/source_cursor.rbs +36 -3
  283. data/sig/ibex/frontend/source_document.rbs +119 -0
  284. data/sig/ibex/frontend/source_loader.rbs +66 -0
  285. data/sig/ibex/frontend/source_span.rbs +53 -0
  286. data/sig/ibex/frontend/token_adapter/declaration_document_state.rbs +21 -0
  287. data/sig/ibex/frontend/token_adapter/declaration_lexer_state.rbs +27 -0
  288. data/sig/ibex/frontend/token_adapter/declaration_state.rbs +73 -5
  289. data/sig/ibex/frontend/token_adapter/rule_state.rbs +20 -0
  290. data/sig/ibex/frontend/token_adapter.rbs +5 -2
  291. data/sig/ibex/frontend.rbs +1 -1
  292. data/sig/ibex/generation_input.rbs +37 -0
  293. data/sig/ibex/generation_manifest.rbs +67 -0
  294. data/sig/ibex/generation_transaction.rbs +82 -0
  295. data/sig/ibex/generation_transaction_recovery.rbs +36 -0
  296. data/sig/ibex/generation_transaction_validation.rbs +65 -0
  297. data/sig/ibex/grammar_tests.rbs +93 -0
  298. data/sig/ibex/ir/automaton_ir.rbs +8 -2
  299. data/sig/ibex/ir/grammar_ir.rbs +75 -15
  300. data/sig/ibex/ir/lexer_ir.rbs +57 -0
  301. data/sig/ibex/ir/migration.rbs +34 -0
  302. data/sig/ibex/ir/serialize.rbs +23 -6
  303. data/sig/ibex/ir/validator/automaton.rbs +109 -0
  304. data/sig/ibex/ir/validator/base.rbs +65 -0
  305. data/sig/ibex/ir/validator/grammar.rbs +184 -0
  306. data/sig/ibex/ir/validator/lexer.rbs +37 -0
  307. data/sig/ibex/ir/validator.rbs +16 -0
  308. data/sig/ibex/ir.rbs +38 -4
  309. data/sig/ibex/lalr/build_metrics.rbs +20 -0
  310. data/sig/ibex/lalr/builder.rbs +95 -15
  311. data/sig/ibex/lalr/conflict_search.rbs +7 -3
  312. data/sig/ibex/lalr/counterexample.rbs +5 -2
  313. data/sig/ibex/lalr/direct_lookaheads.rbs +86 -0
  314. data/sig/ibex/lalr/ielr_partition.rbs +59 -0
  315. data/sig/ibex/lalr/on_error_reductions.rbs +22 -0
  316. data/sig/ibex/lalr.rbs +6 -0
  317. data/sig/ibex/location.rbs +67 -0
  318. data/sig/ibex/lsp/document_handlers.rbs +24 -0
  319. data/sig/ibex/lsp/document_store.rbs +94 -0
  320. data/sig/ibex/lsp/document_store_diagnostics.rbs +20 -0
  321. data/sig/ibex/lsp/document_store_validation.rbs +26 -0
  322. data/sig/ibex/lsp/initialization_handlers.rbs +30 -0
  323. data/sig/ibex/lsp/navigation_handlers.rbs +30 -0
  324. data/sig/ibex/lsp/position_codec.rbs +40 -0
  325. data/sig/ibex/lsp/protocol_error.rbs +32 -0
  326. data/sig/ibex/lsp/request_handlers.rbs +26 -0
  327. data/sig/ibex/lsp/request_support.rbs +40 -0
  328. data/sig/ibex/lsp/server.rbs +55 -0
  329. data/sig/ibex/lsp/symbol_index.rbs +78 -0
  330. data/sig/ibex/lsp/symbol_index_builder.rbs +87 -0
  331. data/sig/ibex/lsp/symbol_index_precedence_references.rbs +18 -0
  332. data/sig/ibex/lsp/symbol_index_source_queries.rbs +26 -0
  333. data/sig/ibex/lsp/symbol_occurrence.rbs +25 -0
  334. data/sig/ibex/lsp/transport.rbs +35 -0
  335. data/sig/ibex/lsp/workspace.rbs +41 -0
  336. data/sig/ibex/lsp/workspace_analyzer.rbs +69 -0
  337. data/sig/ibex/lsp.rbs +7 -0
  338. data/sig/ibex/normalize/declarations.rbs +31 -0
  339. data/sig/ibex/normalize/diagnostics.rbs +9 -0
  340. data/sig/ibex/normalize/expander.rbs +20 -14
  341. data/sig/ibex/normalize/expression.rbs +10 -10
  342. data/sig/ibex/normalize/inline_expansion.rbs +120 -0
  343. data/sig/ibex/normalize/inline_validation.rbs +40 -0
  344. data/sig/ibex/normalize/lexer.rbs +37 -0
  345. data/sig/ibex/normalize/named_references.rbs +20 -0
  346. data/sig/ibex/normalize/nodes.rbs +14 -0
  347. data/sig/ibex/normalize/parameter_ebnf_lowering.rbs +32 -0
  348. data/sig/ibex/normalize/parameter_lowering.rbs +35 -0
  349. data/sig/ibex/normalize/parameter_substitution.rbs +42 -0
  350. data/sig/ibex/normalize/parameter_validation.rbs +44 -0
  351. data/sig/ibex/normalize/parameters.rbs +46 -0
  352. data/sig/ibex/normalize/recovery_declarations.rbs +24 -0
  353. data/sig/ibex/normalize.rbs +123 -18
  354. data/sig/ibex/racc_migration/checker.rbs +36 -0
  355. data/sig/ibex/racc_migration/harness.rbs +16 -0
  356. data/sig/ibex/racc_migration/report.rbs +53 -0
  357. data/sig/ibex/racc_migration.rbs +8 -0
  358. data/sig/ibex/rake_task.rbs +51 -0
  359. data/sig/ibex/samples.rbs +51 -0
  360. data/sig/ibex/table_simulation/result.rbs +32 -0
  361. data/sig/ibex/table_simulation/simulator.rbs +98 -0
  362. data/sig/ibex/table_simulation/step.rbs +40 -0
  363. data/sig/ibex/table_simulation/text.rbs +14 -0
  364. data/sig/ibex/table_simulation.rbs +7 -0
  365. data/sig/ibex/tables.rbs +0 -26
  366. data/sig/ibex/watch/runner.rbs +48 -0
  367. data/sig/ibex/watch/source_snapshot.rbs +42 -0
  368. data/sig/ibex/watch.rbs +7 -0
  369. data/sig/ibex.rbs +2 -0
  370. metadata +301 -16
  371. data/.rubocop.yml +0 -43
  372. data/CHANGELOG.md +0 -30
  373. data/Rakefile +0 -25
  374. data/Steepfile +0 -10
  375. data/docs/compat-notes.md +0 -37
  376. data/docs/lexer-coverage.md +0 -14
  377. data/docs/phase10-extensions.md +0 -27
  378. data/gemfiles/Gemfile +0 -7
  379. data/gemfiles/Gemfile.lock +0 -98
  380. data/lib/ibex/frontend/grammar.y +0 -156
  381. data/lib/ibex/runtime/parser.rb +0 -360
  382. data/lib/ibex/runtime.rb +0 -8
  383. data/sig/ibex/runtime/parser.rbs +0 -167
  384. data/sig/ibex/runtime.rbs +0 -6
@@ -1,21 +1,52 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative "normalize/declarations"
4
+ require_relative "normalize/lexer"
5
+ require_relative "normalize/recovery_declarations"
4
6
  require_relative "normalize/expression"
7
+ require_relative "normalize/parameter_validation"
8
+ require_relative "normalize/inline_validation"
9
+ require_relative "normalize/parameter_substitution"
10
+ require_relative "normalize/parameter_ebnf_lowering"
11
+ require_relative "normalize/parameter_lowering"
12
+ require_relative "normalize/parameters"
13
+ require_relative "normalize/named_references"
14
+ require_relative "normalize/nodes"
5
15
  require_relative "normalize/expander"
16
+ require_relative "normalize/inline_expansion"
6
17
  require_relative "normalize/diagnostics"
18
+ require_relative "frontend/resolution"
7
19
 
8
20
  module Ibex
9
21
  # Converts a frontend AST into immutable Grammar IR.
10
22
  class Normalizer
11
23
  include NormalizeDeclarations
24
+ include NormalizeLexer
25
+ include NormalizeRecoveryDeclarations
26
+ include NormalizeParameterValidation
27
+ include NormalizeInlineValidation
28
+ include NormalizeParameterSubstitution
29
+ include NormalizeParameterEbnfLowering
30
+ include NormalizeParameterLowering
31
+ include NormalizeParameters
32
+ include NormalizeNamedReferences
33
+ include NormalizeNodes
12
34
  include NormalizeExpander
35
+ include NormalizeInlineExpansion
13
36
  include NormalizeDiagnostics
14
37
 
15
38
  RESERVED_NAMES = %w[result val _values].freeze #: Array[String]
39
+ RUBY_KEYWORDS = %w[
40
+ __ENCODING__ __FILE__ __LINE__ alias and begin break case class def defined do else elsif end ensure false for if
41
+ in module next nil not or redo rescue retry return self super then true undef unless until when while yield
42
+ ].freeze #: Array[String]
43
+ DEFAULT_MAX_PARAMETER_SPECIALIZATIONS = 1_000
44
+ DEFAULT_MAX_INLINE_EXPANSIONS = 10_000
16
45
 
17
46
  # @rbs @ast: Frontend::AST::Root
18
- # @rbs @mode: Symbol
47
+ # @rbs @resolution: Frontend::Resolution?
48
+ # @rbs @current_include_chain: Array[IR::source_provenance]
49
+ # @rbs @mode: IR::grammar_mode
19
50
  # @rbs @symbols: Array[IR::GrammarSymbol]
20
51
  # @rbs @symbols_by_name: Hash[String, IR::GrammarSymbol]
21
52
  # @rbs @productions: Array[IR::Production]
@@ -24,40 +55,115 @@ module Ibex
24
55
  # @rbs @declared_tokens: Hash[String, IR::location]
25
56
  # @rbs @precedence: Hash[String, IR::precedence]
26
57
  # @rbs @precedence_locations: Hash[String, IR::location]
58
+ # @rbs @display_names: Hash[String, String]
59
+ # @rbs @display_name_locations: Hash[String, IR::location]
60
+ # @rbs @semantic_types: Hash[String, String]
61
+ # @rbs @semantic_type_locations: Hash[String, IR::location]
27
62
  # @rbs @options: IR::grammar_options
28
63
  # @rbs @expected_conflicts: Integer
64
+ # @rbs @expected_rr_conflicts: Integer?
29
65
  # @rbs @conversions: Hash[String, String]
30
- # @rbs @explicit_start: String?
66
+ # @rbs @parser_parameters: Array[IR::parser_parameter]
67
+ # @rbs @value_printers: Hash[String, IR::value_printer]
68
+ # @rbs @recovery_sync_tokens: Array[String]
69
+ # @rbs @on_error_reduce_groups: Array[Array[String]]
70
+ # @rbs @grammar_tests: Array[IR::grammar_test]
71
+ # @rbs @lexer_declaration: Frontend::AST::Lexer?
72
+ # @rbs @explicit_starts: Array[String]?
73
+ # @rbs @start_names: Array[String]
31
74
  # @rbs @start_name: String
32
75
  # @rbs @start_location: Frontend::Location?
76
+ # @rbs @parameter_templates: Hash[String, Array[Frontend::AST::Rule]]
77
+ # @rbs @parameter_formals: Hash[String, Array[String]]
78
+ # @rbs @parameter_specializations: Hash[[String, Array[String]], String]
79
+ # @rbs @parameter_worklist: Array[Hash[Symbol, untyped]]
80
+ # @rbs @parameter_worklist_active: bool
81
+ # @rbs @max_parameter_specializations: Integer
82
+ # @rbs @current_parameter_expansion: IR::parameter_expansion?
83
+ # @rbs @rule_documentation: Hash[String, String]
84
+ # @rbs @inline_rule_names: Set[String]
85
+ # @rbs @inline_symbol_ids: Set[Integer]
86
+ # @rbs @inline_rule_by_symbol: Hash[Integer, String]
87
+ # @rbs @max_inline_expansions: Integer
88
+ # @rbs @inline_expansion_count: Integer
89
+ # @rbs @node_shapes: Hash[String, Array[String]]
90
+
91
+ # @rbs (Frontend::AST::Root | Frontend::AST::Fragment | Frontend::Resolution input,
92
+ # ?mode: Symbol | String, ?max_parameter_specializations: Integer,
93
+ # ?max_inline_expansions: Integer) -> void
94
+ def initialize(input, mode: :default, max_parameter_specializations: DEFAULT_MAX_PARAMETER_SPECIALIZATIONS,
95
+ max_inline_expansions: DEFAULT_MAX_INLINE_EXPANSIONS)
96
+ validate_positive_limit!(:max_parameter_specializations, max_parameter_specializations)
97
+ validate_positive_limit!(:max_inline_expansions, max_inline_expansions)
98
+ normalized_mode = mode.to_sym
99
+ raise ArgumentError, "mode must be :default or :extended" unless %i[default extended].include?(normalized_mode)
100
+
101
+ @resolution = input if input.is_a?(Frontend::Resolution)
102
+ ast = input.is_a?(Frontend::Resolution) ? input.root : input
103
+ normalized_mode = :extended if ast.is_a?(Frontend::AST::Root) && ast.extended
104
+ fail_at(ast.loc, "fragments must be resolved before normalization") if ast.is_a?(Frontend::AST::Fragment)
105
+ unresolved = ast.declarations.find { |declaration| declaration.is_a?(Frontend::AST::Include) }
106
+ fail_at(unresolved.loc, "includes must be resolved before normalization") if unresolved
33
107
 
34
- # @rbs (Frontend::AST::Root ast, ?mode: Symbol | String) -> void
35
- def initialize(ast, mode: :racc)
36
108
  @ast = ast
37
- @mode = mode
109
+ @mode = normalized_mode #: IR::grammar_mode
38
110
  @symbols = [] #: Array[IR::GrammarSymbol]
39
111
  @symbols_by_name = {} #: Hash[String, IR::GrammarSymbol]
40
112
  @productions = [] #: Array[IR::Production]
41
113
  @warnings = [] #: Array[IR::grammar_warning]
42
114
  @helper_sequence = 0
115
+ @current_include_chain = [] #: Array[IR::source_provenance]
116
+ @parameter_specializations = {} #: Hash[[String, Array[String]], String]
117
+ @parameter_worklist = [] #: Array[Hash[Symbol, untyped]]
118
+ @parameter_worklist_active = false
119
+ @max_parameter_specializations = max_parameter_specializations
120
+ @max_inline_expansions = max_inline_expansions
121
+ @inline_expansion_count = 0
122
+ @inline_symbol_ids = Set.new #: Set[Integer]
123
+ @inline_rule_by_symbol = {} #: Hash[Integer, String]
43
124
  end
44
125
 
45
126
  # @rbs () -> IR::Grammar
46
127
  def normalize
47
128
  read_declarations
129
+ gather_parameter_templates
130
+ gather_inline_rules
48
131
  intern_reserved_symbols
49
132
  intern_declared_terminals
50
133
  intern_user_nonterminals
51
134
  normalize_user_productions
135
+ expand_inline_rules
136
+ validate_value_printers
137
+ validate_recovery_declarations
52
138
  validate_grammar
53
139
  IR::Grammar.new(class_name: @ast.class_name, superclass: @ast.superclass, start: @start_name,
54
140
  expect: @expected_conflicts, options: @options, symbols: @symbols,
141
+ mode: @mode, starts: @start_names,
142
+ expect_rr: @expected_rr_conflicts,
143
+ parser_parameters: @parser_parameters,
144
+ value_printers: @value_printers.values,
145
+ grammar_tests: @grammar_tests,
146
+ lexer: normalize_lexer,
147
+ recovery: {
148
+ sync_tokens: @recovery_sync_tokens,
149
+ on_error_reduce: @on_error_reduce_groups
150
+ },
55
151
  productions: @productions, user_code: normalized_user_code,
56
- conversions: @conversions, warnings: @warnings, user_code_chunks: normalized_user_code_chunks)
152
+ conversions: @conversions, warnings: @warnings, user_code_chunks: normalized_user_code_chunks,
153
+ source_provenance: {
154
+ file: @ast.loc.file, root: @resolution&.root_directory, byte_span: nil
155
+ })
57
156
  end
58
157
 
59
158
  private
60
159
 
160
+ # @rbs (Symbol name, untyped value) -> void
161
+ def validate_positive_limit!(name, value)
162
+ return if value.is_a?(Integer) && value.positive?
163
+
164
+ raise ArgumentError, "#{name} must be a positive Integer"
165
+ end
166
+
61
167
  # @rbs () -> void
62
168
  def intern_reserved_symbols
63
169
  intern("$eof", :terminal, reserved: true)
@@ -72,16 +178,62 @@ module Ibex
72
178
 
73
179
  # @rbs () -> void
74
180
  def intern_user_nonterminals
75
- @ast.rules.each { |rule| intern(rule.lhs, :nonterminal, location: rule.loc.to_h) }
76
- @start_name = @explicit_start || @ast.rules.first&.lhs
77
- fail_at(@ast.loc, "grammar has no start rule") unless @start_name
78
- return if symbol(@start_name)&.nonterminal?
181
+ @ast.rules.reject { |rule| parameterized_rule?(rule) }.each do |rule|
182
+ intern_user_nonterminal(rule)
183
+ end
184
+ @start_names = normalized_start_names
185
+ fail_at(@ast.loc, "grammar has no start rule") if @start_names.empty?
186
+ @start_name = @start_names.fetch(0)
187
+ @start_names.each do |name|
188
+ next if symbol(name)&.nonterminal?
79
189
 
80
- fail_at(@start_location || @ast.loc, "undefined start symbol #{@start_name}")
190
+ fail_at(@start_location || @ast.loc, "undefined start symbol #{name}")
191
+ end
81
192
  end
82
193
 
83
- # @rbs (String name, Symbol kind, ?reserved: bool, ?location: IR::location?) -> IR::GrammarSymbol
84
- def intern(name, kind, reserved: false, location: nil)
194
+ # @rbs () -> Array[String]
195
+ def normalized_start_names
196
+ explicit = @explicit_starts
197
+ return explicit if explicit
198
+
199
+ candidate = @ast.rules.find { |rule| start_rule_candidate?(rule) }&.lhs
200
+ candidate ? [candidate] : []
201
+ end
202
+
203
+ # @rbs (Frontend::AST::Rule rule) -> bool
204
+ def start_rule_candidate?(rule)
205
+ !parameterized_rule?(rule) && !rule.inline
206
+ end
207
+
208
+ # @rbs (Frontend::AST::Rule rule) -> void
209
+ def intern_user_nonterminal(rule)
210
+ definition = intern(
211
+ rule.lhs, :nonterminal, location: rule.loc.to_h, documentation: @rule_documentation[rule.lhs]
212
+ )
213
+ return unless rule.inline
214
+
215
+ @inline_symbol_ids << definition.id
216
+ @inline_rule_by_symbol[definition.id] = rule.lhs
217
+ end
218
+
219
+ # @rbs () -> Hash[String, String]
220
+ def validated_rule_documentation
221
+ documentation = {} #: Hash[String, String]
222
+ @ast.rules.each do |rule|
223
+ text = rule.documentation
224
+ next unless text
225
+
226
+ existing = documentation[rule.lhs]
227
+ fail_at(rule.loc, "conflicting documentation for rule #{rule.lhs}") if existing && existing != text
228
+
229
+ documentation[rule.lhs] ||= text
230
+ end
231
+ documentation
232
+ end
233
+
234
+ # @rbs (String name, Symbol kind, ?reserved: bool, ?location: IR::location?,
235
+ # ?documentation: String?, ?metadata_name: String?) -> IR::GrammarSymbol
236
+ def intern(name, kind, reserved: false, location: nil, documentation: nil, metadata_name: nil)
85
237
  existing = symbol(name)
86
238
  if existing
87
239
  fail_hash(location, "symbol #{name} is both terminal and nonterminal") if existing.kind != kind
@@ -89,8 +241,11 @@ module Ibex
89
241
  end
90
242
 
91
243
  precedence = @precedence[name]
244
+ metadata_key = metadata_name || name
92
245
  definition = IR::GrammarSymbol.new(id: @symbols.length, name: name, kind: kind, reserved: reserved,
93
- precedence: precedence, location: location)
246
+ precedence: precedence, location: location,
247
+ display_name: @display_names[metadata_key],
248
+ semantic_type: @semantic_types[metadata_key], documentation: documentation)
94
249
  @symbols << definition
95
250
  @symbols_by_name[name] = definition
96
251
  definition
@@ -110,6 +265,9 @@ module Ibex
110
265
  def symbol_for_reference(reference)
111
266
  existing = symbol(reference.name)
112
267
  return existing if existing
268
+
269
+ fail_at(reference.loc, "parameterized rule #{reference.name} requires arguments") if
270
+ parameter_template?(reference.name)
113
271
  return undefined_nonterminal(reference) if nonterminal_name?(reference.name)
114
272
 
115
273
  warn_undeclared_terminal(reference)
@@ -162,5 +320,12 @@ module Ibex
162
320
  rendered = location ? "#{location[:file]}:#{location[:line]}:#{location[:column]}" : "(grammar):1:1"
163
321
  raise Ibex::Error, "#{rendered}: #{message}"
164
322
  end
323
+
324
+ # @rbs () -> IR::production_expansion?
325
+ def resolved_expansion
326
+ return unless @resolution || @current_parameter_expansion
327
+
328
+ { parameter: @current_parameter_expansion, inline: nil, include_chain: @current_include_chain }
329
+ end
165
330
  end
166
331
  end
@@ -0,0 +1,122 @@
1
+ # frozen_string_literal: true
2
+ # rbs_inline: enabled
3
+
4
+ module Ibex
5
+ module RaccMigration
6
+ # Checks syntax, normalization, superclass, and runtime-coupling hazards.
7
+ # Semantic action code is treated as opaque data and never executed.
8
+ class Checker
9
+ RACC_REQUIRE = %r{(?:require|require_relative)\s*\(?\s*["']racc(?:/parser)?["']} #: Regexp
10
+ RACC_CONSTANT = /\bRacc::[A-Za-z_]\w*/ #: Regexp
11
+
12
+ # @rbs (String source, file: String) -> Report
13
+ def check(source, file:)
14
+ result = Frontend::Parser.new(source, file: file, mode: :default).parse_with_diagnostics
15
+ findings = result.diagnostics.map { |diagnostic| syntax_finding(diagnostic) }
16
+ ast = result.ast
17
+ return Report.new(file: file, class_name: nil, findings: findings) unless ast
18
+
19
+ findings.concat(superclass_findings(ast))
20
+ findings.concat(user_code_findings(ast))
21
+ findings.concat(normalization_findings(ast))
22
+ Report.new(file: file, class_name: ast.class_name, findings: sort_findings(findings))
23
+ end
24
+
25
+ private
26
+
27
+ # @rbs (Frontend::Diagnostic diagnostic) -> Finding
28
+ def syntax_finding(diagnostic)
29
+ Finding.new(
30
+ code: "racc.syntax",
31
+ severity: :error,
32
+ message: diagnostic.message,
33
+ location: diagnostic.location,
34
+ suggestion: "keep the grammar in default mode, or adopt the reported extension explicitly " \
35
+ "with `pragma extended`"
36
+ )
37
+ end
38
+
39
+ # @rbs (Frontend::AST::Root ast) -> Array[Finding]
40
+ def superclass_findings(ast)
41
+ return [] unless ast.superclass&.start_with?("Racc::")
42
+
43
+ [
44
+ Finding.new(
45
+ code: "racc.runtime_superclass",
46
+ severity: :error,
47
+ message: "generated class inherits #{ast.superclass}",
48
+ location: ast.loc,
49
+ suggestion: "remove the explicit Racc superclass or generate with `--superclass=Ibex::Runtime::Parser`"
50
+ )
51
+ ]
52
+ end
53
+
54
+ # @rbs (Frontend::AST::Root ast) -> Array[Finding]
55
+ def user_code_findings(ast)
56
+ ast.user_code.values.flatten.flat_map do |chunk|
57
+ findings = [] #: Array[Finding]
58
+ if chunk.code.match?(RACC_REQUIRE)
59
+ findings << Finding.new(
60
+ code: "racc.runtime_require",
61
+ severity: :warning,
62
+ message: "user code requires the racc runtime",
63
+ location: chunk.loc,
64
+ suggestion: "remove this require after switching generated parsers to the Ibex runtime"
65
+ )
66
+ end
67
+ if chunk.code.match?(RACC_CONSTANT)
68
+ findings << Finding.new(
69
+ code: "racc.runtime_constant",
70
+ severity: :warning,
71
+ message: "user code references a Racc runtime constant",
72
+ location: chunk.loc,
73
+ suggestion: "replace the runtime-specific constant or keep an explicit compatibility adapter"
74
+ )
75
+ end
76
+ findings
77
+ end
78
+ end
79
+
80
+ # @rbs (Frontend::AST::Root ast) -> Array[Finding]
81
+ def normalization_findings(ast)
82
+ Normalizer.new(ast, mode: :default).normalize
83
+ []
84
+ rescue Ibex::Error => e
85
+ location, message = error_location(e.message, ast.loc)
86
+ [
87
+ Finding.new(
88
+ code: "racc.normalization",
89
+ severity: :error,
90
+ message: message,
91
+ location: location,
92
+ suggestion: "apply the diagnostic before generating either parser"
93
+ )
94
+ ]
95
+ end
96
+
97
+ # @rbs (String message, Frontend::Location fallback) -> [Frontend::Location, String]
98
+ def error_location(message, fallback)
99
+ match = message.match(/\A(.+):(\d+):(\d+): (.*)\z/m)
100
+ return [fallback, message] unless match
101
+
102
+ [
103
+ Frontend::Location.new(
104
+ file: match[1].to_s,
105
+ line: Integer(match[2].to_s, 10),
106
+ column: Integer(match[3].to_s, 10)
107
+ ),
108
+ match[4].to_s
109
+ ]
110
+ end
111
+
112
+ # @rbs (Array[Finding] findings) -> Array[Finding]
113
+ def sort_findings(findings)
114
+ rank = { error: 0, warning: 1, info: 2 }
115
+ findings.sort_by do |finding|
116
+ location = finding.location
117
+ [location&.file || "", location&.line || 0, location&.column || 0, rank.fetch(finding.severity), finding.code]
118
+ end
119
+ end
120
+ end
121
+ end
122
+ end
@@ -0,0 +1,177 @@
1
+ # frozen_string_literal: true
2
+ # rbs_inline: enabled
3
+
4
+ module Ibex
5
+ module RaccMigration
6
+ # Emits, but never executes, a subprocess-separated differential harness.
7
+ class Harness
8
+ CLASS_NAME = /\A[A-Z]\w*(?:::[A-Z]\w*)*\z/ #: Regexp
9
+
10
+ # @rbs (String class_name) -> String
11
+ def self.generate(class_name)
12
+ raise ArgumentError, "invalid parser class name #{class_name.inspect}" unless class_name.match?(CLASS_NAME)
13
+
14
+ template.sub("__PARSER_CLASS_NAME__", class_name.dump)
15
+ end
16
+
17
+ # @rbs () -> String
18
+ def self.template
19
+ <<~'RUBY'
20
+ #!/usr/bin/env ruby
21
+ # frozen_string_literal: true
22
+
23
+ # Generated migration differential harness.
24
+ # Running this file executes both generated parsers and all grammar
25
+ # user code in child Ruby processes. Review the grammar and CASES,
26
+ # then run this only in an isolation boundary you trust.
27
+
28
+ require "rbconfig"
29
+ require "tempfile"
30
+ require "tmpdir"
31
+
32
+ PARSER_CLASS = __PARSER_CLASS_NAME__
33
+ TIMEOUT_SECONDS = 15
34
+ MAX_OUTPUT_BYTES = 1_048_576
35
+ CASES = [
36
+ # { name: "example", tokens: [[:TOKEN, "value"], ["+", nil]] }
37
+ ].freeze
38
+
39
+ CHILD_RUNNER = <<~'CHILD'
40
+ begin
41
+ load ARGV.fetch(0)
42
+ parser_class = ARGV.fetch(1).split("::").inject(Object) { |scope, name| scope.const_get(name, false) }
43
+ tokens = Marshal.load(ARGV.fetch(2).unpack1("m0"))
44
+ parser = parser_class.new
45
+ parser.define_singleton_method(:next_token) { tokens.shift }
46
+ result = parser.do_parse
47
+ puts "IBEX_HARNESS_RESULT=#{[Marshal.dump([:ok, result])].pack("m0")}"
48
+ rescue StandardError => error
49
+ payload = [:error, error.class.name, error.message]
50
+ puts "IBEX_HARNESS_RESULT=#{[Marshal.dump(payload)].pack("m0")}"
51
+ end
52
+ CHILD
53
+
54
+ def terminate(pid)
55
+ Process.kill("TERM", pid)
56
+ rescue Errno::ESRCH
57
+ nil
58
+ ensure
59
+ deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 0.5
60
+ while Process.clock_gettime(Process::CLOCK_MONOTONIC) < deadline
61
+ waited = Process.waitpid2(pid, Process::WNOHANG)
62
+ return waited.last if waited
63
+
64
+ sleep 0.01
65
+ end
66
+ Process.kill("KILL", pid)
67
+ Process.waitpid2(pid).last
68
+ end
69
+
70
+ def run_command(*command)
71
+ Tempfile.create("migration-stdout") do |stdout|
72
+ Tempfile.create("migration-stderr") do |stderr|
73
+ pid = Process.spawn(*command, out: stdout, err: stderr)
74
+ deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + TIMEOUT_SECONDS
75
+ status = nil
76
+ until status
77
+ waited = Process.waitpid2(pid, Process::WNOHANG)
78
+ status = waited.last if waited
79
+ break if status
80
+ if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
81
+ status = terminate(pid)
82
+ stdout.rewind
83
+ stderr.rewind
84
+ return [
85
+ stdout.read(MAX_OUTPUT_BYTES) || "", stderr.read(MAX_OUTPUT_BYTES) || "", status, true, false
86
+ ]
87
+ end
88
+ if stdout.size > MAX_OUTPUT_BYTES || stderr.size > MAX_OUTPUT_BYTES
89
+ status = terminate(pid)
90
+ stdout.rewind
91
+ stderr.rewind
92
+ return [
93
+ stdout.read(MAX_OUTPUT_BYTES) || "", stderr.read(MAX_OUTPUT_BYTES) || "", status, false, true
94
+ ]
95
+ end
96
+ sleep 0.01
97
+ end
98
+ stdout.rewind
99
+ stderr.rewind
100
+ [
101
+ stdout.read(MAX_OUTPUT_BYTES) || "", stderr.read(MAX_OUTPUT_BYTES) || "", status, false, false
102
+ ]
103
+ end
104
+ end
105
+ end
106
+
107
+ def compile!(label, *command)
108
+ stdout, stderr, status, timed_out, output_limited = run_command(*command)
109
+ abort "#{label} generation timed out" if timed_out
110
+ abort "#{label} generation exceeded #{MAX_OUTPUT_BYTES} output bytes" if output_limited
111
+ return if status.success?
112
+
113
+ abort "#{label} generation failed\n#{stdout}#{stderr}"
114
+ end
115
+
116
+ def observe(parser_path, tokens)
117
+ payload = [Marshal.dump(tokens)].pack("m0")
118
+ stdout, stderr, status, timed_out, output_limited = run_command(
119
+ RbConfig.ruby, "-e", CHILD_RUNNER, parser_path, PARSER_CLASS, payload
120
+ )
121
+ return { timeout: true } if timed_out
122
+ return { output_limit: MAX_OUTPUT_BYTES } if output_limited
123
+
124
+ lines = stdout.lines
125
+ marker = lines.reverse.find { |line| line.start_with?("IBEX_HARNESS_RESULT=") }
126
+ unless status.success? && marker
127
+ return {
128
+ infrastructure_error: File.basename(parser_path),
129
+ status: status.exitstatus,
130
+ stdout: stdout,
131
+ stderr: stderr
132
+ }
133
+ end
134
+ outcome = marker && Marshal.load(marker.split("=", 2).last.strip.unpack1("m0"))
135
+ {
136
+ status: status.exitstatus,
137
+ outcome: outcome,
138
+ stdout: lines.reject { |line| line.equal?(marker) }.join,
139
+ stderr: stderr
140
+ }
141
+ end
142
+
143
+ abort "add at least one reviewed entry to CASES before running the harness" if CASES.empty?
144
+ grammar = File.expand_path(ARGV.fetch(0))
145
+ abort "grammar file does not exist: #{grammar}" unless File.file?(grammar)
146
+ racc = ENV.fetch("RACC", "racc")
147
+ ibex = ENV.fetch("IBEX", "ibex")
148
+
149
+ failures = 0
150
+ Dir.mktmpdir("ibex-migration-harness-") do |directory|
151
+ racc_parser = File.join(directory, "racc_parser.rb")
152
+ ibex_parser = File.join(directory, "ibex_parser.rb")
153
+ compile!("racc", racc, "-o", racc_parser, grammar)
154
+ compile!("ibex", ibex, "-E", "-o", ibex_parser, grammar)
155
+
156
+ CASES.each do |test_case|
157
+ name = test_case.fetch(:name)
158
+ tokens = test_case.fetch(:tokens)
159
+ expected = observe(racc_parser, tokens)
160
+ actual = observe(ibex_parser, tokens)
161
+ if expected == actual
162
+ puts "ok #{name}"
163
+ else
164
+ failures += 1
165
+ warn "difference #{name}"
166
+ warn " racc: #{expected.inspect}"
167
+ warn " ibex: #{actual.inspect}"
168
+ end
169
+ end
170
+ end
171
+ exit(failures.zero? ? 0 : 1)
172
+ RUBY
173
+ end
174
+ private_class_method :template
175
+ end
176
+ end
177
+ end
@@ -0,0 +1,94 @@
1
+ # frozen_string_literal: true
2
+ # rbs_inline: enabled
3
+
4
+ require "json"
5
+
6
+ module Ibex
7
+ module RaccMigration
8
+ # One immutable migration compatibility observation.
9
+ class Finding
10
+ SEVERITIES = %i[error warning info].freeze #: Array[Symbol]
11
+
12
+ attr_reader :code #: String
13
+ attr_reader :severity #: Symbol
14
+ attr_reader :message #: String
15
+ attr_reader :location #: Frontend::Location?
16
+ attr_reader :suggestion #: String
17
+
18
+ # @rbs (code: String, severity: Symbol, message: String, location: Frontend::Location?,
19
+ # suggestion: String) -> void
20
+ def initialize(code:, severity:, message:, location:, suggestion:)
21
+ unless SEVERITIES.include?(severity)
22
+ raise ArgumentError, "unknown migration finding severity #{severity.inspect}"
23
+ end
24
+
25
+ @code = code.dup.freeze
26
+ @severity = severity
27
+ @message = message.dup.freeze
28
+ @location = location
29
+ @suggestion = suggestion.dup.freeze
30
+ freeze
31
+ end
32
+
33
+ # @rbs () -> Hash[String, untyped]
34
+ def to_h
35
+ {
36
+ "code" => @code,
37
+ "severity" => @severity.to_s,
38
+ "message" => @message,
39
+ "location" => @location&.to_h&.transform_keys(&:to_s),
40
+ "suggestion" => @suggestion
41
+ }
42
+ end
43
+ end
44
+
45
+ # Versioned result of checking one grammar without executing its code.
46
+ class Report
47
+ SCHEMA_VERSION = 1 #: Integer
48
+
49
+ attr_reader :file #: String
50
+ attr_reader :class_name #: String?
51
+ attr_reader :findings #: Array[Finding]
52
+
53
+ # @rbs (file: String, class_name: String?, findings: Array[Finding]) -> void
54
+ def initialize(file:, class_name:, findings:)
55
+ @file = file.dup.freeze
56
+ @class_name = class_name&.dup&.freeze
57
+ @findings = findings.dup.freeze
58
+ freeze
59
+ end
60
+
61
+ # @rbs () -> bool
62
+ def compatible? = @findings.none? { |finding| finding.severity == :error }
63
+
64
+ # @rbs () -> Hash[String, untyped]
65
+ def to_h
66
+ {
67
+ "ibex_migration_check" => "racc",
68
+ "schema_version" => SCHEMA_VERSION,
69
+ "compatible" => compatible?,
70
+ "file" => @file,
71
+ "class_name" => @class_name,
72
+ "findings" => @findings.map(&:to_h)
73
+ }
74
+ end
75
+
76
+ # @rbs (*untyped) -> String
77
+ def to_json(*) = JSON.pretty_generate(to_h)
78
+
79
+ # @rbs () -> String
80
+ def to_text
81
+ return "#{@file}: compatible with the checked racc migration surface\n" if @findings.empty?
82
+
83
+ lines = @findings.flat_map do |finding|
84
+ prefix = finding.location ? finding.location.to_s : "#{@file}:1:1"
85
+ [
86
+ "#{prefix}: #{finding.severity}: #{finding.message} [#{finding.code}]",
87
+ " suggestion: #{finding.suggestion}"
88
+ ]
89
+ end
90
+ "#{lines.join("\n")}\n"
91
+ end
92
+ end
93
+ end
94
+ end
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "racc_migration/report"
4
+ require_relative "racc_migration/checker"
5
+ require_relative "racc_migration/harness"
6
+
7
+ module Ibex
8
+ # Static checks and explicitly generated differential tooling for migration
9
+ # from racc grammar files.
10
+ module RaccMigration
11
+ end
12
+ end