ibex 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (437) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +95 -272
  3. data/docs/architecture.md +187 -35
  4. data/docs/bison-import.md +103 -0
  5. data/docs/comparison-policy.md +127 -0
  6. data/docs/configuration-model.md +59 -0
  7. data/docs/configuration-report.md +28 -0
  8. data/docs/conflict-explanation-reviews/v1/records/README.md +28 -0
  9. data/docs/conflict-explanation-study.md +62 -0
  10. data/docs/construction-profiling.md +157 -0
  11. data/docs/cst-migration.md +3 -4
  12. data/docs/cst.md +24 -0
  13. data/docs/decisions/0000-template.md +24 -0
  14. data/docs/decisions/0001-separate-ir-pipeline.md +34 -0
  15. data/docs/decisions/0002-opaque-user-code-boundary.md +32 -0
  16. data/docs/decisions/0003-self-hosted-grammar-frontend.md +32 -0
  17. data/docs/decisions/0004-shared-semantic-and-lossless-source-model.md +33 -0
  18. data/docs/decisions/0005-contained-grammar-composition.md +34 -0
  19. data/docs/decisions/0006-bounded-structural-grammar-lowering.md +32 -0
  20. data/docs/decisions/0007-shared-parser-construction-pipeline.md +33 -0
  21. data/docs/decisions/0008-versioned-runtime-package-boundary.md +36 -0
  22. data/docs/decisions/0009-isolated-parser-sessions.md +34 -0
  23. data/docs/decisions/0010-committed-runtime-observation.md +34 -0
  24. data/docs/decisions/0011-versioned-semantic-action-boundary.md +37 -0
  25. data/docs/decisions/0012-bounded-nonexecuting-analysis.md +37 -0
  26. data/docs/decisions/0013-transactional-generation-publication.md +37 -0
  27. data/docs/decisions/0014-versioned-generated-lexer.md +35 -0
  28. data/docs/decisions/0015-worker-isolated-browser-analysis.md +32 -0
  29. data/docs/decisions/0016-red-green-concrete-syntax.md +34 -0
  30. data/docs/decisions/0017-persistent-syntax-artifacts.md +35 -0
  31. data/docs/decisions/0018-conservative-incremental-syntax-reuse.md +38 -0
  32. data/docs/decisions/0019-runtime-syntax-session-boundary.md +51 -0
  33. data/docs/decisions/0020-grammar-ir-parser-contract.md +65 -0
  34. data/docs/decisions/0021-data-only-parser-table-sidecar.md +50 -0
  35. data/docs/decisions/0022-manifest-bound-verification-report.md +77 -0
  36. data/docs/decisions/0023-syntax-only-repair-results.md +48 -0
  37. data/docs/decisions/0024-direct-ielr-construction.md +42 -0
  38. data/docs/decisions/README.md +74 -0
  39. data/docs/declarative-configuration.md +262 -0
  40. data/docs/development.md +179 -2
  41. data/docs/direct-ielr-decision.md +120 -0
  42. data/docs/direct-multi-entry-decision.md +82 -0
  43. data/docs/editor-setup.md +2 -1
  44. data/docs/error-ux-review-rubric-v1.md +117 -0
  45. data/docs/error-ux-reviews/v1/records/README.md +22 -0
  46. data/docs/error-ux-round2-review-status-v1.json +27 -0
  47. data/docs/error-ux-round2-reviews/v1/records/README.md +30 -0
  48. data/docs/error-ux-round2-v1.json +1581 -0
  49. data/docs/error-ux-round2.md +111 -0
  50. data/docs/error-ux.md +54 -0
  51. data/docs/getting-started.md +80 -0
  52. data/docs/grammar-reference.md +184 -13
  53. data/docs/ielr-design.md +43 -0
  54. data/docs/ielr.md +92 -0
  55. data/docs/ja/bison-import.md +51 -0
  56. data/docs/ja/stable-api.md +53 -0
  57. data/docs/lexer-construction-profile.md +109 -0
  58. data/docs/lexer-migration.md +5 -0
  59. data/docs/maturity.md +128 -0
  60. data/docs/project-site-strategy.md +54 -0
  61. data/docs/racc-migration-evidence.md +45 -0
  62. data/docs/racc-migration.md +11 -0
  63. data/docs/release-readiness.md +126 -68
  64. data/docs/repair-semantics.md +123 -0
  65. data/docs/runtime-abi-evolution.md +307 -0
  66. data/docs/stability.md +144 -34
  67. data/docs/status.md +43 -0
  68. data/docs/syntax-sessions.md +183 -0
  69. data/docs/table-artifact.md +125 -0
  70. data/docs/test-interactions.md +219 -0
  71. data/docs/verification-report.md +137 -0
  72. data/docs/verifier-trust-boundary.md +199 -0
  73. data/docs/workloads.md +132 -0
  74. data/lib/ibex/analysis/digraph.rb +128 -0
  75. data/lib/ibex/analysis.rb +1 -0
  76. data/lib/ibex/bison_import/importer.rb +516 -0
  77. data/lib/ibex/bison_import/tokenizer.rb +260 -0
  78. data/lib/ibex/bison_import.rb +200 -0
  79. data/lib/ibex/bounded_subprocess.rb +171 -0
  80. data/lib/ibex/cli/ambiguity.rb +14 -5
  81. data/lib/ibex/cli/analysis.rb +134 -0
  82. data/lib/ibex/cli/bison_import.rb +121 -0
  83. data/lib/ibex/cli/config.rb +128 -0
  84. data/lib/ibex/cli/diagnostics.rb +31 -11
  85. data/lib/ibex/cli/documentation.rb +11 -8
  86. data/lib/ibex/cli/equiv.rb +169 -0
  87. data/lib/ibex/cli/error_messages.rb +11 -4
  88. data/lib/ibex/cli/explain.rb +14 -5
  89. data/lib/ibex/cli/fix.rb +203 -0
  90. data/lib/ibex/cli/formatting.rb +11 -10
  91. data/lib/ibex/cli/fuzz.rb +200 -0
  92. data/lib/ibex/cli/fuzz_regressions.rb +145 -0
  93. data/lib/ibex/cli/generation_artifacts.rb +53 -2
  94. data/lib/ibex/cli/generation_error_messages.rb +1 -1
  95. data/lib/ibex/cli/grammar_tests.rb +32 -14
  96. data/lib/ibex/cli/ir_tools.rb +5 -61
  97. data/lib/ibex/cli/outputs.rb +60 -33
  98. data/lib/ibex/cli/reduce.rb +202 -0
  99. data/lib/ibex/cli/reduce_reporting.rb +71 -0
  100. data/lib/ibex/cli/samples.rb +49 -23
  101. data/lib/ibex/cli/verify.rb +116 -0
  102. data/lib/ibex/cli/watch.rb +6 -5
  103. data/lib/ibex/cli.rb +357 -56
  104. data/lib/ibex/codegen/action_locations.rb +1 -1
  105. data/lib/ibex/codegen/action_method_source.rb +5 -6
  106. data/lib/ibex/codegen/action_source.rb +3 -2
  107. data/lib/ibex/codegen/ambiguity.rb +5 -4
  108. data/lib/ibex/codegen/explain.rb +123 -55
  109. data/lib/ibex/codegen/generated_action_abi.rb +10 -5
  110. data/lib/ibex/codegen/rbs.rb +15 -7
  111. data/lib/ibex/codegen/report.rb +25 -13
  112. data/lib/ibex/codegen/ruby.rb +28 -64
  113. data/lib/ibex/codegen/ruby_actions.rb +9 -6
  114. data/lib/ibex/codegen/ruby_syntax.rb +15 -9
  115. data/lib/ibex/configuration/analysis_grammar.rb +35 -0
  116. data/lib/ibex/configuration/explanation.rb +421 -0
  117. data/lib/ibex/configuration/inspector.rb +232 -0
  118. data/lib/ibex/configuration.rb +580 -0
  119. data/lib/ibex/coverage/collector.rb +21 -11
  120. data/lib/ibex/coverage/event_stream.rb +13 -7
  121. data/lib/ibex/coverage/report.rb +26 -18
  122. data/lib/ibex/coverage/runtime_event_validator.rb +30 -21
  123. data/lib/ibex/delta_reducer.rb +99 -0
  124. data/lib/ibex/diff.rb +140 -0
  125. data/lib/ibex/equiv/machine.rb +135 -0
  126. data/lib/ibex/equiv.rb +373 -0
  127. data/lib/ibex/fix.rb +557 -0
  128. data/lib/ibex/frontend/ast.rb +22 -2
  129. data/lib/ibex/frontend/bootstrap_parser.rb +6 -13
  130. data/lib/ibex/frontend/diagnostic.rb +1 -1
  131. data/lib/ibex/frontend/formatter.rb +72 -26
  132. data/lib/ibex/frontend/generated_parser.rb +147 -117
  133. data/lib/ibex/frontend/generated_parser_base.rb +7 -3
  134. data/lib/ibex/frontend/generated_parser_includes.rb +1 -11
  135. data/lib/ibex/frontend/generation.rb +1 -1
  136. data/lib/ibex/frontend/parser/declarations.rb +17 -2
  137. data/lib/ibex/frontend/parser_configuration_support.rb +56 -0
  138. data/lib/ibex/frontend/regenerator.rb +1 -0
  139. data/lib/ibex/frontend/resolution.rb +2 -1
  140. data/lib/ibex/frontend/resolver.rb +1 -1
  141. data/lib/ibex/frontend/rule_documentation.rb +2 -1
  142. data/lib/ibex/frontend/source_cursor.rb +2 -2
  143. data/lib/ibex/frontend/source_span.rb +9 -1
  144. data/lib/ibex/frontend/token_adapter/declaration_state.rb +38 -6
  145. data/lib/ibex/frontend/token_adapter.rb +10 -0
  146. data/lib/ibex/frontend.rb +1 -0
  147. data/lib/ibex/fuzz.rb +219 -0
  148. data/lib/ibex/generation_input.rb +1 -1
  149. data/lib/ibex/generation_manifest.rb +36 -25
  150. data/lib/ibex/generation_transaction.rb +1 -1
  151. data/lib/ibex/generation_transaction_recovery.rb +4 -4
  152. data/lib/ibex/generation_transaction_validation.rb +2 -2
  153. data/lib/ibex/grammar_tests.rb +1 -1
  154. data/lib/ibex/ir/automaton_ir.rb +61 -19
  155. data/lib/ibex/ir/grammar_ir.rb +76 -45
  156. data/lib/ibex/ir/lexer_ir.rb +5 -5
  157. data/lib/ibex/ir/parser_contract.rb +135 -0
  158. data/lib/ibex/ir/serialize.rb +203 -76
  159. data/lib/ibex/ir/validator/automaton.rb +64 -34
  160. data/lib/ibex/ir/validator/base.rb +51 -33
  161. data/lib/ibex/ir/validator/grammar.rb +113 -83
  162. data/lib/ibex/ir/validator/lexer.rb +2 -2
  163. data/lib/ibex/ir/validator.rb +2 -1
  164. data/lib/ibex/ir.rb +16 -3
  165. data/lib/ibex/lalr/build_metrics.rb +50 -2
  166. data/lib/ibex/lalr/builder.rb +191 -56
  167. data/lib/ibex/lalr/conflict_search.rb +15 -4
  168. data/lib/ibex/lalr/counterexample.rb +19 -14
  169. data/lib/ibex/lalr/direct_lookaheads.rb +110 -57
  170. data/lib/ibex/lalr/goto_follows.rb +229 -0
  171. data/lib/ibex/lalr/ielr/annotator.rb +214 -0
  172. data/lib/ibex/lalr/ielr/bits.rb +28 -0
  173. data/lib/ibex/lalr/ielr/inadequacy.rb +47 -0
  174. data/lib/ibex/lalr/ielr/item_lookaheads.rb +78 -0
  175. data/lib/ibex/lalr/ielr/pipeline.rb +75 -0
  176. data/lib/ibex/lalr/ielr/split_stability.rb +78 -0
  177. data/lib/ibex/lalr/ielr/split_state.rb +20 -0
  178. data/lib/ibex/lalr/ielr/state_splitter.rb +258 -0
  179. data/lib/ibex/lalr/ielr_partition.rb +22 -8
  180. data/lib/ibex/lalr/inadequacy_report.rb +50 -0
  181. data/lib/ibex/lalr/lookahead_propagation.rb +111 -0
  182. data/lib/ibex/lalr/lr0_collection.rb +121 -0
  183. data/lib/ibex/lalr/unreachable_states.rb +80 -0
  184. data/lib/ibex/lalr.rb +52 -1
  185. data/lib/ibex/location.rb +3 -3
  186. data/lib/ibex/lsp/document_handlers.rb +9 -7
  187. data/lib/ibex/lsp/initialization_handlers.rb +9 -7
  188. data/lib/ibex/lsp/navigation_handlers.rb +25 -11
  189. data/lib/ibex/lsp/parser_configuration_assistance.rb +149 -0
  190. data/lib/ibex/lsp/position_codec.rb +6 -2
  191. data/lib/ibex/lsp/request_handlers.rb +3 -2
  192. data/lib/ibex/lsp/request_support.rb +14 -7
  193. data/lib/ibex/lsp/server.rb +13 -11
  194. data/lib/ibex/lsp/symbol_index.rb +13 -13
  195. data/lib/ibex/lsp/symbol_index_builder.rb +30 -20
  196. data/lib/ibex/lsp/symbol_occurrence.rb +8 -2
  197. data/lib/ibex/lsp/transport.rb +13 -3
  198. data/lib/ibex/lsp/workspace.rb +5 -4
  199. data/lib/ibex/lsp/workspace_analyzer.rb +1 -1
  200. data/lib/ibex/lsp.rb +1 -0
  201. data/lib/ibex/messages/en.yml +28 -0
  202. data/lib/ibex/messages/ja.yml +28 -0
  203. data/lib/ibex/messages.rb +68 -0
  204. data/lib/ibex/metrics.rb +175 -0
  205. data/lib/ibex/normalize/declarations.rb +2 -0
  206. data/lib/ibex/normalize/diagnostics.rb +63 -38
  207. data/lib/ibex/normalize/expander.rb +5 -4
  208. data/lib/ibex/normalize/expression.rb +8 -6
  209. data/lib/ibex/normalize/grammar_builder.rb +26 -0
  210. data/lib/ibex/normalize/inline_expansion.rb +100 -49
  211. data/lib/ibex/normalize/lexer.rb +3 -3
  212. data/lib/ibex/normalize/parameter_ebnf_lowering.rb +11 -10
  213. data/lib/ibex/normalize/parameter_lowering.rb +40 -23
  214. data/lib/ibex/normalize/parameters.rb +31 -4
  215. data/lib/ibex/normalize/parser_configuration.rb +70 -0
  216. data/lib/ibex/normalize.rb +10 -20
  217. data/lib/ibex/racc_migration/report.rb +6 -3
  218. data/lib/ibex/rake_task.rb +1 -1
  219. data/lib/ibex/samples.rb +43 -13
  220. data/lib/ibex/table_artifact/builder.rb +266 -0
  221. data/lib/ibex/table_artifact/cst_projection.rb +70 -0
  222. data/lib/ibex/table_artifact/document.rb +39 -0
  223. data/lib/ibex/table_artifact/executor.rb +187 -0
  224. data/lib/ibex/table_artifact/serializer.rb +57 -0
  225. data/lib/ibex/table_artifact/validator/metadata.rb +179 -0
  226. data/lib/ibex/table_artifact/validator/support.rb +86 -0
  227. data/lib/ibex/table_artifact/validator/tables.rb +238 -0
  228. data/lib/ibex/table_artifact/validator.rb +253 -0
  229. data/lib/ibex/table_artifact.rb +88 -0
  230. data/lib/ibex/table_simulation/result.rb +6 -2
  231. data/lib/ibex/table_simulation/step.rb +3 -1
  232. data/lib/ibex/tables.rb +14 -0
  233. data/lib/ibex/verifiable_generation_bundle.rb +86 -0
  234. data/lib/ibex/verification_report/builder.rb +159 -0
  235. data/lib/ibex/verification_report/canonical_ir.rb +123 -0
  236. data/lib/ibex/verification_report/logical_path.rb +68 -0
  237. data/lib/ibex/verification_report/validator.rb +329 -0
  238. data/lib/ibex/verification_report.rb +69 -0
  239. data/lib/ibex/verify/action_correspondence.rb +138 -0
  240. data/lib/ibex/verify/language_witness.rb +295 -0
  241. data/lib/ibex/verify/reference_collection.rb +190 -0
  242. data/lib/ibex/verify/result.rb +73 -0
  243. data/lib/ibex/verify/verifier.rb +598 -0
  244. data/lib/ibex/verify.rb +15 -0
  245. data/lib/ibex/version.rb +1 -1
  246. data/lib/ibex/watch/runner.rb +10 -1
  247. data/lib/ibex/watch/source_snapshot.rb +13 -8
  248. data/lib/ibex.rb +10 -0
  249. data/schema/{automaton-ir-v1.schema.json → automaton-ir-definitions.schema.json} +6 -6
  250. data/schema/{automaton-ir-v2.schema.json → automaton-ir.schema.json} +14 -7
  251. data/schema/bison-import-v1.schema.json +106 -0
  252. data/schema/conflict-explanation-review-v1.schema.json +196 -0
  253. data/schema/conflict-explanation-study-v1.schema.json +528 -0
  254. data/schema/construction-profile-v1.schema.json +779 -0
  255. data/schema/diff-v1.schema.json +41 -0
  256. data/schema/direct-ielr-decision-v1.schema.json +200 -0
  257. data/schema/equiv-v1.schema.json +64 -0
  258. data/schema/error-ux-review-v1.schema.json +609 -0
  259. data/schema/error-ux-round2-review-v1.schema.json +191 -0
  260. data/schema/error-ux-round2-v1.schema.json +519 -0
  261. data/schema/explain-v1.schema.json +63 -1
  262. data/schema/fix-v1.schema.json +64 -0
  263. data/schema/fix-v2.schema.json +66 -0
  264. data/schema/fix-v3.schema.json +101 -0
  265. data/schema/fuzz-regression-v1.schema.json +79 -0
  266. data/schema/fuzz-v1.schema.json +114 -0
  267. data/schema/{grammar-ir-v2.schema.json → grammar-ir-extensions.schema.json} +20 -65
  268. data/schema/{grammar-ir-v1.schema.json → grammar-ir-foundation.schema.json} +3 -3
  269. data/schema/grammar-ir.schema.json +271 -0
  270. data/schema/ielr-benchmark-v1.schema.json +85 -0
  271. data/schema/lexer-profile-v1.schema.json +491 -0
  272. data/schema/metrics-v1.schema.json +60 -0
  273. data/schema/reduce-v1.schema.json +22 -0
  274. data/schema/reduce-v2.schema.json +63 -0
  275. data/schema/table-artifact-v1.schema.json +377 -0
  276. data/schema/verification-report-v1.schema.json +222 -0
  277. data/schema/verify-v1.schema.json +42 -0
  278. data/sig/ibex/analysis/digraph.rbs +19 -0
  279. data/sig/ibex/bison_import/importer.rbs +140 -0
  280. data/sig/ibex/bison_import/tokenizer.rbs +91 -0
  281. data/sig/ibex/bison_import.rbs +95 -0
  282. data/sig/ibex/bounded_subprocess.rbs +76 -0
  283. data/sig/ibex/cli/ambiguity.rbs +6 -0
  284. data/sig/ibex/cli/analysis.rbs +43 -0
  285. data/sig/ibex/cli/bison_import.rbs +32 -0
  286. data/sig/ibex/cli/config.rbs +35 -0
  287. data/sig/ibex/cli/diagnostics.rbs +16 -3
  288. data/sig/ibex/cli/documentation.rbs +5 -1
  289. data/sig/ibex/cli/equiv.rbs +49 -0
  290. data/sig/ibex/cli/error_messages.rbs +6 -0
  291. data/sig/ibex/cli/explain.rbs +6 -0
  292. data/sig/ibex/cli/fix.rbs +53 -0
  293. data/sig/ibex/cli/formatting.rbs +5 -1
  294. data/sig/ibex/cli/fuzz.rbs +47 -0
  295. data/sig/ibex/cli/fuzz_regressions.rbs +40 -0
  296. data/sig/ibex/cli/generation_artifacts.rbs +21 -2
  297. data/sig/ibex/cli/generation_error_messages.rbs +2 -2
  298. data/sig/ibex/cli/grammar_tests.rbs +15 -3
  299. data/sig/ibex/cli/ir_tools.rbs +7 -19
  300. data/sig/ibex/cli/outputs.rbs +9 -3
  301. data/sig/ibex/cli/reduce.rbs +60 -0
  302. data/sig/ibex/cli/reduce_reporting.rbs +27 -0
  303. data/sig/ibex/cli/samples.rbs +8 -0
  304. data/sig/ibex/cli/verify.rbs +28 -0
  305. data/sig/ibex/cli/watch.rbs +2 -2
  306. data/sig/ibex/cli.rbs +79 -5
  307. data/sig/ibex/codegen/action_method_source.rbs +4 -4
  308. data/sig/ibex/codegen/explain.rbs +45 -25
  309. data/sig/ibex/codegen/generated_action_abi.rbs +14 -10
  310. data/sig/ibex/codegen/rbs.rbs +12 -10
  311. data/sig/ibex/codegen/report.rbs +12 -12
  312. data/sig/ibex/codegen/ruby.rbs +18 -15
  313. data/sig/ibex/codegen/ruby_actions.rbs +6 -6
  314. data/sig/ibex/codegen/ruby_syntax.rbs +10 -8
  315. data/sig/ibex/configuration/analysis_grammar.rbs +13 -0
  316. data/sig/ibex/configuration/explanation.rbs +195 -0
  317. data/sig/ibex/configuration/inspector.rbs +53 -0
  318. data/sig/ibex/configuration.rbs +242 -0
  319. data/sig/ibex/coverage/collector.rbs +16 -14
  320. data/sig/ibex/coverage/event_stream.rbs +12 -10
  321. data/sig/ibex/coverage/report.rbs +20 -18
  322. data/sig/ibex/coverage/runtime_event_validator.rbs +38 -36
  323. data/sig/ibex/delta_reducer.rbs +38 -0
  324. data/sig/ibex/diff.rbs +47 -0
  325. data/sig/ibex/equiv/machine.rbs +54 -0
  326. data/sig/ibex/equiv.rbs +129 -0
  327. data/sig/ibex/fix.rbs +138 -0
  328. data/sig/ibex/frontend/ast.rbs +33 -7
  329. data/sig/ibex/frontend/bootstrap_parser.rbs +2 -0
  330. data/sig/ibex/frontend/diagnostic.rbs +2 -2
  331. data/sig/ibex/frontend/formatter.rbs +38 -30
  332. data/sig/ibex/frontend/generated_parser.rbs +87 -77
  333. data/sig/ibex/frontend/generated_parser_base.rbs +4 -2
  334. data/sig/ibex/frontend/parser/declarations.rbs +3 -0
  335. data/sig/ibex/frontend/parser_configuration_support.rbs +27 -0
  336. data/sig/ibex/frontend/resolution.rbs +3 -2
  337. data/sig/ibex/frontend/source_cursor.rbs +2 -2
  338. data/sig/ibex/frontend/source_span.rbs +5 -2
  339. data/sig/ibex/frontend/token_adapter/declaration_state.rbs +13 -1
  340. data/sig/ibex/frontend/token_adapter.rbs +6 -0
  341. data/sig/ibex/fuzz.rbs +71 -0
  342. data/sig/ibex/generation_input.rbs +2 -2
  343. data/sig/ibex/generation_manifest.rbs +35 -32
  344. data/sig/ibex/generation_transaction_recovery.rbs +2 -2
  345. data/sig/ibex/generation_transaction_validation.rbs +2 -2
  346. data/sig/ibex/grammar_tests.rbs +2 -2
  347. data/sig/ibex/ir/automaton_ir.rbs +30 -11
  348. data/sig/ibex/ir/grammar_ir.rbs +41 -27
  349. data/sig/ibex/ir/lexer_ir.rbs +8 -8
  350. data/sig/ibex/ir/parser_contract.rbs +69 -0
  351. data/sig/ibex/ir/serialize.rbs +63 -28
  352. data/sig/ibex/ir/validator/automaton.rbs +47 -44
  353. data/sig/ibex/ir/validator/base.rbs +35 -31
  354. data/sig/ibex/ir/validator/grammar.rbs +78 -67
  355. data/sig/ibex/ir/validator/lexer.rbs +4 -4
  356. data/sig/ibex/ir.rbs +9 -3
  357. data/sig/ibex/lalr/build_metrics.rbs +42 -2
  358. data/sig/ibex/lalr/builder.rbs +75 -24
  359. data/sig/ibex/lalr/conflict_search.rbs +5 -2
  360. data/sig/ibex/lalr/counterexample.rbs +17 -14
  361. data/sig/ibex/lalr/direct_lookaheads.rbs +35 -28
  362. data/sig/ibex/lalr/goto_follows.rbs +89 -0
  363. data/sig/ibex/lalr/ielr/annotator.rbs +67 -0
  364. data/sig/ibex/lalr/ielr/bits.rbs +14 -0
  365. data/sig/ibex/lalr/ielr/inadequacy.rbs +24 -0
  366. data/sig/ibex/lalr/ielr/item_lookaheads.rbs +35 -0
  367. data/sig/ibex/lalr/ielr/pipeline.rbs +17 -0
  368. data/sig/ibex/lalr/ielr/split_stability.rbs +27 -0
  369. data/sig/ibex/lalr/ielr/split_state.rbs +14 -0
  370. data/sig/ibex/lalr/ielr/state_splitter.rbs +76 -0
  371. data/sig/ibex/lalr/ielr_partition.rbs +17 -5
  372. data/sig/ibex/lalr/inadequacy_report.rbs +21 -0
  373. data/sig/ibex/lalr/lookahead_propagation.rbs +38 -0
  374. data/sig/ibex/lalr/lr0_collection.rbs +50 -0
  375. data/sig/ibex/lalr/unreachable_states.rbs +25 -0
  376. data/sig/ibex/lalr.rbs +11 -1
  377. data/sig/ibex/location.rbs +6 -6
  378. data/sig/ibex/lsp/document_handlers.rbs +8 -8
  379. data/sig/ibex/lsp/initialization_handlers.rbs +12 -12
  380. data/sig/ibex/lsp/navigation_handlers.rbs +15 -12
  381. data/sig/ibex/lsp/parser_configuration_assistance.rbs +49 -0
  382. data/sig/ibex/lsp/position_codec.rbs +8 -4
  383. data/sig/ibex/lsp/request_handlers.rbs +4 -4
  384. data/sig/ibex/lsp/request_support.rbs +8 -8
  385. data/sig/ibex/lsp/server.rbs +16 -16
  386. data/sig/ibex/lsp/symbol_index.rbs +20 -21
  387. data/sig/ibex/lsp/symbol_index_builder.rbs +9 -8
  388. data/sig/ibex/lsp/symbol_occurrence.rbs +11 -5
  389. data/sig/ibex/lsp/transport.rbs +18 -6
  390. data/sig/ibex/lsp/workspace.rbs +3 -2
  391. data/sig/ibex/lsp/workspace_analyzer.rbs +1 -1
  392. data/sig/ibex/messages.rbs +26 -0
  393. data/sig/ibex/metrics.rbs +50 -0
  394. data/sig/ibex/normalize/diagnostics.rbs +4 -4
  395. data/sig/ibex/normalize/expander.rbs +2 -2
  396. data/sig/ibex/normalize/expression.rbs +10 -8
  397. data/sig/ibex/normalize/grammar_builder.rbs +11 -0
  398. data/sig/ibex/normalize/inline_expansion.rbs +59 -50
  399. data/sig/ibex/normalize/lexer.rbs +4 -4
  400. data/sig/ibex/normalize/parameter_ebnf_lowering.rbs +15 -14
  401. data/sig/ibex/normalize/parameter_lowering.rbs +18 -18
  402. data/sig/ibex/normalize/parameters.rbs +13 -8
  403. data/sig/ibex/normalize/parser_configuration.rbs +23 -0
  404. data/sig/ibex/normalize.rbs +11 -5
  405. data/sig/ibex/racc_migration/report.rbs +8 -6
  406. data/sig/ibex/rake_task.rbs +1 -1
  407. data/sig/ibex/samples.rbs +13 -4
  408. data/sig/ibex/table_artifact/builder.rbs +93 -0
  409. data/sig/ibex/table_artifact/cst_projection.rbs +20 -0
  410. data/sig/ibex/table_artifact/document.rbs +27 -0
  411. data/sig/ibex/table_artifact/executor.rbs +66 -0
  412. data/sig/ibex/table_artifact/serializer.rbs +25 -0
  413. data/sig/ibex/table_artifact/validator/metadata.rbs +52 -0
  414. data/sig/ibex/table_artifact/validator/support.rbs +46 -0
  415. data/sig/ibex/table_artifact/validator/tables.rbs +63 -0
  416. data/sig/ibex/table_artifact/validator.rbs +66 -0
  417. data/sig/ibex/table_artifact.rbs +32 -0
  418. data/sig/ibex/table_simulation/result.rbs +8 -4
  419. data/sig/ibex/table_simulation/step.rbs +4 -2
  420. data/sig/ibex/tables.rbs +10 -9
  421. data/sig/ibex/verifiable_generation_bundle.rbs +24 -0
  422. data/sig/ibex/verification_report/builder.rbs +53 -0
  423. data/sig/ibex/verification_report/canonical_ir.rbs +41 -0
  424. data/sig/ibex/verification_report/logical_path.rbs +34 -0
  425. data/sig/ibex/verification_report/validator.rbs +114 -0
  426. data/sig/ibex/verification_report.rbs +44 -0
  427. data/sig/ibex/verify/action_correspondence.rbs +50 -0
  428. data/sig/ibex/verify/language_witness.rbs +112 -0
  429. data/sig/ibex/verify/reference_collection.rbs +63 -0
  430. data/sig/ibex/verify/result.rbs +52 -0
  431. data/sig/ibex/verify/verifier.rbs +170 -0
  432. data/sig/ibex/verify.rbs +6 -0
  433. data/sig/ibex/watch/runner.rbs +12 -2
  434. data/sig/ibex/watch/source_snapshot.rbs +13 -9
  435. metadata +232 -12
  436. data/lib/ibex/ir/migration.rb +0 -120
  437. data/sig/ibex/ir/migration.rbs +0 -34
data/lib/ibex/cli.rb CHANGED
@@ -1,9 +1,12 @@
1
1
  # frozen_string_literal: true
2
+ # rbs_inline: enabled
2
3
 
3
4
  require "json"
4
5
  require "optparse"
5
6
  require_relative "version"
6
7
  require_relative "error"
8
+ require_relative "configuration"
9
+ require_relative "messages"
7
10
  require_relative "artifact_set"
8
11
  require_relative "generation_input"
9
12
  require_relative "generation_transaction"
@@ -18,6 +21,7 @@ require_relative "lalr/conflict"
18
21
  require_relative "lalr/on_error_reductions"
19
22
  require_relative "lalr/default_reductions"
20
23
  require_relative "lalr/build_metrics"
24
+ require_relative "lalr/inadequacy_report"
21
25
  require_relative "lalr/direct_lookaheads"
22
26
  require_relative "lalr/ielr_partition"
23
27
  require_relative "lalr/builder"
@@ -30,24 +34,31 @@ require_relative "cli/outputs"
30
34
  module Ibex
31
35
  CLI_FEATURE_ROOT = File.expand_path("cli", __dir__ || raise("CLI source directory is unavailable")) #: String
32
36
  autoload :CLIAmbiguity, File.join(CLI_FEATURE_ROOT, "ambiguity")
37
+ autoload :CLIAnalysis, File.join(CLI_FEATURE_ROOT, "analysis")
38
+ autoload :CLIBisonImport, File.join(CLI_FEATURE_ROOT, "bison_import")
33
39
  autoload :CLICoverage, File.join(CLI_FEATURE_ROOT, "coverage")
34
40
  autoload :CLIDebug, File.join(CLI_FEATURE_ROOT, "debug")
35
41
  autoload :CLIDiagnostics, File.join(CLI_FEATURE_ROOT, "diagnostics")
36
42
  autoload :CLIDocumentation, File.join(CLI_FEATURE_ROOT, "documentation")
37
43
  autoload :CLIErrorMessages, File.join(CLI_FEATURE_ROOT, "error_messages")
44
+ autoload :CLIEquiv, File.join(CLI_FEATURE_ROOT, "equiv")
38
45
  autoload :CLIExplain, File.join(CLI_FEATURE_ROOT, "explain")
39
46
  autoload :CLIFormatting, File.join(CLI_FEATURE_ROOT, "formatting")
47
+ autoload :CLIFix, File.join(CLI_FEATURE_ROOT, "fix")
48
+ autoload :CLIFuzz, File.join(CLI_FEATURE_ROOT, "fuzz")
40
49
  autoload :CLIGrammarTests, File.join(CLI_FEATURE_ROOT, "grammar_tests")
41
50
  autoload :CLIIRTools, File.join(CLI_FEATURE_ROOT, "ir_tools")
42
51
  autoload :CLILSP, File.join(CLI_FEATURE_ROOT, "lsp")
43
52
  autoload :CLIRaccMigration, File.join(CLI_FEATURE_ROOT, "racc_migration")
53
+ autoload :CLIReduce, File.join(CLI_FEATURE_ROOT, "reduce")
44
54
  autoload :CLISamples, File.join(CLI_FEATURE_ROOT, "samples")
55
+ autoload :CLIVerify, File.join(CLI_FEATURE_ROOT, "verify")
45
56
  autoload :CLIWatch, File.join(CLI_FEATURE_ROOT, "watch")
46
57
 
47
58
  # @rbs!
48
59
  # interface _CLIOutput
49
- # def puts: (*untyped) -> untyped
50
- # def write: (String) -> untyped
60
+ # def puts: (*Object?) -> Object?
61
+ # def write: (String) -> Object?
51
62
  # end
52
63
  # interface _CLIInput
53
64
  # def read: () -> String
@@ -66,6 +77,9 @@ module Ibex
66
77
  # ?algorithm: Symbol,
67
78
  # ?suggest_ielr: bool,
68
79
  # ?entry_isolation: bool,
80
+ # ?ielr_strategy: Symbol,
81
+ # ?ielr_report: bool,
82
+ # ?remove_unreachable: bool,
69
83
  # ?warnings: Array[Symbol],
70
84
  # ?output: String,
71
85
  # ?embedded: bool,
@@ -105,7 +119,31 @@ module Ibex
105
119
  # ?explain_token: String,
106
120
  # ?explain_format: String,
107
121
  # ?check_ambiguity: bool,
108
- # ?check_format: String
122
+ # ?check_format: String,
123
+ # ?fuzz_seed: Integer,
124
+ # ?fuzz_count: Integer,
125
+ # ?fuzz_max_tokens: Integer,
126
+ # ?fuzz_max_depth: Integer,
127
+ # ?fuzz_max_expansions: Integer,
128
+ # ?fuzz_max_actions: Integer,
129
+ # ?fuzz_max_stack: Integer,
130
+ # ?fuzz_coverage_guided: bool,
131
+ # ?fuzz_path_length: Integer,
132
+ # ?fuzz_against: String,
133
+ # ?fuzz_against_runtime: String,
134
+ # ?fuzz_against_timeout: Integer,
135
+ # ?fuzz_against_max_output: Integer,
136
+ # ?fuzz_max_reduction_trials: Integer,
137
+ # ?fuzz_regression_dir: String,
138
+ # ?fuzz_save_regression: bool,
139
+ # ?fuzz_format: String,
140
+ # ?reduce_command: String,
141
+ # ?reduce_mode: Symbol,
142
+ # ?reduce_max_trials: Integer,
143
+ # ?reduce_timeout: Integer,
144
+ # ?reduce_max_output_bytes: Integer,
145
+ # ?reduce_max_input_bytes: Integer,
146
+ # ?reduce_format: String
109
147
  # }
110
148
 
111
149
  # Command-line pipeline coordinator.
@@ -113,21 +151,28 @@ module Ibex
113
151
  class CLI
114
152
  SUBCOMMAND_HANDLERS = {
115
153
  "check" => %i[CLIAmbiguity run_check_command],
154
+ "diff" => %i[CLIAnalysis run_diff_command],
116
155
  "diagnose" => %i[CLIDiagnostics run_diagnose_command],
117
- "coverage" => %i[CLICoverage run_coverage_command],
156
+ "coverage" => %i[CLICoverage run_coverage_command], "config" => %i[CLIConfig run_config_command],
118
157
  "debug" => %i[CLIDebug run_debug_command],
119
158
  "doc" => %i[CLIDocumentation run_documentation_command],
120
159
  "errors" => %i[CLIErrorMessages run_error_messages_command],
160
+ "equiv" => %i[CLIEquiv run_equiv_command],
121
161
  "explain" => %i[CLIExplain run_explain_command],
122
162
  "fmt" => %i[CLIFormatting run_format_command],
163
+ "fix" => %i[CLIFix run_fix_command],
164
+ "fuzz" => %i[CLIFuzz run_fuzz_command],
123
165
  "test" => %i[CLIGrammarTests run_grammar_tests_command],
124
166
  "lsp" => %i[CLILSP run_lsp_command],
167
+ "import" => %i[CLIBisonImport run_bison_import_command],
168
+ "metrics" => %i[CLIAnalysis run_metrics_command],
125
169
  "migrate-check" => %i[CLIRaccMigration run_migrate_check_command],
126
170
  "migrate-harness" => %i[CLIRaccMigration run_migrate_harness_command],
171
+ "reduce" => %i[CLIReduce run_reduce_command],
127
172
  "samples" => %i[CLISamples run_samples_command],
173
+ "verify" => %i[CLIVerify run_verify_command],
128
174
  "validate-ir" => %i[CLIIRTools run_validate_ir_command],
129
- "compare" => %i[CLIIRTools run_compare_command],
130
- "migrate-ir" => %i[CLIIRTools run_migrate_ir_command]
175
+ "compare" => %i[CLIIRTools run_compare_command]
131
176
  }.freeze #: Hash[String, [Symbol, Symbol]]
132
177
 
133
178
  include CLICounterexampleOptions
@@ -138,6 +183,10 @@ module Ibex
138
183
  # @rbs @stdout: _CLIOutput
139
184
  # @rbs @stderr: _CLIOutput
140
185
  # @rbs @options: cli_options
186
+ # @rbs @language: String
187
+ # @rbs @configuration_explicit_options: Hash[Symbol, bool]
188
+ # @rbs @generation_grammar: IR::Grammar?
189
+ # @rbs @generation_automaton: IR::Automaton?
141
190
 
142
191
  # rubocop:disable Layout/LineLength
143
192
  # @rbs (Array[String] arguments, ?stdin: _CLIInput, ?stdout: _CLIOutput, ?stderr: _CLIOutput, ?watch_clock: (^() -> Float)?, ?watch_sleeper: (^(Float) -> void)?, ?watch_iteration_hook: (^(Symbol, Integer, Array[String]) -> (Integer | Symbol | nil))?) -> Integer
@@ -157,13 +206,23 @@ module Ibex
157
206
  @watch_clock = watch_clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
158
207
  @watch_sleeper = watch_sleeper || ->(seconds) { sleep(seconds) }
159
208
  @watch_iteration_hook = watch_iteration_hook || ->(_event, _iteration, _paths) {}
160
- @options = { emit: "ruby", mode: :default, table: :compact, line_convert: true }
209
+ @language = Messages.language(ENV.fetch("IBEX_LANG", nil))
210
+ @options = {
211
+ emit: "ruby", mode: Configuration::Registry.fetch("grammar.mode").default,
212
+ table: Configuration::Registry.fetch("table.representation").default, line_convert: true
213
+ }
161
214
  .merge(CLICounterexampleOptions::DEFAULTS)
215
+ @configuration_explicit_options = {}
216
+ @generation_grammar = nil
217
+ @generation_automaton = @analysis_configuration = nil
162
218
  end
163
219
  # rubocop:enable Layout/LineLength
164
220
 
165
221
  # @rbs (Array[String] arguments) -> Integer
166
222
  def run(arguments)
223
+ arguments = extract_language(arguments)
224
+ @generation_grammar = nil
225
+ @generation_automaton = @analysis_configuration = nil
167
226
  subcommand = dispatch_subcommand(arguments)
168
227
  return subcommand unless subcommand.nil?
169
228
 
@@ -183,12 +242,94 @@ module Ibex
183
242
  process_grammar(path)
184
243
  end
185
244
  rescue OptionParser::ParseError, Ibex::Error, SystemCallError, SystemStackError => e
186
- @stderr.puts(e.message)
245
+ @stderr.puts(Messages.translate("cli.error", language: @language, detail: e.message))
187
246
  1
188
247
  end
189
248
 
190
249
  private
191
250
 
251
+ # @rbs (Symbol name) -> void
252
+ def mark_configuration_option(name)
253
+ @configuration_explicit_options[name] = true
254
+ end
255
+
256
+ # OptionParser accepts a heterogeneous set of operation and configuration
257
+ # values. The CLIAdapter is the validation boundary for this dynamic map.
258
+ # @rbs (Symbol name, untyped value) -> void
259
+ def set_configuration_option(name, value)
260
+ @options[name] = value
261
+ mark_configuration_option(name)
262
+ end
263
+
264
+ # @rbs (String value) -> void
265
+ def select_configuration_mode(value)
266
+ set_configuration_option(:mode, value.to_sym)
267
+ end
268
+
269
+ # @rbs () -> Configuration::Resolver
270
+ def effective_configuration
271
+ grammar = @generation_grammar
272
+ contract = grammar&.parser_contract
273
+ @analysis_configuration || Configuration::CLIAdapter.new(
274
+ @options, explicit_keys: @configuration_explicit_options.keys
275
+ ).resolve(
276
+ grammar: contract&.configuration_values || {},
277
+ locations: contract&.configuration_locations || {}
278
+ )
279
+ end
280
+
281
+ # Resolve a command-local legacy option hash without promoting it into the
282
+ # reusable CLI instance's generation settings.
283
+ # @rbs (Hash[Symbol, untyped] options, explicit_keys: Array[Symbol]) -> Configuration::Resolver
284
+ def resolve_configuration_options(options, explicit_keys:)
285
+ Configuration::CLIAdapter.new(options, explicit_keys: explicit_keys).resolve
286
+ end
287
+
288
+ # @rbs (Hash[Symbol, untyped] options, String name) -> untyped
289
+ def local_configuration_value(options, name)
290
+ explicit_keys = options.fetch(:configuration_explicit)
291
+ resolve_configuration_options(options, explicit_keys: explicit_keys).value(name)
292
+ end
293
+
294
+ # @rbs (Hash[Symbol, untyped] options, Symbol name, untyped value) -> void
295
+ def set_local_configuration_option(options, name, value)
296
+ options[name] = value
297
+ options.fetch(:configuration_explicit) << name
298
+ end
299
+
300
+ # Configuration values have already been validated by Configuration::Resolver;
301
+ # callers narrow the value according to the option they are consuming.
302
+ # @rbs (String name) -> untyped
303
+ def configuration_value(name)
304
+ effective_configuration.value(name)
305
+ end
306
+
307
+ # @rbs (Array[String] arguments) -> Array[String]
308
+ def extract_language(arguments)
309
+ remaining = [] #: Array[String]
310
+ index = 0
311
+ while index < arguments.length
312
+ argument = arguments.fetch(index)
313
+ if argument == "--lang"
314
+ value = arguments[index + 1]
315
+ raise OptionParser::MissingArgument, "--lang" unless value
316
+
317
+ @language = Messages.language(value)
318
+ index += 2
319
+ elsif argument.start_with?("--lang=")
320
+ value = argument.delete_prefix("--lang=")
321
+ raise OptionParser::MissingArgument, "--lang" if value.empty?
322
+
323
+ @language = Messages.language(value)
324
+ index += 1
325
+ else
326
+ remaining << argument
327
+ index += 1
328
+ end
329
+ end
330
+ remaining
331
+ end
332
+
192
333
  # @rbs (Array[String] arguments) -> Integer?
193
334
  def dispatch_subcommand(arguments)
194
335
  definition = SUBCOMMAND_HANDLERS[arguments.first]
@@ -220,28 +361,41 @@ module Ibex
220
361
  add_error_messages_generation_option(options)
221
362
  add_compatibility_options(options)
222
363
  add_information_options(options)
223
- options.separator("")
224
- options.separator("Subcommands:")
225
- options.separator(" check --ambiguity search for ambiguity within explicit budgets")
226
- options.separator(" coverage collect, merge, or check runtime coverage")
227
- options.separator(" debug AUTOMATON [TOKEN] simulate validated Automaton IR tables")
228
- options.separator(" diagnose collect frontend diagnostics")
229
- options.separator(" doc render grammar documentation")
230
- options.separator(" errors --list|--update list or update example-keyed syntax error messages")
231
- options.separator(" explain explain selected parser conflicts")
232
- options.separator(" fmt format grammar source")
233
- options.separator(" test run grammar-declared source examples")
234
- options.separator(" lsp run the language server over stdio")
235
- options.separator(" migrate-check statically check a racc grammar migration")
236
- options.separator(" migrate-harness generate a differential subprocess harness")
237
- options.separator(" samples generate bounded terminal sentences")
238
- options.separator(" validate-ir FILE validate a versioned IR document")
239
- options.separator(" compare BEFORE AFTER compare two versioned IR documents")
240
- options.separator(" migrate-ir INPUT --to=2 migrate a versioned IR document")
364
+ add_subcommand_help(options)
241
365
  end
242
366
  end
243
367
 
244
368
  # @rbs (OptionParser options) -> void
369
+ def add_subcommand_help(options)
370
+ options.separator("")
371
+ options.separator("Subcommands:")
372
+ options.separator(" check --ambiguity search for ambiguity within explicit budgets")
373
+ options.separator(CONFIG_SUBCOMMAND_HELP)
374
+ options.separator(" debug AUTOMATON [TOKEN] simulate validated Automaton IR tables")
375
+ options.separator(" diagnose collect frontend diagnostics")
376
+ options.separator(" diff OLD NEW classify grammar and automaton changes")
377
+ options.separator(" doc render grammar documentation")
378
+ options.separator(" errors --list|--update list or update example-keyed syntax error messages")
379
+ options.separator(" equiv LEFT RIGHT search for bounded language differences")
380
+ options.separator(" explain explain selected parser conflicts")
381
+ options.separator(" fmt format grammar source")
382
+ options.separator(" fix propose bounded-equivalent conflict repairs")
383
+ options.separator(" fuzz run bounded grammar-derived differential fuzzing")
384
+ options.separator(" test run grammar-declared source examples")
385
+ options.separator(" lsp run the language server over stdio")
386
+ options.separator(" import bison FILE import Bison grammar structure for analysis")
387
+ options.separator(" migrate-check statically check a racc grammar migration")
388
+ options.separator(" migrate-harness generate a differential subprocess harness")
389
+ options.separator(" metrics GRAMMAR report deterministic grammar/table metrics")
390
+ options.separator(" reduce delta-debug a failing token, line, or byte sequence")
391
+ options.separator(" samples generate bounded terminal sentences")
392
+ options.separator(" verify AUTOMATON independently verify Automaton IR semantics")
393
+ options.separator(" validate-ir FILE validate the current IR document")
394
+ options.separator(" compare BEFORE AFTER compare current IR documents")
395
+ end
396
+
397
+ # @rbs (OptionParser options) -> void
398
+ # rubocop:disable Metrics/MethodLength -- pipeline flags share one parser boundary.
245
399
  def add_pipeline_options(options)
246
400
  options.on("--emit=FORMAT", "ast, sets, lexer-ir, grammar-ir, automaton-ir, or ruby") do |value|
247
401
  @options[:emit] = value
@@ -249,38 +403,54 @@ module Ibex
249
403
  options.on("--from=FORMAT", %w[grammar-ir automaton-ir], "resume from IR JSON") do |value|
250
404
  @options[:from] = value
251
405
  end
252
- options.on("--mode=MODE", %w[default extended], "grammar mode") { |value| @options[:mode] = value.to_sym }
406
+ options.on("--mode=MODE", %w[default extended], "grammar mode") { |value| select_configuration_mode(value) }
253
407
  options.on("--table=FORMAT", %w[plain compact], "parser table format") do |value|
254
- @options[:table] = value.to_sym
408
+ set_configuration_option(:table, value.to_sym)
255
409
  end
256
410
  options.on(
257
411
  "--cst-trivia=POLICY",
258
412
  %w[leading balanced drop attach],
259
413
  "leading, balanced, drop, or attach (alias for leading)"
260
414
  ) do |value|
261
- @options[:cst_trivia] = value.to_sym
415
+ set_configuration_option(:cst_trivia, value.to_sym)
262
416
  end
263
417
  options.on("--algorithm=NAME", %w[slr lalr ielr lr1], "parser construction algorithm") do |value|
264
- @options[:algorithm] = value.to_sym
418
+ set_configuration_option(:algorithm, value.to_sym)
419
+ end
420
+ options.on("--ielr-strategy=NAME", %w[direct partition], "IELR construction strategy") do |value|
421
+ @options[:ielr_strategy] = value.to_sym
422
+ end
423
+ options.on("--ielr-report", "report canonical-vs-IELR action differences") do
424
+ @options[:ielr_report] = true
425
+ end
426
+ options.on("--remove-unreachable", "compact unreachable IELR states after resolution") do
427
+ @options[:remove_unreachable] = true
265
428
  end
266
429
  options.on("--suggest-ielr", "check whether IELR removes unexpected LALR conflicts") do
267
430
  @options[:suggest_ielr] = true
268
431
  end
269
432
  options.on("--entry-isolation", "build independent state sets for each start symbol") do
270
- @options[:entry_isolation] = true
433
+ set_configuration_option(:entry_isolation, true)
271
434
  end
272
435
  options.on("--warnings=CATEGORIES", "all, error, all,error, or none") do |value|
273
436
  @options[:warnings] = warning_categories(value)
274
437
  end
275
438
  options.on("--watch", "regenerate file outputs when grammar sources change") { @options[:watch] = true }
276
439
  end
440
+ # rubocop:enable Metrics/MethodLength
277
441
 
278
442
  # @rbs (OptionParser options) -> void
279
443
  def add_output_options(options)
280
444
  options.on("-o", "--output-file=FILE", "generated parser path") { |value| @options[:output] = value }
281
- options.on("-E", "--embedded", "embed the Pure Ruby runtime") { @options[:embedded] = true }
282
- options.on("-t", "--debug", "generate a debug-capable parser") { @options[:debug] = true }
283
- options.on("-g", "obsolete alias for --debug") { @options[:debug] = true }
445
+ options.on("-E", "--embedded", "embed the Pure Ruby runtime") do
446
+ set_configuration_option(:embedded, true)
447
+ end
448
+ options.on("-t", "--debug", "generate a debug-capable parser") do
449
+ set_configuration_option(:debug, true)
450
+ end
451
+ options.on("-g", "obsolete alias for --debug") do
452
+ set_configuration_option(:debug, true)
453
+ end
284
454
  options.on("-v", "--verbose", "write an automaton report") { @options[:verbose] = true }
285
455
  add_counterexample_options(options)
286
456
  add_signature_output_options(options)
@@ -298,7 +468,7 @@ module Ibex
298
468
  @options[:log_file] = value
299
469
  end
300
470
  options.on("-e", "--executable [RUBY]", "add a shebang") do |value|
301
- @options[:executable] = value || "/usr/bin/env ruby"
471
+ set_configuration_option(:executable, value || "/usr/bin/env ruby")
302
472
  end
303
473
  end
304
474
 
@@ -314,17 +484,23 @@ module Ibex
314
484
 
315
485
  # @rbs (OptionParser options) -> void
316
486
  def add_compatibility_options(options)
317
- options.on("-F", "--frozen", "emit frozen string literals") { @options[:frozen] = true }
487
+ options.on("-F", "--frozen", "emit frozen string literals") do
488
+ set_configuration_option(:frozen, true)
489
+ end
318
490
  options.on("--line-convert-all", "convert all source lines") do
319
- @options[:line_convert] = true
320
491
  @options[:line_convert_all] = true
492
+ set_configuration_option(:line_convert, true)
321
493
  end
322
494
  options.on("-l", "--no-line-convert", "use generated-file action lines") do
323
- @options[:line_convert] = false
324
495
  @options[:line_convert_all] = false
496
+ set_configuration_option(:line_convert, false)
497
+ end
498
+ options.on("-a", "--no-omit-actions", "generate implicit action methods") do
499
+ set_configuration_option(:omit_actions, false)
500
+ end
501
+ options.on("--superclass=CLASS", "override parser superclass") do |value|
502
+ set_configuration_option(:superclass, value)
325
503
  end
326
- options.on("-a", "--no-omit-actions", "generate implicit action methods") { @options[:omit_actions] = false }
327
- options.on("--superclass=CLASS", "override parser superclass") { |value| @options[:superclass] = value }
328
504
  options.on("--check", "verify generated parser content without rewriting") { @options[:verify_output] = true }
329
505
  options.on("-C", "--check-only", "check grammar and exit") { @options[:check_only] = true }
330
506
  options.on("-S", "--output-status", "show pipeline status") { @options[:status] = true }
@@ -334,12 +510,55 @@ module Ibex
334
510
 
335
511
  # @rbs (OptionParser options) -> void
336
512
  def add_information_options(options)
513
+ options.on("--lang=LANG", "built-in diagnostic language (en or ja)") do |value|
514
+ @language = Messages.language(value)
515
+ end
337
516
  options.on("--version", "show version") { @options[:version] = true }
338
517
  options.on("--runtime-version", "show runtime version") { @options[:runtime_version] = true }
339
518
  options.on("--copyright", "show copyright") { @options[:copyright] = true }
340
519
  options.on("--help", "show help") { @options[:help] = true }
341
520
  end
342
521
 
522
+ # @rbs @analysis_configuration: Configuration::Resolver?
523
+
524
+ # @rbs (IR::Grammar grammar, ?options: Hash[Symbol, untyped], ?explicit_keys: Array[Symbol]) -> IR::Grammar
525
+ def activate_analysis_grammar(grammar, options: @options, explicit_keys: @configuration_explicit_options.keys)
526
+ adapter = Configuration::CLIAdapter.new(options, explicit_keys: explicit_keys)
527
+ cli = adapter.configuration_values
528
+ contract = grammar.parser_contract
529
+ grammar_values = contract&.configuration_values || {}
530
+ locations = contract&.configuration_locations || {}
531
+ overrides = {} #: Hash[String, Configuration::config_value]
532
+ algorithm = "parser.algorithm"
533
+ if grammar_values.key?(algorithm) && cli.key?(algorithm) &&
534
+ grammar_values.fetch(algorithm) != cli.fetch(algorithm)
535
+ overrides[algorithm] = cli.delete(algorithm)
536
+ end
537
+ source_locations = {} #: Hash[Symbol, Hash[String, Location]]
538
+ source_locations[:grammar] = locations unless locations.empty?
539
+ configuration = Configuration::Resolver.new(
540
+ grammar: grammar_values, cli: cli, analysis_overrides: overrides, locations: source_locations
541
+ )
542
+ @analysis_configuration = configuration
543
+ active = noncanonical_analysis_grammar(grammar, configuration)
544
+ @generation_grammar = active
545
+ report_noncanonical_analysis(configuration)
546
+ active
547
+ end
548
+
549
+ # @rbs (IR::Grammar grammar, Hash[Symbol, untyped] options, Array[Symbol] explicit_keys) ->
550
+ # [IR::Grammar, Symbol, IR::Automaton]
551
+ def construct_analysis_automaton(grammar, options, explicit_keys)
552
+ active = activate_analysis_grammar(grammar, options: options, explicit_keys: explicit_keys)
553
+ algorithm = configuration_value("parser.algorithm") #: Symbol
554
+ automaton = LALR::Builder.new(
555
+ active, algorithm: algorithm, ielr_strategy: options.fetch(:ielr_strategy, :partition),
556
+ remove_unreachable: options.fetch(:remove_unreachable, false),
557
+ entry_isolation: configuration_value("parser.entries") == :isolated
558
+ ).build
559
+ [active, algorithm, automaton]
560
+ end
561
+
343
562
  # @rbs (OptionParser parser) -> Integer?
344
563
  def informational_result(parser)
345
564
  return print_version if @options[:version] || @options[:runtime_version]
@@ -359,9 +578,7 @@ module Ibex
359
578
 
360
579
  # @rbs () -> void
361
580
  def validate_generation_options
362
- if @options[:entry_isolation] && @options[:from] == "automaton-ir"
363
- raise Ibex::Error, "(cli):1:1: --entry-isolation cannot be combined with --from=automaton-ir"
364
- end
581
+ validate_automaton_construction_options
365
582
 
366
583
  validate_watch_generation_options if @options[:watch]
367
584
  validate_manifest_generation_options
@@ -369,6 +586,18 @@ module Ibex
369
586
  validate_verification_generation_options
370
587
  end
371
588
 
589
+ # @rbs () -> void
590
+ def validate_automaton_construction_options
591
+ return unless @options[:from] == "automaton-ir"
592
+
593
+ if @configuration_explicit_options[:algorithm]
594
+ raise Ibex::Error, "(cli):1:1: --algorithm cannot be combined with --from=automaton-ir"
595
+ end
596
+ return unless @configuration_explicit_options[:entry_isolation]
597
+
598
+ raise Ibex::Error, "(cli):1:1: --entry-isolation cannot be combined with --from=automaton-ir"
599
+ end
600
+
372
601
  # @rbs () -> void
373
602
  def validate_manifest_generation_options
374
603
  return unless @options[:manifest]
@@ -511,20 +740,33 @@ module Ibex
511
740
  validate_generation_paths!(path, source_paths: resolution.files)
512
741
  return emit_ast(resolution.root) if @options[:emit] == "ast"
513
742
 
514
- grammar = Normalizer.new(resolution, mode: @options[:mode]).normalize
743
+ grammar = Normalizer.new(resolution, mode: configuration_value("grammar.mode")).normalize
515
744
  dispatch_grammar(grammar, path)
516
745
  end
517
746
 
518
747
  # @rbs (String path) -> Frontend::Resolution
519
748
  def resolve_grammar_path(path)
520
749
  loader = Frontend::SourceLoader.new(record_reads: true)
521
- @last_resolver = Frontend::Resolver.new(path, mode: @options[:mode], loader: loader)
750
+ @last_resolver = Frontend::Resolver.new(path, mode: configuration_value("grammar.mode"), loader: loader)
522
751
  @last_resolver.resolve
523
752
  end
524
753
 
525
754
  # @rbs (String path) -> IR::Grammar
526
755
  def normalize_grammar_path(path)
527
- Normalizer.new(resolve_grammar_path(path), mode: @options[:mode]).normalize
756
+ source = File.binread(path)
757
+ if bison_source?(source)
758
+ require_relative "bison_import"
759
+ imported = BisonImport::Importer.new(source, file: path).run
760
+ ast = Frontend::Parser.new(imported.source, file: path, mode: :extended).parse
761
+ return Normalizer.new(ast, mode: :extended).normalize
762
+ end
763
+
764
+ Normalizer.new(resolve_grammar_path(path), mode: configuration_value("grammar.mode")).normalize
765
+ end
766
+
767
+ # @rbs (String source) -> bool
768
+ def bison_source?(source)
769
+ source.lines.count { |line| line.match?(%r{^\s*%%(?:\s|/|$)}) } >= 2
528
770
  end
529
771
 
530
772
  # @rbs (String path) -> Integer
@@ -548,6 +790,7 @@ module Ibex
548
790
 
549
791
  # @rbs (IR::Grammar grammar, String path) -> Integer
550
792
  def dispatch_grammar(grammar, path)
793
+ activate_generation_grammar(grammar)
551
794
  handle_grammar_warnings(grammar, path)
552
795
  if @options[:check_only]
553
796
  build_automaton(grammar, path) if @options[:warnings]&.include?(:error)
@@ -566,6 +809,7 @@ module Ibex
566
809
 
567
810
  # @rbs (IR::Automaton automaton, String path) -> Integer
568
811
  def dispatch_automaton(automaton, path)
812
+ activate_generation_grammar(automaton.grammar)
569
813
  handle_grammar_warnings(automaton.grammar, path)
570
814
  return 0 if @options[:check_only]
571
815
 
@@ -583,6 +827,36 @@ module Ibex
583
827
  raise Ibex::Error, "(cli):1:1: AST cannot be reconstructed from Automaton IR"
584
828
  end
585
829
 
830
+ # @rbs (IR::Grammar grammar) -> void
831
+ def activate_generation_grammar(grammar)
832
+ @analysis_configuration = nil
833
+ @generation_grammar = grammar
834
+ effective_configuration
835
+ end
836
+
837
+ # @rbs (IR::Grammar grammar, Configuration::Resolver configuration) -> IR::Grammar
838
+ def noncanonical_analysis_grammar(grammar, configuration)
839
+ selection = configuration.fetch("parser.algorithm")
840
+ return grammar if selection.canonical
841
+
842
+ require_relative "configuration/analysis_grammar"
843
+ algorithm = selection.value #: Symbol
844
+ Configuration::AnalysisGrammar.for_algorithm(grammar, algorithm)
845
+ end
846
+
847
+ # @rbs (Configuration::Resolver configuration) -> void
848
+ def report_noncanonical_analysis(configuration)
849
+ selection = configuration.fetch("parser.algorithm")
850
+ return if selection.canonical
851
+
852
+ analysis = selection.to_h.fetch("analysis") #: Hash[String, Configuration::json_value]
853
+ @stderr.puts(
854
+ "noncanonical analysis configuration: parser.algorithm " \
855
+ "declared=#{analysis.fetch('declared')} selected=#{analysis.fetch('selected')} " \
856
+ "override=true canonical_generation=false"
857
+ )
858
+ end
859
+
586
860
  # @rbs () -> Integer
587
861
  def print_version
588
862
  @stdout.puts("ibex #{VERSION}")
@@ -654,18 +928,27 @@ module Ibex
654
928
 
655
929
  # @rbs (IR::Automaton automaton, String input_path) -> Integer
656
930
  def generate_ruby(automaton, input_path)
931
+ @generation_automaton = automaton
932
+ line_mapping = configuration_value("source.line_mapping") #: Symbol
933
+ executable = configuration_value("build.executable") #: String?
934
+ table = configuration_value("table.representation") #: Symbol | String
935
+ embedded = configuration_value("runtime.embedded") #: bool
936
+ debug = configuration_value("build.debug") #: bool
937
+ omit_action_call = configuration_value("actions.omit_calls") #: bool?
938
+ superclass = configuration_value("parser.superclass") #: String?
939
+ cst_trivia = configuration_value("cst.trivia") #: Symbol | String
657
940
  source = Codegen::Ruby.new(
658
- automaton, table: @options[:table], embedded: @options.fetch(:embedded, false),
659
- line_convert: @options.fetch(:line_convert), debug: @options.fetch(:debug, false),
660
- line_convert_all: @options.fetch(:line_convert_all, false),
661
- omit_action_call: @options[:omit_actions], superclass: @options[:superclass],
662
- executable: @options[:executable], cst_trivia: @options.fetch(:cst_trivia, :leading),
941
+ automaton, table: table, embedded: embedded,
942
+ line_convert: line_mapping != :none, debug: debug,
943
+ line_convert_all: line_mapping == :all,
944
+ omit_action_call: omit_action_call, superclass: superclass,
945
+ executable: executable, cst_trivia: cst_trivia,
663
946
  error_messages: configured_error_messages(automaton)
664
947
  ).generate
665
948
  output_path = @options[:output] || default_output_path(input_path, ".rb")
666
949
  action_source = action_source_source(automaton) if @options[:action_source]
667
950
  write_action_source(output_path, action_source) if action_source
668
- register_artifact(:parser, output_path, source, mode: (0o755 if @options[:executable]), status: true)
951
+ register_artifact(:parser, output_path, source, mode: (0o755 if executable), status: true)
669
952
  write_rbs(automaton, output_path) if @options[:rbs]
670
953
  finish_artifact_generation(@generation_sources)
671
954
  end
@@ -706,8 +989,10 @@ module Ibex
706
989
  # @rbs (IR::Automaton automaton) -> String
707
990
  def rbs_source(automaton)
708
991
  require_relative "codegen/rbs"
992
+ superclass = configuration_value("parser.superclass") #: String?
993
+ omit_action_call = configuration_value("actions.omit_calls") #: bool?
709
994
  Codegen::RBS.new(
710
- automaton, superclass: @options[:superclass], omit_action_call: @options[:omit_actions]
995
+ automaton, superclass: superclass, omit_action_call: omit_action_call
711
996
  ).generate
712
997
  end
713
998
 
@@ -729,7 +1014,9 @@ module Ibex
729
1014
  # @rbs (IR::Automaton automaton) -> String
730
1015
  def action_source_source(automaton)
731
1016
  require_relative "codegen/action_source"
732
- Codegen::ActionSource.new(automaton, omit_action_call: @options[:omit_actions]).generate
1017
+ Codegen::ActionSource.new(
1018
+ automaton, omit_action_call: configuration_value("actions.omit_calls")
1019
+ ).generate
733
1020
  end
734
1021
 
735
1022
  # @rbs (IR::Automaton automaton, String input_path) -> Integer
@@ -742,11 +1029,14 @@ module Ibex
742
1029
 
743
1030
  # @rbs (IR::Grammar grammar, String input_path) -> IR::Automaton
744
1031
  def build_automaton(grammar, input_path)
745
- algorithm = @options[:algorithm] || :lalr
1032
+ algorithm = configuration_value("parser.algorithm")
746
1033
  report_status("building #{algorithm} automaton")
747
1034
  automaton = LALR::Builder.new(
748
- grammar, algorithm: algorithm, entry_isolation: @options[:entry_isolation] == true
1035
+ grammar, algorithm: algorithm, ielr_strategy: @options.fetch(:ielr_strategy, :partition),
1036
+ remove_unreachable: @options.fetch(:remove_unreachable, false),
1037
+ entry_isolation: configuration_value("parser.entries") == :isolated
749
1038
  ).build
1039
+ write_ielr_report(grammar, automaton) if @options[:ielr_report]
750
1040
  report_conflicts(automaton, input_path)
751
1041
  suggest_ielr(automaton, input_path)
752
1042
  write_report(automaton, input_path) if @options[:verbose] && !@options[:verify_output]
@@ -754,6 +1044,12 @@ module Ibex
754
1044
  automaton
755
1045
  end
756
1046
 
1047
+ # @rbs (IR::Grammar grammar, IR::Automaton automaton) -> void
1048
+ def write_ielr_report(grammar, automaton)
1049
+ canonical = LALR::Builder.new(grammar, algorithm: :lr1).build
1050
+ @stdout.write(LALR::InadequacyReport.new(canonical, automaton).to_json)
1051
+ end
1052
+
757
1053
  # @rbs (IR::Automaton automaton, String input_path) -> void
758
1054
  def prepare_loaded_automaton(automaton, input_path)
759
1055
  report_conflicts(automaton, input_path)
@@ -798,5 +1094,10 @@ module Ibex
798
1094
  default_output_path(output_path, ".ibex.json")
799
1095
  end
800
1096
  end
1097
+ CONFIG_SUBCOMMAND_HELP = [
1098
+ " coverage collect, merge, or check runtime coverage",
1099
+ " config explain effective configuration without running user code"
1100
+ ].join("\n").freeze #: String
1101
+ autoload :CLIConfig, File.join(CLI_FEATURE_ROOT, "config")
801
1102
  # rubocop:enable Metrics/ClassLength
802
1103
  end