ibex 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (603) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +89 -123
  3. data/docs/architecture.md +477 -31
  4. data/docs/bison-import.md +103 -0
  5. data/docs/comparison-policy.md +127 -0
  6. data/docs/configuration-model.md +59 -0
  7. data/docs/configuration-report.md +28 -0
  8. data/docs/conflict-explanation-reviews/v1/records/README.md +28 -0
  9. data/docs/conflict-explanation-study.md +62 -0
  10. data/docs/construction-profiling.md +157 -0
  11. data/docs/cst-migration.md +106 -0
  12. data/docs/cst.md +237 -0
  13. data/docs/decisions/0000-template.md +24 -0
  14. data/docs/decisions/0001-separate-ir-pipeline.md +34 -0
  15. data/docs/decisions/0002-opaque-user-code-boundary.md +32 -0
  16. data/docs/decisions/0003-self-hosted-grammar-frontend.md +32 -0
  17. data/docs/decisions/0004-shared-semantic-and-lossless-source-model.md +33 -0
  18. data/docs/decisions/0005-contained-grammar-composition.md +34 -0
  19. data/docs/decisions/0006-bounded-structural-grammar-lowering.md +32 -0
  20. data/docs/decisions/0007-shared-parser-construction-pipeline.md +33 -0
  21. data/docs/decisions/0008-versioned-runtime-package-boundary.md +36 -0
  22. data/docs/decisions/0009-isolated-parser-sessions.md +34 -0
  23. data/docs/decisions/0010-committed-runtime-observation.md +34 -0
  24. data/docs/decisions/0011-versioned-semantic-action-boundary.md +37 -0
  25. data/docs/decisions/0012-bounded-nonexecuting-analysis.md +37 -0
  26. data/docs/decisions/0013-transactional-generation-publication.md +37 -0
  27. data/docs/decisions/0014-versioned-generated-lexer.md +35 -0
  28. data/docs/decisions/0015-worker-isolated-browser-analysis.md +32 -0
  29. data/docs/decisions/0016-red-green-concrete-syntax.md +34 -0
  30. data/docs/decisions/0017-persistent-syntax-artifacts.md +35 -0
  31. data/docs/decisions/0018-conservative-incremental-syntax-reuse.md +38 -0
  32. data/docs/decisions/0019-runtime-syntax-session-boundary.md +51 -0
  33. data/docs/decisions/0020-grammar-ir-parser-contract.md +65 -0
  34. data/docs/decisions/0021-data-only-parser-table-sidecar.md +50 -0
  35. data/docs/decisions/0022-manifest-bound-verification-report.md +77 -0
  36. data/docs/decisions/0023-syntax-only-repair-results.md +48 -0
  37. data/docs/decisions/0024-direct-ielr-construction.md +42 -0
  38. data/docs/decisions/README.md +74 -0
  39. data/docs/declarative-configuration.md +262 -0
  40. data/docs/development.md +306 -0
  41. data/docs/direct-ielr-decision.md +120 -0
  42. data/docs/direct-multi-entry-decision.md +82 -0
  43. data/docs/editor-setup.md +26 -0
  44. data/docs/error-ux-review-rubric-v1.md +117 -0
  45. data/docs/error-ux-reviews/v1/records/README.md +22 -0
  46. data/docs/error-ux-round2-review-status-v1.json +27 -0
  47. data/docs/error-ux-round2-reviews/v1/records/README.md +30 -0
  48. data/docs/error-ux-round2-v1.json +1581 -0
  49. data/docs/error-ux-round2.md +111 -0
  50. data/docs/error-ux.md +114 -0
  51. data/docs/getting-started.md +80 -0
  52. data/docs/grammar-reference.md +629 -18
  53. data/docs/ielr-design.md +43 -0
  54. data/docs/ielr.md +92 -0
  55. data/docs/ja/bison-import.md +51 -0
  56. data/docs/ja/stable-api.md +53 -0
  57. data/docs/lexer-construction-profile.md +109 -0
  58. data/docs/lexer-migration.md +56 -0
  59. data/docs/maturity.md +128 -0
  60. data/docs/project-site-strategy.md +54 -0
  61. data/docs/racc-migration-evidence.md +45 -0
  62. data/docs/racc-migration.md +45 -6
  63. data/docs/release-readiness.md +221 -0
  64. data/docs/repair-semantics.md +123 -0
  65. data/docs/runtime-abi-evolution.md +307 -0
  66. data/docs/stability.md +234 -0
  67. data/docs/status.md +43 -0
  68. data/docs/syntax-sessions.md +183 -0
  69. data/docs/table-artifact.md +125 -0
  70. data/docs/test-interactions.md +219 -0
  71. data/docs/verification-report.md +137 -0
  72. data/docs/verifier-trust-boundary.md +199 -0
  73. data/docs/workloads.md +132 -0
  74. data/examples/README.md +60 -0
  75. data/examples/calculator.y +44 -0
  76. data/examples/csv.y +32 -0
  77. data/examples/ini.y +70 -0
  78. data/examples/json.y +48 -0
  79. data/examples/tiny_language.y +75 -0
  80. data/lib/ibex/analysis/digraph.rb +128 -0
  81. data/lib/ibex/analysis/sets.rb +5 -4
  82. data/lib/ibex/analysis.rb +1 -0
  83. data/lib/ibex/artifact_set.rb +63 -0
  84. data/lib/ibex/bison_import/importer.rb +516 -0
  85. data/lib/ibex/bison_import/tokenizer.rb +260 -0
  86. data/lib/ibex/bison_import.rb +200 -0
  87. data/lib/ibex/bounded_subprocess.rb +171 -0
  88. data/lib/ibex/cli/ambiguity.rb +75 -0
  89. data/lib/ibex/cli/analysis.rb +134 -0
  90. data/lib/ibex/cli/bison_import.rb +121 -0
  91. data/lib/ibex/cli/config.rb +128 -0
  92. data/lib/ibex/cli/counterexample_options.rb +6 -2
  93. data/lib/ibex/cli/coverage.rb +173 -0
  94. data/lib/ibex/cli/debug.rb +106 -0
  95. data/lib/ibex/cli/diagnostics.rb +136 -0
  96. data/lib/ibex/cli/documentation.rb +70 -0
  97. data/lib/ibex/cli/equiv.rb +169 -0
  98. data/lib/ibex/cli/error_messages.rb +184 -0
  99. data/lib/ibex/cli/explain.rb +85 -0
  100. data/lib/ibex/cli/fix.rb +203 -0
  101. data/lib/ibex/cli/formatting.rb +387 -0
  102. data/lib/ibex/cli/fuzz.rb +200 -0
  103. data/lib/ibex/cli/fuzz_regressions.rb +145 -0
  104. data/lib/ibex/cli/generation_artifacts.rb +189 -0
  105. data/lib/ibex/cli/generation_error_messages.rb +39 -0
  106. data/lib/ibex/cli/grammar_tests.rb +140 -0
  107. data/lib/ibex/cli/ir_tools.rb +128 -0
  108. data/lib/ibex/cli/lsp.rb +33 -0
  109. data/lib/ibex/cli/outputs.rb +133 -23
  110. data/lib/ibex/cli/racc_migration.rb +87 -0
  111. data/lib/ibex/cli/reduce.rb +202 -0
  112. data/lib/ibex/cli/reduce_reporting.rb +71 -0
  113. data/lib/ibex/cli/samples.rb +118 -0
  114. data/lib/ibex/cli/verify.rb +116 -0
  115. data/lib/ibex/cli/watch.rb +104 -0
  116. data/lib/ibex/cli.rb +825 -55
  117. data/lib/ibex/codegen/action_locations.rb +103 -0
  118. data/lib/ibex/codegen/action_method_source.rb +209 -0
  119. data/lib/ibex/codegen/action_source.rb +119 -0
  120. data/lib/ibex/codegen/ambiguity.rb +174 -0
  121. data/lib/ibex/codegen/cst_metadata.rb +171 -0
  122. data/lib/ibex/codegen/documentation.rb +170 -0
  123. data/lib/ibex/codegen/explain.rb +380 -0
  124. data/lib/ibex/codegen/generated_action_abi.rb +293 -0
  125. data/lib/ibex/codegen/html.rb +94 -10
  126. data/lib/ibex/codegen/mermaid.rb +43 -0
  127. data/lib/ibex/codegen/railroad.rb +197 -0
  128. data/lib/ibex/codegen/railroad_documentation.rb +59 -0
  129. data/lib/ibex/codegen/rbs.rb +331 -2
  130. data/lib/ibex/codegen/report.rb +67 -15
  131. data/lib/ibex/codegen/ruby.rb +305 -66
  132. data/lib/ibex/codegen/ruby_actions.rb +146 -0
  133. data/lib/ibex/codegen/ruby_ast.rb +121 -0
  134. data/lib/ibex/codegen/ruby_error_messages.rb +28 -0
  135. data/lib/ibex/codegen/ruby_lexer.rb +83 -0
  136. data/lib/ibex/codegen/ruby_syntax.rb +96 -0
  137. data/lib/ibex/codegen/ruby_table_metadata.rb +57 -0
  138. data/lib/ibex/codegen/ruby_value_printers.rb +56 -0
  139. data/lib/ibex/codegen/symbol_labels.rb +1 -1
  140. data/lib/ibex/configuration/analysis_grammar.rb +35 -0
  141. data/lib/ibex/configuration/explanation.rb +421 -0
  142. data/lib/ibex/configuration/inspector.rb +232 -0
  143. data/lib/ibex/configuration.rb +580 -0
  144. data/lib/ibex/coverage/collector.rb +198 -0
  145. data/lib/ibex/coverage/event_stream.rb +103 -0
  146. data/lib/ibex/coverage/report.rb +267 -0
  147. data/lib/ibex/coverage/runtime_event_validator.rb +169 -0
  148. data/lib/ibex/coverage.rb +13 -0
  149. data/lib/ibex/delta_reducer.rb +99 -0
  150. data/lib/ibex/diff.rb +140 -0
  151. data/lib/ibex/equiv/machine.rb +135 -0
  152. data/lib/ibex/equiv.rb +373 -0
  153. data/lib/ibex/error_messages/parser.rb +159 -0
  154. data/lib/ibex/error_messages/parser_v2.rb +198 -0
  155. data/lib/ibex/error_messages/renderer.rb +65 -0
  156. data/lib/ibex/error_messages/sentence_search.rb +196 -0
  157. data/lib/ibex/error_messages/update.rb +169 -0
  158. data/lib/ibex/error_messages.rb +165 -0
  159. data/lib/ibex/fix.rb +557 -0
  160. data/lib/ibex/frontend/ast.rb +166 -7
  161. data/lib/ibex/frontend/bootstrap_parser.rb +40 -4
  162. data/lib/ibex/frontend/diagnostic.rb +81 -0
  163. data/lib/ibex/frontend/diagnostic_recovery.rb +268 -0
  164. data/lib/ibex/frontend/dsl.rb +66 -7
  165. data/lib/ibex/frontend/formatter.rb +453 -0
  166. data/lib/ibex/frontend/generated_parser.rb +644 -190
  167. data/lib/ibex/frontend/generated_parser_base.rb +170 -32
  168. data/lib/ibex/frontend/generated_parser_includes.rb +51 -0
  169. data/lib/ibex/frontend/generated_parser_metadata.rb +61 -0
  170. data/lib/ibex/frontend/generated_parser_parameters.rb +60 -0
  171. data/lib/ibex/frontend/generation.rb +33 -0
  172. data/lib/ibex/frontend/lexer.rb +162 -10
  173. data/lib/ibex/frontend/lexer_recovery.rb +84 -0
  174. data/lib/ibex/frontend/parser/declarations.rb +262 -11
  175. data/lib/ibex/frontend/parser/parameters.rb +82 -0
  176. data/lib/ibex/frontend/parser/rules.rb +43 -6
  177. data/lib/ibex/frontend/parser.rb +182 -4
  178. data/lib/ibex/frontend/parser_configuration_support.rb +56 -0
  179. data/lib/ibex/frontend/regenerator.rb +27 -1
  180. data/lib/ibex/frontend/resolution.rb +70 -0
  181. data/lib/ibex/frontend/resolver.rb +217 -0
  182. data/lib/ibex/frontend/rule_documentation.rb +104 -0
  183. data/lib/ibex/frontend/source_cursor.rb +133 -9
  184. data/lib/ibex/frontend/source_document.rb +229 -0
  185. data/lib/ibex/frontend/source_loader.rb +150 -0
  186. data/lib/ibex/frontend/source_span.rb +89 -0
  187. data/lib/ibex/frontend/token_adapter/declaration_document_state.rb +47 -0
  188. data/lib/ibex/frontend/token_adapter/declaration_lexer_state.rb +83 -0
  189. data/lib/ibex/frontend/token_adapter/declaration_state.rb +244 -22
  190. data/lib/ibex/frontend/token_adapter/delimiter_tracker.rb +8 -2
  191. data/lib/ibex/frontend/token_adapter/rule_state.rb +60 -2
  192. data/lib/ibex/frontend/token_adapter.rb +18 -3
  193. data/lib/ibex/frontend.rb +14 -2
  194. data/lib/ibex/fuzz.rb +219 -0
  195. data/lib/ibex/generation_input.rb +57 -0
  196. data/lib/ibex/generation_manifest.rb +211 -0
  197. data/lib/ibex/generation_transaction.rb +261 -0
  198. data/lib/ibex/generation_transaction_recovery.rb +109 -0
  199. data/lib/ibex/generation_transaction_validation.rb +196 -0
  200. data/lib/ibex/grammar_tests.rb +206 -0
  201. data/lib/ibex/ir/automaton_ir.rb +86 -11
  202. data/lib/ibex/ir/grammar_ir.rb +170 -26
  203. data/lib/ibex/ir/lexer_ir.rb +76 -0
  204. data/lib/ibex/ir/parser_contract.rb +135 -0
  205. data/lib/ibex/ir/serialize.rb +268 -50
  206. data/lib/ibex/ir/validator/automaton.rb +375 -0
  207. data/lib/ibex/ir/validator/base.rb +147 -0
  208. data/lib/ibex/ir/validator/grammar.rb +634 -0
  209. data/lib/ibex/ir/validator/lexer.rb +113 -0
  210. data/lib/ibex/ir/validator.rb +63 -0
  211. data/lib/ibex/ir.rb +71 -5
  212. data/lib/ibex/lalr/build_metrics.rb +71 -0
  213. data/lib/ibex/lalr/builder.rb +514 -52
  214. data/lib/ibex/lalr/conflict.rb +1 -0
  215. data/lib/ibex/lalr/conflict_search.rb +26 -9
  216. data/lib/ibex/lalr/counterexample.rb +38 -18
  217. data/lib/ibex/lalr/direct_lookaheads.rb +289 -0
  218. data/lib/ibex/lalr/goto_follows.rb +229 -0
  219. data/lib/ibex/lalr/ielr/annotator.rb +214 -0
  220. data/lib/ibex/lalr/ielr/bits.rb +28 -0
  221. data/lib/ibex/lalr/ielr/inadequacy.rb +47 -0
  222. data/lib/ibex/lalr/ielr/item_lookaheads.rb +78 -0
  223. data/lib/ibex/lalr/ielr/pipeline.rb +75 -0
  224. data/lib/ibex/lalr/ielr/split_stability.rb +78 -0
  225. data/lib/ibex/lalr/ielr/split_state.rb +20 -0
  226. data/lib/ibex/lalr/ielr/state_splitter.rb +258 -0
  227. data/lib/ibex/lalr/ielr_partition.rb +166 -0
  228. data/lib/ibex/lalr/inadequacy_report.rb +50 -0
  229. data/lib/ibex/lalr/lookahead_propagation.rb +111 -0
  230. data/lib/ibex/lalr/lr0_collection.rb +121 -0
  231. data/lib/ibex/lalr/on_error_reductions.rb +74 -0
  232. data/lib/ibex/lalr/unreachable_states.rb +80 -0
  233. data/lib/ibex/lalr.rb +59 -1
  234. data/lib/ibex/location.rb +129 -0
  235. data/lib/ibex/lsp/document_handlers.rb +68 -0
  236. data/lib/ibex/lsp/document_store.rb +264 -0
  237. data/lib/ibex/lsp/document_store_diagnostics.rb +44 -0
  238. data/lib/ibex/lsp/document_store_validation.rb +61 -0
  239. data/lib/ibex/lsp/initialization_handlers.rb +80 -0
  240. data/lib/ibex/lsp/navigation_handlers.rb +70 -0
  241. data/lib/ibex/lsp/parser_configuration_assistance.rb +149 -0
  242. data/lib/ibex/lsp/position_codec.rb +113 -0
  243. data/lib/ibex/lsp/protocol_error.rb +33 -0
  244. data/lib/ibex/lsp/request_handlers.rb +46 -0
  245. data/lib/ibex/lsp/request_support.rb +94 -0
  246. data/lib/ibex/lsp/server.rb +148 -0
  247. data/lib/ibex/lsp/symbol_index.rb +241 -0
  248. data/lib/ibex/lsp/symbol_index_builder.rb +277 -0
  249. data/lib/ibex/lsp/symbol_index_precedence_references.rb +44 -0
  250. data/lib/ibex/lsp/symbol_index_source_queries.rb +61 -0
  251. data/lib/ibex/lsp/symbol_occurrence.rb +23 -0
  252. data/lib/ibex/lsp/transport.rb +128 -0
  253. data/lib/ibex/lsp/workspace.rb +128 -0
  254. data/lib/ibex/lsp/workspace_analyzer.rb +199 -0
  255. data/lib/ibex/lsp.rb +33 -0
  256. data/lib/ibex/messages/en.yml +28 -0
  257. data/lib/ibex/messages/ja.yml +28 -0
  258. data/lib/ibex/messages.rb +68 -0
  259. data/lib/ibex/metrics.rb +175 -0
  260. data/lib/ibex/normalize/declarations.rb +142 -7
  261. data/lib/ibex/normalize/diagnostics.rb +94 -24
  262. data/lib/ibex/normalize/expander.rb +64 -59
  263. data/lib/ibex/normalize/expression.rb +63 -40
  264. data/lib/ibex/normalize/grammar_builder.rb +26 -0
  265. data/lib/ibex/normalize/inline_expansion.rb +465 -0
  266. data/lib/ibex/normalize/inline_validation.rb +174 -0
  267. data/lib/ibex/normalize/lexer.rb +131 -0
  268. data/lib/ibex/normalize/named_references.rb +60 -0
  269. data/lib/ibex/normalize/nodes.rb +46 -0
  270. data/lib/ibex/normalize/parameter_ebnf_lowering.rb +70 -0
  271. data/lib/ibex/normalize/parameter_lowering.rb +143 -0
  272. data/lib/ibex/normalize/parameter_substitution.rb +125 -0
  273. data/lib/ibex/normalize/parameter_validation.rb +140 -0
  274. data/lib/ibex/normalize/parameters.rb +226 -0
  275. data/lib/ibex/normalize/parser_configuration.rb +70 -0
  276. data/lib/ibex/normalize/recovery_declarations.rb +84 -0
  277. data/lib/ibex/normalize.rb +172 -17
  278. data/lib/ibex/racc_migration/checker.rb +122 -0
  279. data/lib/ibex/racc_migration/harness.rb +177 -0
  280. data/lib/ibex/racc_migration/report.rb +97 -0
  281. data/lib/ibex/racc_migration.rb +12 -0
  282. data/lib/ibex/rake_task.rb +116 -0
  283. data/lib/ibex/samples.rb +216 -0
  284. data/lib/ibex/table_artifact/builder.rb +266 -0
  285. data/lib/ibex/table_artifact/cst_projection.rb +70 -0
  286. data/lib/ibex/table_artifact/document.rb +39 -0
  287. data/lib/ibex/table_artifact/executor.rb +187 -0
  288. data/lib/ibex/table_artifact/serializer.rb +57 -0
  289. data/lib/ibex/table_artifact/validator/metadata.rb +179 -0
  290. data/lib/ibex/table_artifact/validator/support.rb +86 -0
  291. data/lib/ibex/table_artifact/validator/tables.rb +238 -0
  292. data/lib/ibex/table_artifact/validator.rb +253 -0
  293. data/lib/ibex/table_artifact.rb +88 -0
  294. data/lib/ibex/table_simulation/result.rb +55 -0
  295. data/lib/ibex/table_simulation/simulator.rb +253 -0
  296. data/lib/ibex/table_simulation/step.rb +62 -0
  297. data/lib/ibex/table_simulation/text.rb +31 -0
  298. data/lib/ibex/table_simulation.rb +13 -0
  299. data/lib/ibex/tables.rb +22 -69
  300. data/lib/ibex/verifiable_generation_bundle.rb +86 -0
  301. data/lib/ibex/verification_report/builder.rb +159 -0
  302. data/lib/ibex/verification_report/canonical_ir.rb +123 -0
  303. data/lib/ibex/verification_report/logical_path.rb +68 -0
  304. data/lib/ibex/verification_report/validator.rb +329 -0
  305. data/lib/ibex/verification_report.rb +69 -0
  306. data/lib/ibex/verify/action_correspondence.rb +138 -0
  307. data/lib/ibex/verify/language_witness.rb +295 -0
  308. data/lib/ibex/verify/reference_collection.rb +190 -0
  309. data/lib/ibex/verify/result.rb +73 -0
  310. data/lib/ibex/verify/verifier.rb +598 -0
  311. data/lib/ibex/verify.rb +15 -0
  312. data/lib/ibex/version.rb +1 -1
  313. data/lib/ibex/watch/runner.rb +181 -0
  314. data/lib/ibex/watch/source_snapshot.rb +98 -0
  315. data/lib/ibex/watch.rb +11 -0
  316. data/lib/ibex.rb +35 -1
  317. data/schema/automaton-ir-definitions.schema.json +401 -0
  318. data/schema/automaton-ir.schema.json +65 -0
  319. data/schema/benchmark-v1.schema.json +212 -0
  320. data/schema/benchmark-v2.schema.json +61 -0
  321. data/schema/bison-import-v1.schema.json +106 -0
  322. data/schema/conflict-explanation-review-v1.schema.json +196 -0
  323. data/schema/conflict-explanation-study-v1.schema.json +528 -0
  324. data/schema/construction-profile-v1.schema.json +779 -0
  325. data/schema/cst-v1.json +170 -0
  326. data/schema/diff-v1.schema.json +41 -0
  327. data/schema/direct-ielr-decision-v1.schema.json +200 -0
  328. data/schema/equiv-v1.schema.json +64 -0
  329. data/schema/error-ux-review-v1.schema.json +609 -0
  330. data/schema/error-ux-round2-review-v1.schema.json +191 -0
  331. data/schema/error-ux-round2-v1.schema.json +519 -0
  332. data/schema/error-ux-v1.schema.json +258 -0
  333. data/schema/explain-v1.schema.json +495 -0
  334. data/schema/fix-v1.schema.json +64 -0
  335. data/schema/fix-v2.schema.json +66 -0
  336. data/schema/fix-v3.schema.json +101 -0
  337. data/schema/frontend-diagnostics-v1.schema.json +154 -0
  338. data/schema/fuzz-regression-v1.schema.json +79 -0
  339. data/schema/fuzz-v1.schema.json +114 -0
  340. data/schema/generation-manifest-v1.schema.json +115 -0
  341. data/schema/grammar-ir-extensions.schema.json +734 -0
  342. data/schema/grammar-ir-foundation.schema.json +426 -0
  343. data/schema/grammar-ir.schema.json +271 -0
  344. data/schema/ielr-benchmark-v1.schema.json +85 -0
  345. data/schema/lexer-ir-v1.schema.json +215 -0
  346. data/schema/lexer-profile-v1.schema.json +491 -0
  347. data/schema/metrics-v1.schema.json +60 -0
  348. data/schema/migration-check-v1.schema.json +60 -0
  349. data/schema/performance-comparison-v1.schema.json +395 -0
  350. data/schema/public-performance-comparison-v1.schema.json +506 -0
  351. data/schema/public-performance-profile-v1.schema.json +360 -0
  352. data/schema/reduce-v1.schema.json +22 -0
  353. data/schema/reduce-v2.schema.json +63 -0
  354. data/schema/runtime-coverage-v1.schema.json +86 -0
  355. data/schema/runtime-event-v1.schema.json +308 -0
  356. data/schema/table-artifact-v1.schema.json +377 -0
  357. data/schema/table-simulation-v1.schema.json +70 -0
  358. data/schema/verification-report-v1.schema.json +222 -0
  359. data/schema/verify-v1.schema.json +42 -0
  360. data/sig/ibex/analysis/digraph.rbs +19 -0
  361. data/sig/ibex/artifact_set.rbs +37 -0
  362. data/sig/ibex/bison_import/importer.rbs +140 -0
  363. data/sig/ibex/bison_import/tokenizer.rbs +91 -0
  364. data/sig/ibex/bison_import.rbs +95 -0
  365. data/sig/ibex/bounded_subprocess.rbs +76 -0
  366. data/sig/ibex/cli/ambiguity.rbs +28 -0
  367. data/sig/ibex/cli/analysis.rbs +43 -0
  368. data/sig/ibex/cli/bison_import.rbs +32 -0
  369. data/sig/ibex/cli/config.rbs +35 -0
  370. data/sig/ibex/cli/counterexample_options.rbs +2 -0
  371. data/sig/ibex/cli/coverage.rbs +53 -0
  372. data/sig/ibex/cli/debug.rbs +28 -0
  373. data/sig/ibex/cli/diagnostics.rbs +51 -0
  374. data/sig/ibex/cli/documentation.rbs +29 -0
  375. data/sig/ibex/cli/equiv.rbs +49 -0
  376. data/sig/ibex/cli/error_messages.rbs +63 -0
  377. data/sig/ibex/cli/explain.rbs +31 -0
  378. data/sig/ibex/cli/fix.rbs +53 -0
  379. data/sig/ibex/cli/formatting.rbs +107 -0
  380. data/sig/ibex/cli/fuzz.rbs +47 -0
  381. data/sig/ibex/cli/fuzz_regressions.rbs +40 -0
  382. data/sig/ibex/cli/generation_artifacts.rbs +69 -0
  383. data/sig/ibex/cli/generation_error_messages.rbs +19 -0
  384. data/sig/ibex/cli/grammar_tests.rbs +48 -0
  385. data/sig/ibex/cli/ir_tools.rbs +39 -0
  386. data/sig/ibex/cli/lsp.rbs +14 -0
  387. data/sig/ibex/cli/outputs.rbs +26 -3
  388. data/sig/ibex/cli/racc_migration.rbs +30 -0
  389. data/sig/ibex/cli/reduce.rbs +60 -0
  390. data/sig/ibex/cli/reduce_reporting.rbs +27 -0
  391. data/sig/ibex/cli/samples.rbs +38 -0
  392. data/sig/ibex/cli/verify.rbs +28 -0
  393. data/sig/ibex/cli/watch.rbs +43 -0
  394. data/sig/ibex/cli.rbs +182 -9
  395. data/sig/ibex/codegen/action_locations.rbs +45 -0
  396. data/sig/ibex/codegen/action_method_source.rbs +65 -0
  397. data/sig/ibex/codegen/action_source.rbs +50 -0
  398. data/sig/ibex/codegen/ambiguity.rbs +60 -0
  399. data/sig/ibex/codegen/cst_metadata.rbs +59 -0
  400. data/sig/ibex/codegen/documentation.rbs +50 -0
  401. data/sig/ibex/codegen/explain.rbs +105 -0
  402. data/sig/ibex/codegen/generated_action_abi.rbs +105 -0
  403. data/sig/ibex/codegen/html.rbs +18 -2
  404. data/sig/ibex/codegen/mermaid.rbs +16 -0
  405. data/sig/ibex/codegen/railroad.rbs +82 -0
  406. data/sig/ibex/codegen/railroad_documentation.rbs +31 -0
  407. data/sig/ibex/codegen/rbs.rbs +86 -4
  408. data/sig/ibex/codegen/report.rbs +16 -8
  409. data/sig/ibex/codegen/ruby.rbs +105 -28
  410. data/sig/ibex/codegen/ruby_actions.rbs +54 -0
  411. data/sig/ibex/codegen/ruby_ast.rbs +34 -0
  412. data/sig/ibex/codegen/ruby_error_messages.rbs +16 -0
  413. data/sig/ibex/codegen/ruby_lexer.rbs +25 -0
  414. data/sig/ibex/codegen/ruby_syntax.rbs +27 -0
  415. data/sig/ibex/codegen/ruby_table_metadata.rbs +26 -0
  416. data/sig/ibex/codegen/ruby_value_printers.rbs +28 -0
  417. data/sig/ibex/configuration/analysis_grammar.rbs +13 -0
  418. data/sig/ibex/configuration/explanation.rbs +195 -0
  419. data/sig/ibex/configuration/inspector.rbs +53 -0
  420. data/sig/ibex/configuration.rbs +242 -0
  421. data/sig/ibex/coverage/collector.rbs +78 -0
  422. data/sig/ibex/coverage/event_stream.rbs +44 -0
  423. data/sig/ibex/coverage/report.rbs +102 -0
  424. data/sig/ibex/coverage/runtime_event_validator.rbs +70 -0
  425. data/sig/ibex/coverage.rbs +7 -0
  426. data/sig/ibex/delta_reducer.rbs +38 -0
  427. data/sig/ibex/diff.rbs +47 -0
  428. data/sig/ibex/equiv/machine.rbs +54 -0
  429. data/sig/ibex/equiv.rbs +129 -0
  430. data/sig/ibex/error_messages/parser.rbs +58 -0
  431. data/sig/ibex/error_messages/parser_v2.rbs +67 -0
  432. data/sig/ibex/error_messages/renderer.rbs +23 -0
  433. data/sig/ibex/error_messages/sentence_search.rbs +80 -0
  434. data/sig/ibex/error_messages/update.rbs +43 -0
  435. data/sig/ibex/error_messages.rbs +85 -0
  436. data/sig/ibex/fix.rbs +138 -0
  437. data/sig/ibex/frontend/ast.rbs +237 -22
  438. data/sig/ibex/frontend/bootstrap_parser.rbs +13 -0
  439. data/sig/ibex/frontend/diagnostic.rbs +53 -0
  440. data/sig/ibex/frontend/diagnostic_recovery.rbs +98 -0
  441. data/sig/ibex/frontend/dsl.rbs +33 -4
  442. data/sig/ibex/frontend/formatter.rbs +143 -0
  443. data/sig/ibex/frontend/generated_parser.rbs +228 -68
  444. data/sig/ibex/frontend/generated_parser_base.rbs +65 -12
  445. data/sig/ibex/frontend/generated_parser_includes.rbs +23 -0
  446. data/sig/ibex/frontend/generated_parser_metadata.rbs +23 -0
  447. data/sig/ibex/frontend/generated_parser_parameters.rbs +24 -0
  448. data/sig/ibex/frontend/generation.rbs +6 -0
  449. data/sig/ibex/frontend/lexer.rbs +49 -2
  450. data/sig/ibex/frontend/lexer_recovery.rbs +25 -0
  451. data/sig/ibex/frontend/parser/declarations.rbs +57 -0
  452. data/sig/ibex/frontend/parser/parameters.rbs +28 -0
  453. data/sig/ibex/frontend/parser/rules.rbs +3 -0
  454. data/sig/ibex/frontend/parser.rbs +66 -0
  455. data/sig/ibex/frontend/parser_configuration_support.rbs +27 -0
  456. data/sig/ibex/frontend/regenerator.rbs +11 -0
  457. data/sig/ibex/frontend/resolution.rbs +34 -0
  458. data/sig/ibex/frontend/resolver.rbs +91 -0
  459. data/sig/ibex/frontend/rule_documentation.rbs +42 -0
  460. data/sig/ibex/frontend/source_cursor.rbs +38 -5
  461. data/sig/ibex/frontend/source_document.rbs +119 -0
  462. data/sig/ibex/frontend/source_loader.rbs +66 -0
  463. data/sig/ibex/frontend/source_span.rbs +56 -0
  464. data/sig/ibex/frontend/token_adapter/declaration_document_state.rbs +21 -0
  465. data/sig/ibex/frontend/token_adapter/declaration_lexer_state.rbs +27 -0
  466. data/sig/ibex/frontend/token_adapter/declaration_state.rbs +83 -3
  467. data/sig/ibex/frontend/token_adapter/rule_state.rbs +20 -0
  468. data/sig/ibex/frontend/token_adapter.rbs +11 -2
  469. data/sig/ibex/frontend.rbs +1 -1
  470. data/sig/ibex/fuzz.rbs +71 -0
  471. data/sig/ibex/generation_input.rbs +37 -0
  472. data/sig/ibex/generation_manifest.rbs +70 -0
  473. data/sig/ibex/generation_transaction.rbs +82 -0
  474. data/sig/ibex/generation_transaction_recovery.rbs +36 -0
  475. data/sig/ibex/generation_transaction_validation.rbs +65 -0
  476. data/sig/ibex/grammar_tests.rbs +93 -0
  477. data/sig/ibex/ir/automaton_ir.rbs +35 -10
  478. data/sig/ibex/ir/grammar_ir.rbs +96 -22
  479. data/sig/ibex/ir/lexer_ir.rbs +57 -0
  480. data/sig/ibex/ir/parser_contract.rbs +69 -0
  481. data/sig/ibex/ir/serialize.rbs +75 -23
  482. data/sig/ibex/ir/validator/automaton.rbs +112 -0
  483. data/sig/ibex/ir/validator/base.rbs +69 -0
  484. data/sig/ibex/ir/validator/grammar.rbs +195 -0
  485. data/sig/ibex/ir/validator/lexer.rbs +37 -0
  486. data/sig/ibex/ir/validator.rbs +16 -0
  487. data/sig/ibex/ir.rbs +45 -5
  488. data/sig/ibex/lalr/build_metrics.rbs +60 -0
  489. data/sig/ibex/lalr/builder.rbs +150 -19
  490. data/sig/ibex/lalr/conflict_search.rbs +12 -5
  491. data/sig/ibex/lalr/counterexample.rbs +20 -14
  492. data/sig/ibex/lalr/direct_lookaheads.rbs +93 -0
  493. data/sig/ibex/lalr/goto_follows.rbs +89 -0
  494. data/sig/ibex/lalr/ielr/annotator.rbs +67 -0
  495. data/sig/ibex/lalr/ielr/bits.rbs +14 -0
  496. data/sig/ibex/lalr/ielr/inadequacy.rbs +24 -0
  497. data/sig/ibex/lalr/ielr/item_lookaheads.rbs +35 -0
  498. data/sig/ibex/lalr/ielr/pipeline.rbs +17 -0
  499. data/sig/ibex/lalr/ielr/split_stability.rbs +27 -0
  500. data/sig/ibex/lalr/ielr/split_state.rbs +14 -0
  501. data/sig/ibex/lalr/ielr/state_splitter.rbs +76 -0
  502. data/sig/ibex/lalr/ielr_partition.rbs +71 -0
  503. data/sig/ibex/lalr/inadequacy_report.rbs +21 -0
  504. data/sig/ibex/lalr/lookahead_propagation.rbs +38 -0
  505. data/sig/ibex/lalr/lr0_collection.rbs +50 -0
  506. data/sig/ibex/lalr/on_error_reductions.rbs +22 -0
  507. data/sig/ibex/lalr/unreachable_states.rbs +25 -0
  508. data/sig/ibex/lalr.rbs +17 -1
  509. data/sig/ibex/location.rbs +67 -0
  510. data/sig/ibex/lsp/document_handlers.rbs +24 -0
  511. data/sig/ibex/lsp/document_store.rbs +94 -0
  512. data/sig/ibex/lsp/document_store_diagnostics.rbs +20 -0
  513. data/sig/ibex/lsp/document_store_validation.rbs +26 -0
  514. data/sig/ibex/lsp/initialization_handlers.rbs +30 -0
  515. data/sig/ibex/lsp/navigation_handlers.rbs +33 -0
  516. data/sig/ibex/lsp/parser_configuration_assistance.rbs +49 -0
  517. data/sig/ibex/lsp/position_codec.rbs +44 -0
  518. data/sig/ibex/lsp/protocol_error.rbs +32 -0
  519. data/sig/ibex/lsp/request_handlers.rbs +26 -0
  520. data/sig/ibex/lsp/request_support.rbs +40 -0
  521. data/sig/ibex/lsp/server.rbs +55 -0
  522. data/sig/ibex/lsp/symbol_index.rbs +77 -0
  523. data/sig/ibex/lsp/symbol_index_builder.rbs +88 -0
  524. data/sig/ibex/lsp/symbol_index_precedence_references.rbs +18 -0
  525. data/sig/ibex/lsp/symbol_index_source_queries.rbs +26 -0
  526. data/sig/ibex/lsp/symbol_occurrence.rbs +31 -0
  527. data/sig/ibex/lsp/transport.rbs +47 -0
  528. data/sig/ibex/lsp/workspace.rbs +42 -0
  529. data/sig/ibex/lsp/workspace_analyzer.rbs +69 -0
  530. data/sig/ibex/lsp.rbs +7 -0
  531. data/sig/ibex/messages.rbs +26 -0
  532. data/sig/ibex/metrics.rbs +50 -0
  533. data/sig/ibex/normalize/declarations.rbs +31 -0
  534. data/sig/ibex/normalize/diagnostics.rbs +13 -4
  535. data/sig/ibex/normalize/expander.rbs +22 -16
  536. data/sig/ibex/normalize/expression.rbs +11 -9
  537. data/sig/ibex/normalize/grammar_builder.rbs +11 -0
  538. data/sig/ibex/normalize/inline_expansion.rbs +129 -0
  539. data/sig/ibex/normalize/inline_validation.rbs +40 -0
  540. data/sig/ibex/normalize/lexer.rbs +37 -0
  541. data/sig/ibex/normalize/named_references.rbs +20 -0
  542. data/sig/ibex/normalize/nodes.rbs +14 -0
  543. data/sig/ibex/normalize/parameter_ebnf_lowering.rbs +33 -0
  544. data/sig/ibex/normalize/parameter_lowering.rbs +35 -0
  545. data/sig/ibex/normalize/parameter_substitution.rbs +42 -0
  546. data/sig/ibex/normalize/parameter_validation.rbs +44 -0
  547. data/sig/ibex/normalize/parameters.rbs +51 -0
  548. data/sig/ibex/normalize/parser_configuration.rbs +23 -0
  549. data/sig/ibex/normalize/recovery_declarations.rbs +24 -0
  550. data/sig/ibex/normalize.rbs +129 -18
  551. data/sig/ibex/racc_migration/checker.rbs +36 -0
  552. data/sig/ibex/racc_migration/harness.rbs +16 -0
  553. data/sig/ibex/racc_migration/report.rbs +55 -0
  554. data/sig/ibex/racc_migration.rbs +8 -0
  555. data/sig/ibex/rake_task.rbs +51 -0
  556. data/sig/ibex/samples.rbs +60 -0
  557. data/sig/ibex/table_artifact/builder.rbs +93 -0
  558. data/sig/ibex/table_artifact/cst_projection.rbs +20 -0
  559. data/sig/ibex/table_artifact/document.rbs +27 -0
  560. data/sig/ibex/table_artifact/executor.rbs +66 -0
  561. data/sig/ibex/table_artifact/serializer.rbs +25 -0
  562. data/sig/ibex/table_artifact/validator/metadata.rbs +52 -0
  563. data/sig/ibex/table_artifact/validator/support.rbs +46 -0
  564. data/sig/ibex/table_artifact/validator/tables.rbs +63 -0
  565. data/sig/ibex/table_artifact/validator.rbs +66 -0
  566. data/sig/ibex/table_artifact.rbs +32 -0
  567. data/sig/ibex/table_simulation/result.rbs +36 -0
  568. data/sig/ibex/table_simulation/simulator.rbs +98 -0
  569. data/sig/ibex/table_simulation/step.rbs +42 -0
  570. data/sig/ibex/table_simulation/text.rbs +14 -0
  571. data/sig/ibex/table_simulation.rbs +7 -0
  572. data/sig/ibex/tables.rbs +10 -35
  573. data/sig/ibex/verifiable_generation_bundle.rbs +24 -0
  574. data/sig/ibex/verification_report/builder.rbs +53 -0
  575. data/sig/ibex/verification_report/canonical_ir.rbs +41 -0
  576. data/sig/ibex/verification_report/logical_path.rbs +34 -0
  577. data/sig/ibex/verification_report/validator.rbs +114 -0
  578. data/sig/ibex/verification_report.rbs +44 -0
  579. data/sig/ibex/verify/action_correspondence.rbs +50 -0
  580. data/sig/ibex/verify/language_witness.rbs +112 -0
  581. data/sig/ibex/verify/reference_collection.rbs +63 -0
  582. data/sig/ibex/verify/result.rbs +52 -0
  583. data/sig/ibex/verify/verifier.rbs +170 -0
  584. data/sig/ibex/verify.rbs +6 -0
  585. data/sig/ibex/watch/runner.rbs +58 -0
  586. data/sig/ibex/watch/source_snapshot.rbs +46 -0
  587. data/sig/ibex/watch.rbs +7 -0
  588. data/sig/ibex.rbs +2 -0
  589. metadata +524 -19
  590. data/.rubocop.yml +0 -43
  591. data/CHANGELOG.md +0 -30
  592. data/Rakefile +0 -25
  593. data/Steepfile +0 -10
  594. data/docs/compat-notes.md +0 -37
  595. data/docs/lexer-coverage.md +0 -14
  596. data/docs/phase10-extensions.md +0 -27
  597. data/gemfiles/Gemfile +0 -7
  598. data/gemfiles/Gemfile.lock +0 -98
  599. data/lib/ibex/frontend/grammar.y +0 -156
  600. data/lib/ibex/runtime/parser.rb +0 -360
  601. data/lib/ibex/runtime.rb +0 -8
  602. data/sig/ibex/runtime/parser.rbs +0 -167
  603. data/sig/ibex/runtime.rbs +0 -6
@@ -0,0 +1,229 @@
1
+ # frozen_string_literal: true
2
+ # rbs_inline: enabled
3
+
4
+ # steep:ignore:start
5
+
6
+ # Dependency construction mirrors the three formal relation definitions.
7
+ # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
8
+
9
+ require_relative "lookahead_propagation"
10
+
11
+ module Ibex
12
+ module LALR
13
+ # Computes DeRemer–Pennello goto-follow sets over an LR(0) collection.
14
+ # The three dependency closures are kept separately because IELR relies on
15
+ # the distinction between stable (successor/internal) and predecessor
16
+ # dependencies when a state is split.
17
+ class GotoFollows
18
+ # @rbs @grammar: IR::Grammar
19
+ # @rbs @sets: Analysis::Sets
20
+ # @rbs @states: Array[core_set]
21
+ # @rbs @transitions: transitions
22
+ # @rbs @kernel_cores: Array[Array[item_core]]
23
+
24
+ attr_reader :from_state #: Array[Integer]
25
+ attr_reader :to_state #: Array[Integer]
26
+ attr_reader :goto_index #: Hash[[Integer, Integer], Integer]
27
+ attr_reader :direct_reads #: Array[Integer]
28
+ attr_reader :successor_edges #: Array[Array[Integer]]
29
+ attr_reader :internal_edges #: Array[Array[Integer]]
30
+ attr_reader :includes_edges #: Array[Array[Integer]]
31
+ attr_reader :predecessors #: Array[Array[Integer]]
32
+ attr_reader :successor_follows #: Array[Integer]
33
+ attr_reader :always_follows #: Array[Integer]
34
+ attr_reader :goto_follows #: Array[Integer]
35
+ attr_reader :follow_kernel_items #: Array[Integer]
36
+ attr_reader :kernel_cores #: Array[Array[item_core]]
37
+
38
+ # @rbs (IR::Grammar grammar, Analysis::Sets sets, Array[core_set] states, transitions transitions,
39
+ # Array[Integer] start_states, ?start_names: Array[String]?) -> void
40
+ def initialize(grammar, sets, states, transitions, start_states, start_names: nil)
41
+ @grammar = grammar
42
+ @sets = sets
43
+ @states = states
44
+ @transitions = transitions
45
+ @start_states = start_states
46
+ @start_names = start_names || grammar.starts
47
+ @kernel_cores = states.map { |items| kernel_items(items) }
48
+ build_gotos
49
+ build_dependencies
50
+ build_follow_sets
51
+ @reduction_lookaheads = nil
52
+ end
53
+
54
+ # @rbs (Integer state_id, Integer nonterminal_id) -> Integer?
55
+ def goto_for(state_id, nonterminal_id)
56
+ @goto_index[[state_id, nonterminal_id]]
57
+ end
58
+
59
+ # @rbs () -> Array[Hash[Integer, Integer]]
60
+ def reduction_lookaheads
61
+ return @reduction_lookaheads if @reduction_lookaheads
62
+
63
+ result = Array.new(@states.length) { {} }
64
+ seeds = @start_states.each_with_index.map do |state_id, index|
65
+ augmented = -1 - @grammar.starts.index(@start_names.fetch(index))
66
+ [state_id, [augmented, 0], eof_id]
67
+ end
68
+ propagated = LookaheadPropagation.new(
69
+ @grammar, @sets, @states, @transitions, seeds: seeds
70
+ ).build
71
+ @states.each_with_index do |items, state_id|
72
+ items.each do |production_id, dot|
73
+ next unless dot == rhs_for(production_id).length
74
+
75
+ values = propagated.fetch(state_id).fetch([production_id, dot], [])
76
+ result.fetch(state_id)[production_id] = values.reduce(0) do |bits, token|
77
+ bits | (1 << token)
78
+ end
79
+ end
80
+ end
81
+ @reduction_lookaheads = result
82
+ end
83
+
84
+ private
85
+
86
+ # @rbs () -> void
87
+ def build_gotos
88
+ @from_state = []
89
+ @to_state = []
90
+ @goto_symbols = []
91
+ @goto_index = {}
92
+ @transitions.each_with_index do |edges, state_id|
93
+ edges.keys.sort.each do |symbol_id|
94
+ symbol = @grammar.symbol_by_id(symbol_id)
95
+ next unless symbol&.nonterminal?
96
+
97
+ goto_id = @from_state.length
98
+ @from_state << state_id
99
+ @to_state << edges.fetch(symbol_id)
100
+ @goto_symbols << symbol_id
101
+ @goto_index[[state_id, symbol_id]] = goto_id
102
+ end
103
+ end
104
+ end
105
+
106
+ # @rbs () -> void
107
+ def build_dependencies
108
+ count = @from_state.length
109
+ @direct_reads = Array.new(count, 0)
110
+ @successor_edges = Array.new(count) { [] }
111
+ @internal_edges = Array.new(count) { [] }
112
+ @includes_edges = Array.new(count) { [] }
113
+ @predecessors = Array.new(@states.length) { [] }
114
+ @transitions.each_with_index do |edges, state_id|
115
+ edges.each_value { |target| @predecessors.fetch(target) << state_id }
116
+ end
117
+
118
+ count.times do |goto_id|
119
+ destination = @to_state.fetch(goto_id)
120
+ @transitions.fetch(destination).each_key do |symbol_id|
121
+ symbol = @grammar.symbol_by_id(symbol_id)
122
+ if symbol&.terminal?
123
+ @direct_reads[goto_id] |= 1 << symbol_id
124
+ elsif symbol&.nonterminal? && @sets.nullable?(symbol_id)
125
+ successor = @goto_index.fetch([destination, symbol_id])
126
+ @successor_edges.fetch(goto_id) << successor
127
+ end
128
+ end
129
+ end
130
+
131
+ @start_states.each_with_index do |state_id, index|
132
+ start_name = @start_names.fetch(index)
133
+ start_symbol = @grammar.symbol(start_name) || raise(Ibex::Error, "missing start symbol #{start_name}")
134
+ goto_id = @goto_index[[state_id, start_symbol.id]]
135
+ @direct_reads[goto_id] |= 1 << eof_id if goto_id
136
+ end
137
+
138
+ @goto_index.each do |(inner_source, inner_symbol), inner_goto_id|
139
+ @grammar.productions.each do |production|
140
+ production.rhs.each_with_index do |symbol_id, position|
141
+ next unless symbol_id == inner_symbol
142
+ next unless @grammar.symbol_by_id(symbol_id)&.nonterminal?
143
+
144
+ suffix = production.rhs[(position + 1)..] || []
145
+ next unless @sets.sequence_nullable?(suffix)
146
+
147
+ @goto_index.each do |(outer_source, outer_symbol), outer_goto_id|
148
+ next unless outer_symbol == production.lhs
149
+
150
+ reached = advance(outer_source, production.rhs.take(position))
151
+ next unless reached == inner_source
152
+
153
+ @includes_edges.fetch(inner_goto_id) << outer_goto_id
154
+ @internal_edges.fetch(inner_goto_id) << outer_goto_id if position.zero?
155
+ end
156
+ end
157
+ end
158
+ end
159
+ @successor_edges.each(&:uniq!)
160
+ @internal_edges.each(&:uniq!)
161
+ @includes_edges.each(&:uniq!)
162
+ @predecessors.each(&:uniq!)
163
+ end
164
+
165
+ # @rbs () -> void
166
+ def build_follow_sets
167
+ @successor_follows = Analysis::Digraph.closure(@direct_reads, @successor_edges)
168
+ internal_initial = @from_state.each_index.map do |goto_id|
169
+ 1 << symbol_for_goto(goto_id)
170
+ end
171
+ @internal_reachable_symbols = Analysis::Digraph.closure(internal_initial, @internal_edges)
172
+ @always_follows = Analysis::Digraph.closure(@direct_reads, merged_edges(@successor_edges, @internal_edges))
173
+ @goto_follows = Analysis::Digraph.closure(@always_follows, @includes_edges)
174
+ @follow_kernel_items = @from_state.each_index.map { |goto_id| compute_follow_kernel_items(goto_id) }
175
+ end
176
+
177
+ # @rbs (Array[Array[Integer]], Array[Array[Integer]]) -> Array[Array[Integer]]
178
+ def merged_edges(left, right)
179
+ left.each_index.map { |index| (left.fetch(index) + right.fetch(index)).uniq }
180
+ end
181
+
182
+ # @rbs (Integer goto_id) -> Integer
183
+ def compute_follow_kernel_items(goto_id)
184
+ state_id = @from_state.fetch(goto_id)
185
+ reachable = @internal_reachable_symbols.fetch(goto_id)
186
+ @kernel_cores.fetch(state_id).each_with_index.reduce(0) do |mask, ((production_id, dot), index)|
187
+ rhs = rhs_for(production_id)
188
+ symbol_id = rhs[dot]
189
+ next mask unless symbol_id && reachable.anybits?(1 << symbol_id)
190
+ next mask unless @sets.sequence_nullable?(rhs[(dot + 1)..] || [])
191
+
192
+ mask | (1 << index)
193
+ end
194
+ end
195
+
196
+ # @rbs (core_set items) -> Array[item_core]
197
+ def kernel_items(items)
198
+ items.select { |production_id, dot| production_id.negative? || dot.positive? }.to_a.sort
199
+ end
200
+
201
+ # @rbs (Integer goto_id) -> Integer
202
+ def symbol_for_goto(goto_id)
203
+ @goto_symbols.fetch(goto_id)
204
+ end
205
+
206
+ # @rbs (Integer state_id, Array[Integer]) -> Integer?
207
+ def advance(state_id, symbols)
208
+ symbols.reduce(state_id) do |current, symbol_id|
209
+ @transitions.fetch(current).fetch(symbol_id, nil) || break
210
+ end
211
+ end
212
+
213
+ # @rbs (Integer production_id) -> Array[Integer]
214
+ def rhs_for(production_id)
215
+ return [@grammar.symbol(@grammar.starts.fetch(-production_id - 1)).id] if production_id.negative?
216
+
217
+ @grammar.productions.fetch(production_id).rhs
218
+ end
219
+
220
+ # @rbs () -> Integer
221
+ def eof_id
222
+ 0
223
+ end
224
+ end
225
+ end
226
+ end
227
+
228
+ # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
229
+ # steep:ignore:end
@@ -0,0 +1,214 @@
1
+ # frozen_string_literal: true
2
+ # rbs_inline: enabled
3
+
4
+ # steep:ignore:start
5
+
6
+ require "set"
7
+ require_relative "bits"
8
+ require_relative "inadequacy"
9
+ require_relative "item_lookaheads"
10
+ require_relative "split_stability"
11
+
12
+ # The methods follow the paper's matrix definitions directly.
13
+ # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
14
+
15
+ module Ibex
16
+ module LALR
17
+ module IELR
18
+ # Finds grammar-relative inadequacies and traces their possible
19
+ # manifestations through predecessor states.
20
+ class Annotator
21
+ attr_reader :annotation_lists #: Array[Array[Annotation]]
22
+ attr_reader :inadequacies #: Array[Array[Inadequacy]]
23
+ attr_reader :item_lookaheads #: ItemLookaheads
24
+ attr_reader :split_stable_discarded #: Integer
25
+
26
+ # @rbs (IR::Grammar grammar, Array[core_set] states, transitions transitions,
27
+ # Array[packed_items] items, GotoFollows goto_follows, ?resolver: ConflictResolver?) -> void
28
+ def initialize(grammar, states, transitions, items, goto_follows, resolver: nil)
29
+ @grammar = grammar
30
+ @states = states
31
+ @transitions = transitions
32
+ @items = items
33
+ @goto_follows = goto_follows
34
+ @resolver = resolver || ConflictResolver.new(grammar)
35
+ @kernel_cores = states.map { |state| kernel_items(state) }
36
+ @item_lookaheads = ItemLookaheads.new(
37
+ grammar, goto_follows, @kernel_cores, goto_follows.predecessors,
38
+ start_states: (0...grammar.starts.length).to_a
39
+ )
40
+ @annotation_lists = Array.new(states.length) { [] }
41
+ @keys = Array.new(states.length) { Set.new }
42
+ @inadequacies = Array.new(states.length) { [] }
43
+ @next_id = 0
44
+ @split_stable_discarded = 0
45
+ end
46
+
47
+ # @rbs () -> Array[Array[Annotation]]
48
+ def build
49
+ @states.each_index do |state_id|
50
+ @inadequacies[state_id] = build_inadequacies(state_id)
51
+ @inadequacies.fetch(state_id).each do |inadequacy|
52
+ register?(state_id, annotate_manifestation(state_id, inadequacy))
53
+ end
54
+ end
55
+ worklist = @annotation_lists.each_with_index.flat_map do |annotations, state_id|
56
+ annotations.map { |annotation| [state_id, annotation] }
57
+ end
58
+ cursor = 0
59
+ while cursor < worklist.length
60
+ state_id, annotation = worklist.fetch(cursor)
61
+ cursor += 1
62
+ @goto_follows.predecessors.fetch(state_id).each do |previous|
63
+ derived = annotate_predecessor(previous, state_id, annotation)
64
+ worklist << [previous, derived] if register?(previous, derived)
65
+ end
66
+ end
67
+ @annotation_lists
68
+ end
69
+
70
+ private
71
+
72
+ # @rbs (Integer state_id) -> Array[Inadequacy]
73
+ def build_inadequacies(state_id)
74
+ @grammar.terminals.filter_map do |terminal|
75
+ contributions = contributions_for(state_id, terminal.id)
76
+ next if contributions.length < 2
77
+
78
+ inadequacy = Inadequacy.new(state: state_id, token: terminal.id, contributions: contributions, id: @next_id)
79
+ @next_id += 1
80
+ inadequacy
81
+ end
82
+ end
83
+
84
+ # @rbs (Integer state_id, Integer token_id) -> Array[[Symbol, Integer?]]
85
+ def contributions_for(state_id, token_id)
86
+ reductions = @items.fetch(state_id).filter_map do |(production_id, dot), lookaheads|
87
+ next unless dot == rhs_for(production_id).length && lookaheads.include?(token_id)
88
+
89
+ production_id
90
+ end.sort
91
+ contributions = reductions.select(&:negative?).map { |production_id| [:accept, production_id] }
92
+ symbol = @grammar.symbol_by_id(token_id)
93
+ contributions << [:shift, nil] if symbol&.terminal? && @transitions.fetch(state_id).key?(token_id)
94
+ contributions.concat(reductions.reject(&:negative?).map { |production_id| [:reduce, production_id] })
95
+ contributions
96
+ end
97
+
98
+ # @rbs (Integer state_id, Inadequacy inadequacy) -> Annotation
99
+ def annotate_manifestation(state_id, inadequacy)
100
+ matrix = inadequacy.contributions.map do |kind, production_id|
101
+ next nil if kind == :shift
102
+
103
+ rhs = rhs_for(production_id)
104
+ if rhs.empty?
105
+ compute_lhs_contributions(state_id, lhs_for(production_id), inadequacy.token)
106
+ else
107
+ completed_kernel_mask(state_id, production_id, rhs.length)
108
+ end
109
+ end
110
+ Annotation.new(inadequacy: inadequacy, matrix: matrix)
111
+ end
112
+
113
+ # @rbs (Integer state_id, Integer successor_id, Annotation annotation) -> Annotation
114
+ def annotate_predecessor(state_id, successor_id, annotation)
115
+ token = annotation.inadequacy.token
116
+ cores = @kernel_cores.fetch(successor_id)
117
+ matrix = annotation.matrix.map do |row|
118
+ next nil if row.nil?
119
+ next nil if always_via_lhs?(state_id, cores, row, token)
120
+
121
+ project_row(state_id, cores, row, token)
122
+ end
123
+ Annotation.new(inadequacy: annotation.inadequacy, matrix: matrix)
124
+ end
125
+
126
+ # @rbs (Integer state_id, Array[item_core], Integer, Integer) -> bool
127
+ def always_via_lhs?(state_id, cores, row, token)
128
+ Bits.each_set_bit(row).any? do |index|
129
+ production_id, dot = cores.fetch(index)
130
+ next false unless dot == 1
131
+
132
+ compute_lhs_contributions(state_id, lhs_for(production_id), token).nil?
133
+ end
134
+ end
135
+
136
+ # @rbs (Integer state_id, Array[item_core], Integer, Integer) -> Integer
137
+ def project_row(state_id, cores, row, token)
138
+ Bits.each_set_bit(row).reduce(0) do |mask, index|
139
+ production_id, dot = cores.fetch(index)
140
+ if dot > 1
141
+ previous = kernel_index_of(state_id, production_id, dot - 1)
142
+ next mask unless previous && item_lookaheads.fetch(state_id, previous).anybits?(1 << token)
143
+
144
+ mask | (1 << previous)
145
+ else
146
+ lhs_mask = compute_lhs_contributions(state_id, lhs_for(production_id), token)
147
+ mask | lhs_mask.to_i
148
+ end
149
+ end
150
+ end
151
+
152
+ # @rbs (Integer state_id, Integer production_id, Integer length) -> Integer
153
+ def completed_kernel_mask(state_id, production_id, length)
154
+ index = kernel_index_of(state_id, production_id, length)
155
+ index ? (1 << index) : 0
156
+ end
157
+
158
+ # @rbs (Integer state_id, Integer lhs_id, Integer token_id) -> Integer?
159
+ def compute_lhs_contributions(state_id, lhs_id, token_id)
160
+ goto_id = @goto_follows.goto_for(state_id, lhs_id)
161
+ raise Ibex::Error, "missing goto for #{lhs_id} from #{state_id}" unless goto_id
162
+ return nil if @goto_follows.always_follows.fetch(goto_id).anybits?(1 << token_id)
163
+
164
+ mask = 0
165
+ Bits.each_set_bit(@goto_follows.follow_kernel_items.fetch(goto_id)) do |index|
166
+ mask |= 1 << index if @item_lookaheads.fetch(state_id, index).anybits?(1 << token_id)
167
+ end
168
+ mask
169
+ end
170
+
171
+ # @rbs (Integer state_id, Annotation annotation) -> bool
172
+ def register?(state_id, annotation)
173
+ if SplitStability.split_stable?(annotation, @resolver)
174
+ @split_stable_discarded += 1
175
+ return false
176
+ end
177
+
178
+ key = annotation.key
179
+ return false unless @keys.fetch(state_id).add?(key)
180
+
181
+ @annotation_lists.fetch(state_id) << annotation
182
+ true
183
+ end
184
+
185
+ # @rbs (Integer state_id, Integer production_id) -> Integer
186
+ def lhs_for(production_id)
187
+ return @grammar.symbol(@grammar.starts.fetch(-production_id - 1)).id if production_id.negative?
188
+
189
+ @grammar.productions.fetch(production_id).lhs
190
+ end
191
+
192
+ # @rbs (Integer production_id) -> Array[Integer]
193
+ def rhs_for(production_id)
194
+ return [lhs_for(production_id)] if production_id.negative?
195
+
196
+ @grammar.productions.fetch(production_id).rhs
197
+ end
198
+
199
+ # @rbs (core_set items) -> Array[item_core]
200
+ def kernel_items(items)
201
+ items.select { |production_id, dot| production_id.negative? || dot.positive? }.to_a.sort
202
+ end
203
+
204
+ # @rbs (Integer state_id, Integer production_id, Integer dot) -> Integer?
205
+ def kernel_index_of(state_id, production_id, dot)
206
+ @kernel_cores.fetch(state_id).index([production_id, dot])
207
+ end
208
+ end
209
+ end
210
+ end
211
+ end
212
+ # steep:ignore:end
213
+
214
+ # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+ # rbs_inline: enabled
3
+
4
+ # steep:ignore:start
5
+
6
+ module Ibex
7
+ module LALR
8
+ module IELR
9
+ module Bits
10
+ module_function
11
+
12
+ # @rbs (Integer bits) { (Integer) -> void } -> void
13
+ # @rbs (Integer bits) -> Enumerator[Integer, void]
14
+ def each_set_bit(bits)
15
+ return enum_for(:each_set_bit, bits) unless block_given?
16
+
17
+ index = 0
18
+ while bits.positive?
19
+ yield index if bits.odd?
20
+ bits >>= 1
21
+ index += 1
22
+ end
23
+ end
24
+ end
25
+ end
26
+ end
27
+ end
28
+ # steep:ignore:end
@@ -0,0 +1,47 @@
1
+ # frozen_string_literal: true
2
+ # rbs_inline: enabled
3
+
4
+ # steep:ignore:start
5
+
6
+ module Ibex
7
+ module LALR
8
+ module IELR
9
+ # @rbs skip
10
+ Inadequacy = Struct.new(:state, :token, :contributions, :id, keyword_init: true) do
11
+ def initialize(state:, token:, contributions:, id:)
12
+ super(state: state, token: token, contributions: contributions.freeze, id: id)
13
+ freeze
14
+ end
15
+ end
16
+
17
+ # @rbs skip
18
+ Annotation = Struct.new(:inadequacy, :matrix, keyword_init: true) do
19
+ def initialize(inadequacy:, matrix:)
20
+ super(inadequacy: inadequacy, matrix: matrix.freeze)
21
+ freeze
22
+ end
23
+
24
+ def key
25
+ [inadequacy.id, matrix]
26
+ end
27
+ end
28
+
29
+ # @rbs!
30
+ # type contribution = [Symbol, Integer?]
31
+ # class Inadequacy < Struct[Integer | Array[contribution]]
32
+ # attr_accessor state: Integer
33
+ # attr_accessor token: Integer
34
+ # attr_accessor contributions: Array[contribution]
35
+ # attr_accessor id: Integer
36
+ # def self.new: (state: Integer, token: Integer, contributions: Array[contribution], id: Integer) -> instance
37
+ # end
38
+ # class Annotation < Struct[Inadequacy | Array[Integer?]]
39
+ # attr_accessor inadequacy: Inadequacy
40
+ # attr_accessor matrix: Array[Integer?]
41
+ # def self.new: (inadequacy: Inadequacy, matrix: Array[Integer?]) -> instance
42
+ # def key: () -> [Integer, Array[Integer?]]
43
+ # end
44
+ end
45
+ end
46
+ end
47
+ # steep:ignore:end
@@ -0,0 +1,78 @@
1
+ # frozen_string_literal: true
2
+ # rbs_inline: enabled
3
+
4
+ # steep:ignore:start
5
+
6
+ require_relative "bits"
7
+
8
+ module Ibex
9
+ module LALR
10
+ module IELR
11
+ # Lazily derives the lookahead of a kernel item from predecessor states
12
+ # and goto-follow sets. Memoisation is important: predecessor paths can
13
+ # share exponentially many suffixes in a real grammar.
14
+ class ItemLookaheads
15
+ # @rbs @memo: Hash[Integer, Integer]
16
+
17
+ # @rbs (IR::Grammar grammar, GotoFollows goto_follows, Array[Array[item_core]] kernel_cores,
18
+ # Array[Array[Integer]] predecessors, ?start_states: Array[Integer]?) -> void
19
+ def initialize(grammar, goto_follows, kernel_cores, predecessors, start_states: nil)
20
+ @grammar = grammar
21
+ @goto_follows = goto_follows
22
+ @kernel_cores = kernel_cores
23
+ @predecessors = predecessors
24
+ @start_states = start_states || []
25
+ @index = {}
26
+ kernel_cores.each_with_index do |cores, state_id|
27
+ cores.each_with_index { |core, kernel_index| @index[[state_id, core]] = kernel_index }
28
+ end
29
+ @memo = {}
30
+ end
31
+
32
+ # @rbs (Integer state_id, Integer kernel_index) -> Integer
33
+ def fetch(state_id, kernel_index)
34
+ key = [state_id, kernel_index]
35
+ return @memo.fetch(key) if @memo.key?(key)
36
+
37
+ @memo[key] = compute(state_id, kernel_index)
38
+ end
39
+
40
+ # @rbs () -> Integer
41
+ def size
42
+ @memo.length
43
+ end
44
+
45
+ private
46
+
47
+ # @rbs (Integer state_id, Integer kernel_index) -> Integer
48
+ def compute(state_id, kernel_index)
49
+ production_id, dot = @kernel_cores.fetch(state_id).fetch(kernel_index)
50
+ return start_seed(production_id) if production_id.negative? && dot.zero?
51
+ return start_seed(production_id) if production_id.negative? && dot.positive?
52
+ return 0 if dot.zero?
53
+ return goto_follows_for(state_id, production_id) if dot == 1
54
+
55
+ @predecessors.fetch(state_id).reduce(0) do |bits, previous|
56
+ previous_index = @index[[previous, [production_id, dot - 1]]]
57
+ previous_index ? bits | fetch(previous, previous_index) : bits
58
+ end
59
+ end
60
+
61
+ # @rbs (Integer state_id, Integer production_id) -> Integer
62
+ def goto_follows_for(state_id, production_id)
63
+ lhs = @grammar.productions.fetch(production_id).lhs
64
+ @predecessors.fetch(state_id).reduce(0) do |bits, previous|
65
+ goto_id = @goto_follows.goto_for(previous, lhs)
66
+ bits | (goto_id ? @goto_follows.goto_follows.fetch(goto_id) : 0)
67
+ end
68
+ end
69
+
70
+ # @rbs (Integer production_id) -> Integer
71
+ def start_seed(production_id)
72
+ production_id.negative? ? 1 : 0
73
+ end
74
+ end
75
+ end
76
+ end
77
+ end
78
+ # steep:ignore:end
@@ -0,0 +1,75 @@
1
+ # frozen_string_literal: true
2
+ # rbs_inline: enabled
3
+
4
+ # steep:ignore:start
5
+
6
+ require_relative "../lr0_collection"
7
+ require_relative "../goto_follows"
8
+ require_relative "../lookahead_propagation"
9
+ require_relative "state_splitter"
10
+
11
+ # The method follows the fixed six-phase pipeline as one transaction.
12
+ # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
13
+
14
+ module Ibex
15
+ module LALR
16
+ module IELR
17
+ # Coordinates the canonical-free direct IELR phases. The returned
18
+ # profile deliberately reports canonical metrics as unavailable.
19
+ class Pipeline
20
+ # @rbs (IR::Grammar grammar, Analysis::Sets sets, ?starts: Array[String]?, ?profile: bool) -> void
21
+ def initialize(grammar, sets, starts: nil, profile: false)
22
+ @grammar = grammar
23
+ @sets = sets
24
+ @starts = starts || grammar.starts
25
+ @profile = profile
26
+ end
27
+
28
+ # @rbs () -> [Array[packed_items], transitions, LALR::build_collection]
29
+ def build
30
+ states, transitions = LR0Collection.new(@grammar, starts: @starts).build
31
+ seeds = @starts.each_with_index.map do |name, index|
32
+ augmented = -1 - @grammar.starts.index(name)
33
+ [index, [augmented, 0], 0]
34
+ end
35
+ initial_propagation = LookaheadPropagation.new(
36
+ @grammar, @sets, states, transitions, seeds: seeds
37
+ )
38
+ items = initial_propagation.build
39
+ goto_follows = GotoFollows.new(
40
+ @grammar, @sets, states, transitions, @starts.each_index.to_a, start_names: @starts
41
+ )
42
+ splitter = StateSplitter.new(@grammar, states, transitions, items, goto_follows)
43
+ _split_items, split_transitions = splitter.build
44
+ split_states = splitter.states.map { |state| states.fetch(state.lalr_isocore) }
45
+ phase4_items = LookaheadPropagation.new(
46
+ @grammar, @sets, split_states, split_transitions, seeds: seeds
47
+ ).build
48
+ profile = {
49
+ construction_states: phase4_items.length,
50
+ canonical_states: nil,
51
+ strategy: :ielr_direct,
52
+ lr0_states: @profile ? states.length : nil,
53
+ lr0_items: @profile ? states.sum(&:length) : nil,
54
+ canonical_items: nil,
55
+ propagation_edges: @profile ? initial_propagation.propagation_edge_count : nil,
56
+ ielr_initial_partitions: nil,
57
+ ielr_final_partitions: nil,
58
+ ielr_annotations: splitter.annotations.sum(&:length),
59
+ ielr_annotated_states: splitter.annotations.count { |entries| !entries.empty? },
60
+ ielr_inadequacies: splitter.inadequacies.sum(&:length),
61
+ ielr_split_stable_discarded: splitter.split_stable_discarded,
62
+ ielr_lalr_states: states.length,
63
+ ielr_split_states: splitter.split_states,
64
+ ielr_unreachable_removed: 0,
65
+ ielr_remergeable_candidates: nil
66
+ }
67
+ [phase4_items, split_transitions, profile]
68
+ end
69
+ end
70
+ end
71
+ end
72
+ end
73
+ # steep:ignore:end
74
+
75
+ # rubocop:enable Metrics/AbcSize, Metrics/MethodLength