ibex 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (603) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +89 -123
  3. data/docs/architecture.md +477 -31
  4. data/docs/bison-import.md +103 -0
  5. data/docs/comparison-policy.md +127 -0
  6. data/docs/configuration-model.md +59 -0
  7. data/docs/configuration-report.md +28 -0
  8. data/docs/conflict-explanation-reviews/v1/records/README.md +28 -0
  9. data/docs/conflict-explanation-study.md +62 -0
  10. data/docs/construction-profiling.md +157 -0
  11. data/docs/cst-migration.md +106 -0
  12. data/docs/cst.md +237 -0
  13. data/docs/decisions/0000-template.md +24 -0
  14. data/docs/decisions/0001-separate-ir-pipeline.md +34 -0
  15. data/docs/decisions/0002-opaque-user-code-boundary.md +32 -0
  16. data/docs/decisions/0003-self-hosted-grammar-frontend.md +32 -0
  17. data/docs/decisions/0004-shared-semantic-and-lossless-source-model.md +33 -0
  18. data/docs/decisions/0005-contained-grammar-composition.md +34 -0
  19. data/docs/decisions/0006-bounded-structural-grammar-lowering.md +32 -0
  20. data/docs/decisions/0007-shared-parser-construction-pipeline.md +33 -0
  21. data/docs/decisions/0008-versioned-runtime-package-boundary.md +36 -0
  22. data/docs/decisions/0009-isolated-parser-sessions.md +34 -0
  23. data/docs/decisions/0010-committed-runtime-observation.md +34 -0
  24. data/docs/decisions/0011-versioned-semantic-action-boundary.md +37 -0
  25. data/docs/decisions/0012-bounded-nonexecuting-analysis.md +37 -0
  26. data/docs/decisions/0013-transactional-generation-publication.md +37 -0
  27. data/docs/decisions/0014-versioned-generated-lexer.md +35 -0
  28. data/docs/decisions/0015-worker-isolated-browser-analysis.md +32 -0
  29. data/docs/decisions/0016-red-green-concrete-syntax.md +34 -0
  30. data/docs/decisions/0017-persistent-syntax-artifacts.md +35 -0
  31. data/docs/decisions/0018-conservative-incremental-syntax-reuse.md +38 -0
  32. data/docs/decisions/0019-runtime-syntax-session-boundary.md +51 -0
  33. data/docs/decisions/0020-grammar-ir-parser-contract.md +65 -0
  34. data/docs/decisions/0021-data-only-parser-table-sidecar.md +50 -0
  35. data/docs/decisions/0022-manifest-bound-verification-report.md +77 -0
  36. data/docs/decisions/0023-syntax-only-repair-results.md +48 -0
  37. data/docs/decisions/0024-direct-ielr-construction.md +42 -0
  38. data/docs/decisions/README.md +74 -0
  39. data/docs/declarative-configuration.md +262 -0
  40. data/docs/development.md +306 -0
  41. data/docs/direct-ielr-decision.md +120 -0
  42. data/docs/direct-multi-entry-decision.md +82 -0
  43. data/docs/editor-setup.md +26 -0
  44. data/docs/error-ux-review-rubric-v1.md +117 -0
  45. data/docs/error-ux-reviews/v1/records/README.md +22 -0
  46. data/docs/error-ux-round2-review-status-v1.json +27 -0
  47. data/docs/error-ux-round2-reviews/v1/records/README.md +30 -0
  48. data/docs/error-ux-round2-v1.json +1581 -0
  49. data/docs/error-ux-round2.md +111 -0
  50. data/docs/error-ux.md +114 -0
  51. data/docs/getting-started.md +80 -0
  52. data/docs/grammar-reference.md +629 -18
  53. data/docs/ielr-design.md +43 -0
  54. data/docs/ielr.md +92 -0
  55. data/docs/ja/bison-import.md +51 -0
  56. data/docs/ja/stable-api.md +53 -0
  57. data/docs/lexer-construction-profile.md +109 -0
  58. data/docs/lexer-migration.md +56 -0
  59. data/docs/maturity.md +128 -0
  60. data/docs/project-site-strategy.md +54 -0
  61. data/docs/racc-migration-evidence.md +45 -0
  62. data/docs/racc-migration.md +45 -6
  63. data/docs/release-readiness.md +221 -0
  64. data/docs/repair-semantics.md +123 -0
  65. data/docs/runtime-abi-evolution.md +307 -0
  66. data/docs/stability.md +234 -0
  67. data/docs/status.md +43 -0
  68. data/docs/syntax-sessions.md +183 -0
  69. data/docs/table-artifact.md +125 -0
  70. data/docs/test-interactions.md +219 -0
  71. data/docs/verification-report.md +137 -0
  72. data/docs/verifier-trust-boundary.md +199 -0
  73. data/docs/workloads.md +132 -0
  74. data/examples/README.md +60 -0
  75. data/examples/calculator.y +44 -0
  76. data/examples/csv.y +32 -0
  77. data/examples/ini.y +70 -0
  78. data/examples/json.y +48 -0
  79. data/examples/tiny_language.y +75 -0
  80. data/lib/ibex/analysis/digraph.rb +128 -0
  81. data/lib/ibex/analysis/sets.rb +5 -4
  82. data/lib/ibex/analysis.rb +1 -0
  83. data/lib/ibex/artifact_set.rb +63 -0
  84. data/lib/ibex/bison_import/importer.rb +516 -0
  85. data/lib/ibex/bison_import/tokenizer.rb +260 -0
  86. data/lib/ibex/bison_import.rb +200 -0
  87. data/lib/ibex/bounded_subprocess.rb +171 -0
  88. data/lib/ibex/cli/ambiguity.rb +75 -0
  89. data/lib/ibex/cli/analysis.rb +134 -0
  90. data/lib/ibex/cli/bison_import.rb +121 -0
  91. data/lib/ibex/cli/config.rb +128 -0
  92. data/lib/ibex/cli/counterexample_options.rb +6 -2
  93. data/lib/ibex/cli/coverage.rb +173 -0
  94. data/lib/ibex/cli/debug.rb +106 -0
  95. data/lib/ibex/cli/diagnostics.rb +136 -0
  96. data/lib/ibex/cli/documentation.rb +70 -0
  97. data/lib/ibex/cli/equiv.rb +169 -0
  98. data/lib/ibex/cli/error_messages.rb +184 -0
  99. data/lib/ibex/cli/explain.rb +85 -0
  100. data/lib/ibex/cli/fix.rb +203 -0
  101. data/lib/ibex/cli/formatting.rb +387 -0
  102. data/lib/ibex/cli/fuzz.rb +200 -0
  103. data/lib/ibex/cli/fuzz_regressions.rb +145 -0
  104. data/lib/ibex/cli/generation_artifacts.rb +189 -0
  105. data/lib/ibex/cli/generation_error_messages.rb +39 -0
  106. data/lib/ibex/cli/grammar_tests.rb +140 -0
  107. data/lib/ibex/cli/ir_tools.rb +128 -0
  108. data/lib/ibex/cli/lsp.rb +33 -0
  109. data/lib/ibex/cli/outputs.rb +133 -23
  110. data/lib/ibex/cli/racc_migration.rb +87 -0
  111. data/lib/ibex/cli/reduce.rb +202 -0
  112. data/lib/ibex/cli/reduce_reporting.rb +71 -0
  113. data/lib/ibex/cli/samples.rb +118 -0
  114. data/lib/ibex/cli/verify.rb +116 -0
  115. data/lib/ibex/cli/watch.rb +104 -0
  116. data/lib/ibex/cli.rb +825 -55
  117. data/lib/ibex/codegen/action_locations.rb +103 -0
  118. data/lib/ibex/codegen/action_method_source.rb +209 -0
  119. data/lib/ibex/codegen/action_source.rb +119 -0
  120. data/lib/ibex/codegen/ambiguity.rb +174 -0
  121. data/lib/ibex/codegen/cst_metadata.rb +171 -0
  122. data/lib/ibex/codegen/documentation.rb +170 -0
  123. data/lib/ibex/codegen/explain.rb +380 -0
  124. data/lib/ibex/codegen/generated_action_abi.rb +293 -0
  125. data/lib/ibex/codegen/html.rb +94 -10
  126. data/lib/ibex/codegen/mermaid.rb +43 -0
  127. data/lib/ibex/codegen/railroad.rb +197 -0
  128. data/lib/ibex/codegen/railroad_documentation.rb +59 -0
  129. data/lib/ibex/codegen/rbs.rb +331 -2
  130. data/lib/ibex/codegen/report.rb +67 -15
  131. data/lib/ibex/codegen/ruby.rb +305 -66
  132. data/lib/ibex/codegen/ruby_actions.rb +146 -0
  133. data/lib/ibex/codegen/ruby_ast.rb +121 -0
  134. data/lib/ibex/codegen/ruby_error_messages.rb +28 -0
  135. data/lib/ibex/codegen/ruby_lexer.rb +83 -0
  136. data/lib/ibex/codegen/ruby_syntax.rb +96 -0
  137. data/lib/ibex/codegen/ruby_table_metadata.rb +57 -0
  138. data/lib/ibex/codegen/ruby_value_printers.rb +56 -0
  139. data/lib/ibex/codegen/symbol_labels.rb +1 -1
  140. data/lib/ibex/configuration/analysis_grammar.rb +35 -0
  141. data/lib/ibex/configuration/explanation.rb +421 -0
  142. data/lib/ibex/configuration/inspector.rb +232 -0
  143. data/lib/ibex/configuration.rb +580 -0
  144. data/lib/ibex/coverage/collector.rb +198 -0
  145. data/lib/ibex/coverage/event_stream.rb +103 -0
  146. data/lib/ibex/coverage/report.rb +267 -0
  147. data/lib/ibex/coverage/runtime_event_validator.rb +169 -0
  148. data/lib/ibex/coverage.rb +13 -0
  149. data/lib/ibex/delta_reducer.rb +99 -0
  150. data/lib/ibex/diff.rb +140 -0
  151. data/lib/ibex/equiv/machine.rb +135 -0
  152. data/lib/ibex/equiv.rb +373 -0
  153. data/lib/ibex/error_messages/parser.rb +159 -0
  154. data/lib/ibex/error_messages/parser_v2.rb +198 -0
  155. data/lib/ibex/error_messages/renderer.rb +65 -0
  156. data/lib/ibex/error_messages/sentence_search.rb +196 -0
  157. data/lib/ibex/error_messages/update.rb +169 -0
  158. data/lib/ibex/error_messages.rb +165 -0
  159. data/lib/ibex/fix.rb +557 -0
  160. data/lib/ibex/frontend/ast.rb +166 -7
  161. data/lib/ibex/frontend/bootstrap_parser.rb +40 -4
  162. data/lib/ibex/frontend/diagnostic.rb +81 -0
  163. data/lib/ibex/frontend/diagnostic_recovery.rb +268 -0
  164. data/lib/ibex/frontend/dsl.rb +66 -7
  165. data/lib/ibex/frontend/formatter.rb +453 -0
  166. data/lib/ibex/frontend/generated_parser.rb +644 -190
  167. data/lib/ibex/frontend/generated_parser_base.rb +170 -32
  168. data/lib/ibex/frontend/generated_parser_includes.rb +51 -0
  169. data/lib/ibex/frontend/generated_parser_metadata.rb +61 -0
  170. data/lib/ibex/frontend/generated_parser_parameters.rb +60 -0
  171. data/lib/ibex/frontend/generation.rb +33 -0
  172. data/lib/ibex/frontend/lexer.rb +162 -10
  173. data/lib/ibex/frontend/lexer_recovery.rb +84 -0
  174. data/lib/ibex/frontend/parser/declarations.rb +262 -11
  175. data/lib/ibex/frontend/parser/parameters.rb +82 -0
  176. data/lib/ibex/frontend/parser/rules.rb +43 -6
  177. data/lib/ibex/frontend/parser.rb +182 -4
  178. data/lib/ibex/frontend/parser_configuration_support.rb +56 -0
  179. data/lib/ibex/frontend/regenerator.rb +27 -1
  180. data/lib/ibex/frontend/resolution.rb +70 -0
  181. data/lib/ibex/frontend/resolver.rb +217 -0
  182. data/lib/ibex/frontend/rule_documentation.rb +104 -0
  183. data/lib/ibex/frontend/source_cursor.rb +133 -9
  184. data/lib/ibex/frontend/source_document.rb +229 -0
  185. data/lib/ibex/frontend/source_loader.rb +150 -0
  186. data/lib/ibex/frontend/source_span.rb +89 -0
  187. data/lib/ibex/frontend/token_adapter/declaration_document_state.rb +47 -0
  188. data/lib/ibex/frontend/token_adapter/declaration_lexer_state.rb +83 -0
  189. data/lib/ibex/frontend/token_adapter/declaration_state.rb +244 -22
  190. data/lib/ibex/frontend/token_adapter/delimiter_tracker.rb +8 -2
  191. data/lib/ibex/frontend/token_adapter/rule_state.rb +60 -2
  192. data/lib/ibex/frontend/token_adapter.rb +18 -3
  193. data/lib/ibex/frontend.rb +14 -2
  194. data/lib/ibex/fuzz.rb +219 -0
  195. data/lib/ibex/generation_input.rb +57 -0
  196. data/lib/ibex/generation_manifest.rb +211 -0
  197. data/lib/ibex/generation_transaction.rb +261 -0
  198. data/lib/ibex/generation_transaction_recovery.rb +109 -0
  199. data/lib/ibex/generation_transaction_validation.rb +196 -0
  200. data/lib/ibex/grammar_tests.rb +206 -0
  201. data/lib/ibex/ir/automaton_ir.rb +86 -11
  202. data/lib/ibex/ir/grammar_ir.rb +170 -26
  203. data/lib/ibex/ir/lexer_ir.rb +76 -0
  204. data/lib/ibex/ir/parser_contract.rb +135 -0
  205. data/lib/ibex/ir/serialize.rb +268 -50
  206. data/lib/ibex/ir/validator/automaton.rb +375 -0
  207. data/lib/ibex/ir/validator/base.rb +147 -0
  208. data/lib/ibex/ir/validator/grammar.rb +634 -0
  209. data/lib/ibex/ir/validator/lexer.rb +113 -0
  210. data/lib/ibex/ir/validator.rb +63 -0
  211. data/lib/ibex/ir.rb +71 -5
  212. data/lib/ibex/lalr/build_metrics.rb +71 -0
  213. data/lib/ibex/lalr/builder.rb +514 -52
  214. data/lib/ibex/lalr/conflict.rb +1 -0
  215. data/lib/ibex/lalr/conflict_search.rb +26 -9
  216. data/lib/ibex/lalr/counterexample.rb +38 -18
  217. data/lib/ibex/lalr/direct_lookaheads.rb +289 -0
  218. data/lib/ibex/lalr/goto_follows.rb +229 -0
  219. data/lib/ibex/lalr/ielr/annotator.rb +214 -0
  220. data/lib/ibex/lalr/ielr/bits.rb +28 -0
  221. data/lib/ibex/lalr/ielr/inadequacy.rb +47 -0
  222. data/lib/ibex/lalr/ielr/item_lookaheads.rb +78 -0
  223. data/lib/ibex/lalr/ielr/pipeline.rb +75 -0
  224. data/lib/ibex/lalr/ielr/split_stability.rb +78 -0
  225. data/lib/ibex/lalr/ielr/split_state.rb +20 -0
  226. data/lib/ibex/lalr/ielr/state_splitter.rb +258 -0
  227. data/lib/ibex/lalr/ielr_partition.rb +166 -0
  228. data/lib/ibex/lalr/inadequacy_report.rb +50 -0
  229. data/lib/ibex/lalr/lookahead_propagation.rb +111 -0
  230. data/lib/ibex/lalr/lr0_collection.rb +121 -0
  231. data/lib/ibex/lalr/on_error_reductions.rb +74 -0
  232. data/lib/ibex/lalr/unreachable_states.rb +80 -0
  233. data/lib/ibex/lalr.rb +59 -1
  234. data/lib/ibex/location.rb +129 -0
  235. data/lib/ibex/lsp/document_handlers.rb +68 -0
  236. data/lib/ibex/lsp/document_store.rb +264 -0
  237. data/lib/ibex/lsp/document_store_diagnostics.rb +44 -0
  238. data/lib/ibex/lsp/document_store_validation.rb +61 -0
  239. data/lib/ibex/lsp/initialization_handlers.rb +80 -0
  240. data/lib/ibex/lsp/navigation_handlers.rb +70 -0
  241. data/lib/ibex/lsp/parser_configuration_assistance.rb +149 -0
  242. data/lib/ibex/lsp/position_codec.rb +113 -0
  243. data/lib/ibex/lsp/protocol_error.rb +33 -0
  244. data/lib/ibex/lsp/request_handlers.rb +46 -0
  245. data/lib/ibex/lsp/request_support.rb +94 -0
  246. data/lib/ibex/lsp/server.rb +148 -0
  247. data/lib/ibex/lsp/symbol_index.rb +241 -0
  248. data/lib/ibex/lsp/symbol_index_builder.rb +277 -0
  249. data/lib/ibex/lsp/symbol_index_precedence_references.rb +44 -0
  250. data/lib/ibex/lsp/symbol_index_source_queries.rb +61 -0
  251. data/lib/ibex/lsp/symbol_occurrence.rb +23 -0
  252. data/lib/ibex/lsp/transport.rb +128 -0
  253. data/lib/ibex/lsp/workspace.rb +128 -0
  254. data/lib/ibex/lsp/workspace_analyzer.rb +199 -0
  255. data/lib/ibex/lsp.rb +33 -0
  256. data/lib/ibex/messages/en.yml +28 -0
  257. data/lib/ibex/messages/ja.yml +28 -0
  258. data/lib/ibex/messages.rb +68 -0
  259. data/lib/ibex/metrics.rb +175 -0
  260. data/lib/ibex/normalize/declarations.rb +142 -7
  261. data/lib/ibex/normalize/diagnostics.rb +94 -24
  262. data/lib/ibex/normalize/expander.rb +64 -59
  263. data/lib/ibex/normalize/expression.rb +63 -40
  264. data/lib/ibex/normalize/grammar_builder.rb +26 -0
  265. data/lib/ibex/normalize/inline_expansion.rb +465 -0
  266. data/lib/ibex/normalize/inline_validation.rb +174 -0
  267. data/lib/ibex/normalize/lexer.rb +131 -0
  268. data/lib/ibex/normalize/named_references.rb +60 -0
  269. data/lib/ibex/normalize/nodes.rb +46 -0
  270. data/lib/ibex/normalize/parameter_ebnf_lowering.rb +70 -0
  271. data/lib/ibex/normalize/parameter_lowering.rb +143 -0
  272. data/lib/ibex/normalize/parameter_substitution.rb +125 -0
  273. data/lib/ibex/normalize/parameter_validation.rb +140 -0
  274. data/lib/ibex/normalize/parameters.rb +226 -0
  275. data/lib/ibex/normalize/parser_configuration.rb +70 -0
  276. data/lib/ibex/normalize/recovery_declarations.rb +84 -0
  277. data/lib/ibex/normalize.rb +172 -17
  278. data/lib/ibex/racc_migration/checker.rb +122 -0
  279. data/lib/ibex/racc_migration/harness.rb +177 -0
  280. data/lib/ibex/racc_migration/report.rb +97 -0
  281. data/lib/ibex/racc_migration.rb +12 -0
  282. data/lib/ibex/rake_task.rb +116 -0
  283. data/lib/ibex/samples.rb +216 -0
  284. data/lib/ibex/table_artifact/builder.rb +266 -0
  285. data/lib/ibex/table_artifact/cst_projection.rb +70 -0
  286. data/lib/ibex/table_artifact/document.rb +39 -0
  287. data/lib/ibex/table_artifact/executor.rb +187 -0
  288. data/lib/ibex/table_artifact/serializer.rb +57 -0
  289. data/lib/ibex/table_artifact/validator/metadata.rb +179 -0
  290. data/lib/ibex/table_artifact/validator/support.rb +86 -0
  291. data/lib/ibex/table_artifact/validator/tables.rb +238 -0
  292. data/lib/ibex/table_artifact/validator.rb +253 -0
  293. data/lib/ibex/table_artifact.rb +88 -0
  294. data/lib/ibex/table_simulation/result.rb +55 -0
  295. data/lib/ibex/table_simulation/simulator.rb +253 -0
  296. data/lib/ibex/table_simulation/step.rb +62 -0
  297. data/lib/ibex/table_simulation/text.rb +31 -0
  298. data/lib/ibex/table_simulation.rb +13 -0
  299. data/lib/ibex/tables.rb +22 -69
  300. data/lib/ibex/verifiable_generation_bundle.rb +86 -0
  301. data/lib/ibex/verification_report/builder.rb +159 -0
  302. data/lib/ibex/verification_report/canonical_ir.rb +123 -0
  303. data/lib/ibex/verification_report/logical_path.rb +68 -0
  304. data/lib/ibex/verification_report/validator.rb +329 -0
  305. data/lib/ibex/verification_report.rb +69 -0
  306. data/lib/ibex/verify/action_correspondence.rb +138 -0
  307. data/lib/ibex/verify/language_witness.rb +295 -0
  308. data/lib/ibex/verify/reference_collection.rb +190 -0
  309. data/lib/ibex/verify/result.rb +73 -0
  310. data/lib/ibex/verify/verifier.rb +598 -0
  311. data/lib/ibex/verify.rb +15 -0
  312. data/lib/ibex/version.rb +1 -1
  313. data/lib/ibex/watch/runner.rb +181 -0
  314. data/lib/ibex/watch/source_snapshot.rb +98 -0
  315. data/lib/ibex/watch.rb +11 -0
  316. data/lib/ibex.rb +35 -1
  317. data/schema/automaton-ir-definitions.schema.json +401 -0
  318. data/schema/automaton-ir.schema.json +65 -0
  319. data/schema/benchmark-v1.schema.json +212 -0
  320. data/schema/benchmark-v2.schema.json +61 -0
  321. data/schema/bison-import-v1.schema.json +106 -0
  322. data/schema/conflict-explanation-review-v1.schema.json +196 -0
  323. data/schema/conflict-explanation-study-v1.schema.json +528 -0
  324. data/schema/construction-profile-v1.schema.json +779 -0
  325. data/schema/cst-v1.json +170 -0
  326. data/schema/diff-v1.schema.json +41 -0
  327. data/schema/direct-ielr-decision-v1.schema.json +200 -0
  328. data/schema/equiv-v1.schema.json +64 -0
  329. data/schema/error-ux-review-v1.schema.json +609 -0
  330. data/schema/error-ux-round2-review-v1.schema.json +191 -0
  331. data/schema/error-ux-round2-v1.schema.json +519 -0
  332. data/schema/error-ux-v1.schema.json +258 -0
  333. data/schema/explain-v1.schema.json +495 -0
  334. data/schema/fix-v1.schema.json +64 -0
  335. data/schema/fix-v2.schema.json +66 -0
  336. data/schema/fix-v3.schema.json +101 -0
  337. data/schema/frontend-diagnostics-v1.schema.json +154 -0
  338. data/schema/fuzz-regression-v1.schema.json +79 -0
  339. data/schema/fuzz-v1.schema.json +114 -0
  340. data/schema/generation-manifest-v1.schema.json +115 -0
  341. data/schema/grammar-ir-extensions.schema.json +734 -0
  342. data/schema/grammar-ir-foundation.schema.json +426 -0
  343. data/schema/grammar-ir.schema.json +271 -0
  344. data/schema/ielr-benchmark-v1.schema.json +85 -0
  345. data/schema/lexer-ir-v1.schema.json +215 -0
  346. data/schema/lexer-profile-v1.schema.json +491 -0
  347. data/schema/metrics-v1.schema.json +60 -0
  348. data/schema/migration-check-v1.schema.json +60 -0
  349. data/schema/performance-comparison-v1.schema.json +395 -0
  350. data/schema/public-performance-comparison-v1.schema.json +506 -0
  351. data/schema/public-performance-profile-v1.schema.json +360 -0
  352. data/schema/reduce-v1.schema.json +22 -0
  353. data/schema/reduce-v2.schema.json +63 -0
  354. data/schema/runtime-coverage-v1.schema.json +86 -0
  355. data/schema/runtime-event-v1.schema.json +308 -0
  356. data/schema/table-artifact-v1.schema.json +377 -0
  357. data/schema/table-simulation-v1.schema.json +70 -0
  358. data/schema/verification-report-v1.schema.json +222 -0
  359. data/schema/verify-v1.schema.json +42 -0
  360. data/sig/ibex/analysis/digraph.rbs +19 -0
  361. data/sig/ibex/artifact_set.rbs +37 -0
  362. data/sig/ibex/bison_import/importer.rbs +140 -0
  363. data/sig/ibex/bison_import/tokenizer.rbs +91 -0
  364. data/sig/ibex/bison_import.rbs +95 -0
  365. data/sig/ibex/bounded_subprocess.rbs +76 -0
  366. data/sig/ibex/cli/ambiguity.rbs +28 -0
  367. data/sig/ibex/cli/analysis.rbs +43 -0
  368. data/sig/ibex/cli/bison_import.rbs +32 -0
  369. data/sig/ibex/cli/config.rbs +35 -0
  370. data/sig/ibex/cli/counterexample_options.rbs +2 -0
  371. data/sig/ibex/cli/coverage.rbs +53 -0
  372. data/sig/ibex/cli/debug.rbs +28 -0
  373. data/sig/ibex/cli/diagnostics.rbs +51 -0
  374. data/sig/ibex/cli/documentation.rbs +29 -0
  375. data/sig/ibex/cli/equiv.rbs +49 -0
  376. data/sig/ibex/cli/error_messages.rbs +63 -0
  377. data/sig/ibex/cli/explain.rbs +31 -0
  378. data/sig/ibex/cli/fix.rbs +53 -0
  379. data/sig/ibex/cli/formatting.rbs +107 -0
  380. data/sig/ibex/cli/fuzz.rbs +47 -0
  381. data/sig/ibex/cli/fuzz_regressions.rbs +40 -0
  382. data/sig/ibex/cli/generation_artifacts.rbs +69 -0
  383. data/sig/ibex/cli/generation_error_messages.rbs +19 -0
  384. data/sig/ibex/cli/grammar_tests.rbs +48 -0
  385. data/sig/ibex/cli/ir_tools.rbs +39 -0
  386. data/sig/ibex/cli/lsp.rbs +14 -0
  387. data/sig/ibex/cli/outputs.rbs +26 -3
  388. data/sig/ibex/cli/racc_migration.rbs +30 -0
  389. data/sig/ibex/cli/reduce.rbs +60 -0
  390. data/sig/ibex/cli/reduce_reporting.rbs +27 -0
  391. data/sig/ibex/cli/samples.rbs +38 -0
  392. data/sig/ibex/cli/verify.rbs +28 -0
  393. data/sig/ibex/cli/watch.rbs +43 -0
  394. data/sig/ibex/cli.rbs +182 -9
  395. data/sig/ibex/codegen/action_locations.rbs +45 -0
  396. data/sig/ibex/codegen/action_method_source.rbs +65 -0
  397. data/sig/ibex/codegen/action_source.rbs +50 -0
  398. data/sig/ibex/codegen/ambiguity.rbs +60 -0
  399. data/sig/ibex/codegen/cst_metadata.rbs +59 -0
  400. data/sig/ibex/codegen/documentation.rbs +50 -0
  401. data/sig/ibex/codegen/explain.rbs +105 -0
  402. data/sig/ibex/codegen/generated_action_abi.rbs +105 -0
  403. data/sig/ibex/codegen/html.rbs +18 -2
  404. data/sig/ibex/codegen/mermaid.rbs +16 -0
  405. data/sig/ibex/codegen/railroad.rbs +82 -0
  406. data/sig/ibex/codegen/railroad_documentation.rbs +31 -0
  407. data/sig/ibex/codegen/rbs.rbs +86 -4
  408. data/sig/ibex/codegen/report.rbs +16 -8
  409. data/sig/ibex/codegen/ruby.rbs +105 -28
  410. data/sig/ibex/codegen/ruby_actions.rbs +54 -0
  411. data/sig/ibex/codegen/ruby_ast.rbs +34 -0
  412. data/sig/ibex/codegen/ruby_error_messages.rbs +16 -0
  413. data/sig/ibex/codegen/ruby_lexer.rbs +25 -0
  414. data/sig/ibex/codegen/ruby_syntax.rbs +27 -0
  415. data/sig/ibex/codegen/ruby_table_metadata.rbs +26 -0
  416. data/sig/ibex/codegen/ruby_value_printers.rbs +28 -0
  417. data/sig/ibex/configuration/analysis_grammar.rbs +13 -0
  418. data/sig/ibex/configuration/explanation.rbs +195 -0
  419. data/sig/ibex/configuration/inspector.rbs +53 -0
  420. data/sig/ibex/configuration.rbs +242 -0
  421. data/sig/ibex/coverage/collector.rbs +78 -0
  422. data/sig/ibex/coverage/event_stream.rbs +44 -0
  423. data/sig/ibex/coverage/report.rbs +102 -0
  424. data/sig/ibex/coverage/runtime_event_validator.rbs +70 -0
  425. data/sig/ibex/coverage.rbs +7 -0
  426. data/sig/ibex/delta_reducer.rbs +38 -0
  427. data/sig/ibex/diff.rbs +47 -0
  428. data/sig/ibex/equiv/machine.rbs +54 -0
  429. data/sig/ibex/equiv.rbs +129 -0
  430. data/sig/ibex/error_messages/parser.rbs +58 -0
  431. data/sig/ibex/error_messages/parser_v2.rbs +67 -0
  432. data/sig/ibex/error_messages/renderer.rbs +23 -0
  433. data/sig/ibex/error_messages/sentence_search.rbs +80 -0
  434. data/sig/ibex/error_messages/update.rbs +43 -0
  435. data/sig/ibex/error_messages.rbs +85 -0
  436. data/sig/ibex/fix.rbs +138 -0
  437. data/sig/ibex/frontend/ast.rbs +237 -22
  438. data/sig/ibex/frontend/bootstrap_parser.rbs +13 -0
  439. data/sig/ibex/frontend/diagnostic.rbs +53 -0
  440. data/sig/ibex/frontend/diagnostic_recovery.rbs +98 -0
  441. data/sig/ibex/frontend/dsl.rbs +33 -4
  442. data/sig/ibex/frontend/formatter.rbs +143 -0
  443. data/sig/ibex/frontend/generated_parser.rbs +228 -68
  444. data/sig/ibex/frontend/generated_parser_base.rbs +65 -12
  445. data/sig/ibex/frontend/generated_parser_includes.rbs +23 -0
  446. data/sig/ibex/frontend/generated_parser_metadata.rbs +23 -0
  447. data/sig/ibex/frontend/generated_parser_parameters.rbs +24 -0
  448. data/sig/ibex/frontend/generation.rbs +6 -0
  449. data/sig/ibex/frontend/lexer.rbs +49 -2
  450. data/sig/ibex/frontend/lexer_recovery.rbs +25 -0
  451. data/sig/ibex/frontend/parser/declarations.rbs +57 -0
  452. data/sig/ibex/frontend/parser/parameters.rbs +28 -0
  453. data/sig/ibex/frontend/parser/rules.rbs +3 -0
  454. data/sig/ibex/frontend/parser.rbs +66 -0
  455. data/sig/ibex/frontend/parser_configuration_support.rbs +27 -0
  456. data/sig/ibex/frontend/regenerator.rbs +11 -0
  457. data/sig/ibex/frontend/resolution.rbs +34 -0
  458. data/sig/ibex/frontend/resolver.rbs +91 -0
  459. data/sig/ibex/frontend/rule_documentation.rbs +42 -0
  460. data/sig/ibex/frontend/source_cursor.rbs +38 -5
  461. data/sig/ibex/frontend/source_document.rbs +119 -0
  462. data/sig/ibex/frontend/source_loader.rbs +66 -0
  463. data/sig/ibex/frontend/source_span.rbs +56 -0
  464. data/sig/ibex/frontend/token_adapter/declaration_document_state.rbs +21 -0
  465. data/sig/ibex/frontend/token_adapter/declaration_lexer_state.rbs +27 -0
  466. data/sig/ibex/frontend/token_adapter/declaration_state.rbs +83 -3
  467. data/sig/ibex/frontend/token_adapter/rule_state.rbs +20 -0
  468. data/sig/ibex/frontend/token_adapter.rbs +11 -2
  469. data/sig/ibex/frontend.rbs +1 -1
  470. data/sig/ibex/fuzz.rbs +71 -0
  471. data/sig/ibex/generation_input.rbs +37 -0
  472. data/sig/ibex/generation_manifest.rbs +70 -0
  473. data/sig/ibex/generation_transaction.rbs +82 -0
  474. data/sig/ibex/generation_transaction_recovery.rbs +36 -0
  475. data/sig/ibex/generation_transaction_validation.rbs +65 -0
  476. data/sig/ibex/grammar_tests.rbs +93 -0
  477. data/sig/ibex/ir/automaton_ir.rbs +35 -10
  478. data/sig/ibex/ir/grammar_ir.rbs +96 -22
  479. data/sig/ibex/ir/lexer_ir.rbs +57 -0
  480. data/sig/ibex/ir/parser_contract.rbs +69 -0
  481. data/sig/ibex/ir/serialize.rbs +75 -23
  482. data/sig/ibex/ir/validator/automaton.rbs +112 -0
  483. data/sig/ibex/ir/validator/base.rbs +69 -0
  484. data/sig/ibex/ir/validator/grammar.rbs +195 -0
  485. data/sig/ibex/ir/validator/lexer.rbs +37 -0
  486. data/sig/ibex/ir/validator.rbs +16 -0
  487. data/sig/ibex/ir.rbs +45 -5
  488. data/sig/ibex/lalr/build_metrics.rbs +60 -0
  489. data/sig/ibex/lalr/builder.rbs +150 -19
  490. data/sig/ibex/lalr/conflict_search.rbs +12 -5
  491. data/sig/ibex/lalr/counterexample.rbs +20 -14
  492. data/sig/ibex/lalr/direct_lookaheads.rbs +93 -0
  493. data/sig/ibex/lalr/goto_follows.rbs +89 -0
  494. data/sig/ibex/lalr/ielr/annotator.rbs +67 -0
  495. data/sig/ibex/lalr/ielr/bits.rbs +14 -0
  496. data/sig/ibex/lalr/ielr/inadequacy.rbs +24 -0
  497. data/sig/ibex/lalr/ielr/item_lookaheads.rbs +35 -0
  498. data/sig/ibex/lalr/ielr/pipeline.rbs +17 -0
  499. data/sig/ibex/lalr/ielr/split_stability.rbs +27 -0
  500. data/sig/ibex/lalr/ielr/split_state.rbs +14 -0
  501. data/sig/ibex/lalr/ielr/state_splitter.rbs +76 -0
  502. data/sig/ibex/lalr/ielr_partition.rbs +71 -0
  503. data/sig/ibex/lalr/inadequacy_report.rbs +21 -0
  504. data/sig/ibex/lalr/lookahead_propagation.rbs +38 -0
  505. data/sig/ibex/lalr/lr0_collection.rbs +50 -0
  506. data/sig/ibex/lalr/on_error_reductions.rbs +22 -0
  507. data/sig/ibex/lalr/unreachable_states.rbs +25 -0
  508. data/sig/ibex/lalr.rbs +17 -1
  509. data/sig/ibex/location.rbs +67 -0
  510. data/sig/ibex/lsp/document_handlers.rbs +24 -0
  511. data/sig/ibex/lsp/document_store.rbs +94 -0
  512. data/sig/ibex/lsp/document_store_diagnostics.rbs +20 -0
  513. data/sig/ibex/lsp/document_store_validation.rbs +26 -0
  514. data/sig/ibex/lsp/initialization_handlers.rbs +30 -0
  515. data/sig/ibex/lsp/navigation_handlers.rbs +33 -0
  516. data/sig/ibex/lsp/parser_configuration_assistance.rbs +49 -0
  517. data/sig/ibex/lsp/position_codec.rbs +44 -0
  518. data/sig/ibex/lsp/protocol_error.rbs +32 -0
  519. data/sig/ibex/lsp/request_handlers.rbs +26 -0
  520. data/sig/ibex/lsp/request_support.rbs +40 -0
  521. data/sig/ibex/lsp/server.rbs +55 -0
  522. data/sig/ibex/lsp/symbol_index.rbs +77 -0
  523. data/sig/ibex/lsp/symbol_index_builder.rbs +88 -0
  524. data/sig/ibex/lsp/symbol_index_precedence_references.rbs +18 -0
  525. data/sig/ibex/lsp/symbol_index_source_queries.rbs +26 -0
  526. data/sig/ibex/lsp/symbol_occurrence.rbs +31 -0
  527. data/sig/ibex/lsp/transport.rbs +47 -0
  528. data/sig/ibex/lsp/workspace.rbs +42 -0
  529. data/sig/ibex/lsp/workspace_analyzer.rbs +69 -0
  530. data/sig/ibex/lsp.rbs +7 -0
  531. data/sig/ibex/messages.rbs +26 -0
  532. data/sig/ibex/metrics.rbs +50 -0
  533. data/sig/ibex/normalize/declarations.rbs +31 -0
  534. data/sig/ibex/normalize/diagnostics.rbs +13 -4
  535. data/sig/ibex/normalize/expander.rbs +22 -16
  536. data/sig/ibex/normalize/expression.rbs +11 -9
  537. data/sig/ibex/normalize/grammar_builder.rbs +11 -0
  538. data/sig/ibex/normalize/inline_expansion.rbs +129 -0
  539. data/sig/ibex/normalize/inline_validation.rbs +40 -0
  540. data/sig/ibex/normalize/lexer.rbs +37 -0
  541. data/sig/ibex/normalize/named_references.rbs +20 -0
  542. data/sig/ibex/normalize/nodes.rbs +14 -0
  543. data/sig/ibex/normalize/parameter_ebnf_lowering.rbs +33 -0
  544. data/sig/ibex/normalize/parameter_lowering.rbs +35 -0
  545. data/sig/ibex/normalize/parameter_substitution.rbs +42 -0
  546. data/sig/ibex/normalize/parameter_validation.rbs +44 -0
  547. data/sig/ibex/normalize/parameters.rbs +51 -0
  548. data/sig/ibex/normalize/parser_configuration.rbs +23 -0
  549. data/sig/ibex/normalize/recovery_declarations.rbs +24 -0
  550. data/sig/ibex/normalize.rbs +129 -18
  551. data/sig/ibex/racc_migration/checker.rbs +36 -0
  552. data/sig/ibex/racc_migration/harness.rbs +16 -0
  553. data/sig/ibex/racc_migration/report.rbs +55 -0
  554. data/sig/ibex/racc_migration.rbs +8 -0
  555. data/sig/ibex/rake_task.rbs +51 -0
  556. data/sig/ibex/samples.rbs +60 -0
  557. data/sig/ibex/table_artifact/builder.rbs +93 -0
  558. data/sig/ibex/table_artifact/cst_projection.rbs +20 -0
  559. data/sig/ibex/table_artifact/document.rbs +27 -0
  560. data/sig/ibex/table_artifact/executor.rbs +66 -0
  561. data/sig/ibex/table_artifact/serializer.rbs +25 -0
  562. data/sig/ibex/table_artifact/validator/metadata.rbs +52 -0
  563. data/sig/ibex/table_artifact/validator/support.rbs +46 -0
  564. data/sig/ibex/table_artifact/validator/tables.rbs +63 -0
  565. data/sig/ibex/table_artifact/validator.rbs +66 -0
  566. data/sig/ibex/table_artifact.rbs +32 -0
  567. data/sig/ibex/table_simulation/result.rbs +36 -0
  568. data/sig/ibex/table_simulation/simulator.rbs +98 -0
  569. data/sig/ibex/table_simulation/step.rbs +42 -0
  570. data/sig/ibex/table_simulation/text.rbs +14 -0
  571. data/sig/ibex/table_simulation.rbs +7 -0
  572. data/sig/ibex/tables.rbs +10 -35
  573. data/sig/ibex/verifiable_generation_bundle.rbs +24 -0
  574. data/sig/ibex/verification_report/builder.rbs +53 -0
  575. data/sig/ibex/verification_report/canonical_ir.rbs +41 -0
  576. data/sig/ibex/verification_report/logical_path.rbs +34 -0
  577. data/sig/ibex/verification_report/validator.rbs +114 -0
  578. data/sig/ibex/verification_report.rbs +44 -0
  579. data/sig/ibex/verify/action_correspondence.rbs +50 -0
  580. data/sig/ibex/verify/language_witness.rbs +112 -0
  581. data/sig/ibex/verify/reference_collection.rbs +63 -0
  582. data/sig/ibex/verify/result.rbs +52 -0
  583. data/sig/ibex/verify/verifier.rbs +170 -0
  584. data/sig/ibex/verify.rbs +6 -0
  585. data/sig/ibex/watch/runner.rbs +58 -0
  586. data/sig/ibex/watch/source_snapshot.rbs +46 -0
  587. data/sig/ibex/watch.rbs +7 -0
  588. data/sig/ibex.rbs +2 -0
  589. metadata +524 -19
  590. data/.rubocop.yml +0 -43
  591. data/CHANGELOG.md +0 -30
  592. data/Rakefile +0 -25
  593. data/Steepfile +0 -10
  594. data/docs/compat-notes.md +0 -37
  595. data/docs/lexer-coverage.md +0 -14
  596. data/docs/phase10-extensions.md +0 -27
  597. data/gemfiles/Gemfile +0 -7
  598. data/gemfiles/Gemfile.lock +0 -98
  599. data/lib/ibex/frontend/grammar.y +0 -156
  600. data/lib/ibex/runtime/parser.rb +0 -360
  601. data/lib/ibex/runtime.rb +0 -8
  602. data/sig/ibex/runtime/parser.rbs +0 -167
  603. data/sig/ibex/runtime.rbs +0 -6
@@ -15,6 +15,7 @@ module Ibex
15
15
  # @rbs @delimiters: DelimiterTracker
16
16
  # @rbs @lhs_column: Integer?
17
17
  # @rbs @rule_seen: bool?
18
+ # @rbs @node_annotation: bool?
18
19
 
19
20
  # @rbs () -> void
20
21
  def initialize
@@ -28,12 +29,20 @@ module Ibex
28
29
  def classify(token, remaining, last_external:, previous_external:)
29
30
  external = if token.type == :identifier
30
31
  classify_identifier(token, remaining, last_external, previous_external)
32
+ elsif token.type == :node
33
+ @node_annotation = true
34
+ :NODE
35
+ elsif token.type == :inline
36
+ classify_inline
37
+ elsif token.type == :empty
38
+ :EMPTY
31
39
  elsif SCALAR_TYPES.key?(token.type)
32
40
  SCALAR_TYPES.fetch(token.type)
33
41
  else
34
42
  classify_punctuation(token)
35
43
  end
36
44
  @delimiters.observe(token, external, last_external)
45
+ @node_annotation = false if token.type == :")" && @node_annotation
37
46
  external
38
47
  end
39
48
 
@@ -60,21 +69,35 @@ module Ibex
60
69
 
61
70
  private
62
71
 
72
+ # @rbs () -> external_token
73
+ def classify_inline
74
+ reset_rule_boundary if @state == :rule_rhs && @delimiters.empty?
75
+ :INLINE
76
+ end
77
+
63
78
  # @rbs (Token token, Array[Token] remaining, external_token? last_external,
64
79
  # external_token? previous_external) -> external_token
65
80
  def classify_identifier(token, remaining, last_external, previous_external)
81
+ return :IDENTIFIER if @node_annotation
66
82
  return rule_lhs(token) if @state == :rules_lhs
67
83
  return :IDENTIFIER unless @state == :rule_rhs
68
84
  return :IDENTIFIER if named_reference_name?(last_external, previous_external)
69
85
 
86
+ classify_rhs_identifier(token, remaining)
87
+ end
88
+
89
+ # @rbs (Token token, Array[Token] remaining) -> external_token
90
+ def classify_rhs_identifier(token, remaining)
70
91
  value = string_value(token)
71
- return :IDENTIFIER if value == "end" && @delimiters.open_kind == :separated
92
+ return :IDENTIFIER if value == "end" && @delimiters.open_kind
72
93
  return finish_rules if value == "end"
73
94
 
74
95
  following = remaining.first
75
96
  return :IDENTIFIER unless following
76
- return start_rule(token) if rule_boundary?(token, following)
97
+ return start_rule(token) if rule_boundary?(token, following) ||
98
+ parameterized_rule_boundary?(token, remaining)
77
99
  return separated_terminal(token) if separated_list_call?(token, following)
100
+ return :PARAMETERIZED_REFERENCE if parameterized_call?(token, following)
78
101
 
79
102
  :IDENTIFIER
80
103
  end
@@ -105,6 +128,26 @@ module Ibex
105
128
  @delimiters.empty? && following.type == :":" && token.location.column <= lhs_column
106
129
  end
107
130
 
131
+ # @rbs (Token token, Array[Token] remaining) -> bool
132
+ def parameterized_rule_boundary?(token, remaining)
133
+ lhs_column = @lhs_column
134
+ opening = remaining.first
135
+ return false unless lhs_column && @delimiters.empty? && token.location.column <= lhs_column
136
+ return false unless opening&.type == :"(" && adjacent?(token, opening)
137
+
138
+ parameter_definition_tail?(remaining.drop(1))
139
+ end
140
+
141
+ # @rbs (Array[Token] tail) -> bool
142
+ def parameter_definition_tail?(tail)
143
+ closing = tail.index { |entry| entry.type == :")" }
144
+ return false unless closing && tail[closing + 1]&.type == :":"
145
+
146
+ formals = tail.take(closing)
147
+ !formals.empty? && formals.length.odd? &&
148
+ formals.each_with_index.all? { |entry, index| entry.type == (index.even? ? :identifier : :",") }
149
+ end
150
+
108
151
  # @rbs (Token token) -> external_token
109
152
  def start_rule(token)
110
153
  @rule_seen = true
@@ -129,6 +172,21 @@ module Ibex
129
172
  string_value(token) == "separated_list" ? :SEPARATED_LIST : :SEPARATED_NONEMPTY_LIST
130
173
  end
131
174
 
175
+ # @rbs (Token token, Token following) -> bool
176
+ def parameterized_call?(token, following)
177
+ following.type == :"(" && adjacent?(token, following)
178
+ end
179
+
180
+ # @rbs (Token left, Token right) -> bool
181
+ def adjacent?(left, right)
182
+ left_span = left.span
183
+ right_span = right.span
184
+ return left_span.end_byte == right_span.start_byte if left_span && right_span
185
+
186
+ left.location.line == right.location.line &&
187
+ left.location.column + string_value(left).length == right.location.column
188
+ end
189
+
132
190
  # @rbs (Token token) -> external_token
133
191
  def classify_punctuation(token)
134
192
  @state = :rule_rhs if @state == :rule_colon && token.type == :":"
@@ -15,11 +15,11 @@ module Ibex
15
15
  # @rbs @rules: RuleState?
16
16
  # @rbs @last_external: external_token?
17
17
 
18
- # @rbs (Array[Token] tokens) -> void
19
- def initialize(tokens)
18
+ # @rbs (Array[Token] tokens, ?extended: bool) -> void
19
+ def initialize(tokens, extended: false)
20
20
  @tokens = tokens
21
21
  @index = 0
22
- @declarations = DeclarationState.new
22
+ @declarations = DeclarationState.new(extended: extended)
23
23
  @classifier = @declarations
24
24
  end
25
25
 
@@ -60,6 +60,21 @@ module Ibex
60
60
  @declarations.extended_pragma?
61
61
  end
62
62
 
63
+ # @rbs () -> bool
64
+ def cst_pragma?
65
+ @declarations.cst_pragma?
66
+ end
67
+
68
+ # @rbs () -> Location?
69
+ def extended_pragma_location
70
+ @declarations.extended_pragma_location
71
+ end
72
+
73
+ # @rbs () -> Location?
74
+ def cst_pragma_location
75
+ @declarations.cst_pragma_location
76
+ end
77
+
63
78
  # @rbs () -> parser_section
64
79
  def section
65
80
  rules = @rules
data/lib/ibex/frontend.rb CHANGED
@@ -2,18 +2,30 @@
2
2
 
3
3
  require_relative "error"
4
4
  require_relative "tables"
5
- require_relative "runtime"
5
+ require "ibex/runtime"
6
+ require_relative "frontend/source_span"
6
7
  require_relative "frontend/source_cursor"
8
+ require_relative "frontend/source_document"
9
+ require_relative "frontend/diagnostic"
7
10
  require_relative "frontend/action_scanner"
8
11
  require_relative "frontend/lexer"
12
+ require_relative "frontend/lexer_recovery"
9
13
  require_relative "frontend/ast"
14
+ require_relative "frontend/parser_configuration_support"
15
+ require_relative "frontend/rule_documentation"
16
+ require_relative "frontend/token_adapter/declaration_document_state"
10
17
  require_relative "frontend/token_adapter/declaration_state"
11
18
  require_relative "frontend/token_adapter/delimiter_tracker"
12
19
  require_relative "frontend/token_adapter/rule_state"
13
20
  require_relative "frontend/token_adapter"
14
21
  require_relative "frontend/generated_parser_base"
15
22
  require_relative "frontend/generated_parser"
23
+ require_relative "frontend/diagnostic_recovery"
16
24
  require_relative "frontend/parser"
25
+ require_relative "frontend/formatter"
26
+ require_relative "frontend/resolution"
27
+ require_relative "frontend/source_loader"
28
+ require_relative "frontend/resolver"
17
29
  require_relative "frontend/dsl"
18
30
 
19
31
  module Ibex
@@ -23,6 +35,6 @@ module Ibex
23
35
  # type token_value = String | Integer | user_code_token | nil
24
36
  # type external_token = Symbol | String
25
37
  # type parser_section = :declarations | :rules | :user_code
26
- # type delimiter_kind = :group | :separated
38
+ # type delimiter_kind = :group | :separated | :parameter
27
39
  end
28
40
  end
data/lib/ibex/fuzz.rb ADDED
@@ -0,0 +1,219 @@
1
+ # frozen_string_literal: true
2
+ # rbs_inline: enabled
3
+
4
+ require_relative "samples"
5
+ require_relative "table_simulation"
6
+ require_relative "delta_reducer"
7
+
8
+ module Ibex
9
+ # @rbs!
10
+ # type fuzz_outcome = Symbol | [Symbol, String, String]
11
+ # type fuzz_mismatch = {
12
+ # tokens: Array[String],
13
+ # kind: Symbol,
14
+ # sentence: Integer,
15
+ # outcomes: Hash[Symbol, fuzz_outcome],
16
+ # bounds: Hash[Symbol, Integer]
17
+ # }
18
+ # type fuzz_budget = {
19
+ # message: String,
20
+ # bounds: Hash[Symbol, Integer],
21
+ # ?tokens: Array[String],
22
+ # ?phase: String
23
+ # }
24
+
25
+ # Bounded grammar-derived differential fuzzing without semantic execution.
26
+ class Fuzz
27
+ ALGORITHMS = %i[slr lalr ielr lr1].freeze #: Array[Symbol]
28
+ DEFAULT_MAX_ACTIONS = 100_000 #: Integer
29
+ DEFAULT_MAX_STACK = 10_000 #: Integer
30
+
31
+ class Mismatch < Ibex::Error
32
+ attr_reader :details #: fuzz_mismatch
33
+
34
+ # @rbs (fuzz_mismatch details) -> void
35
+ def initialize(details)
36
+ @details = IR.deep_freeze(details)
37
+ super("(fuzz):1:1: differential mismatch for #{details[:tokens].inspect}")
38
+ end
39
+ end
40
+
41
+ class BudgetExceeded < Ibex::Error
42
+ attr_reader :details #: fuzz_budget
43
+
44
+ # @rbs (fuzz_budget details) -> void
45
+ def initialize(details)
46
+ @details = IR.deep_freeze(details)
47
+ super("(fuzz):1:1: configured budget was exhausted: #{details[:message]}")
48
+ end
49
+ end
50
+
51
+ # @rbs (IR::Grammar grammar, ?seed: Integer, ?count: Integer, ?max_tokens: Integer,
52
+ # ?max_depth: Integer, ?max_expansions: Integer, ?max_actions: Integer, ?max_stack: Integer,
53
+ # ?coverage_guided: bool, ?path_length: Integer, ?algorithms: Array[Symbol],
54
+ # ?ielr_strategy: Symbol, ?automata: Hash[Symbol, IR::Automaton]?, ?against: (^(Array[String]) -> Symbol)?,
55
+ # ?against_description: Hash[Symbol, Object?]?) -> void
56
+ # rubocop:disable Metrics/ParameterLists
57
+ def initialize(grammar, seed: 0, count: 100, max_tokens: 32, max_depth: 16,
58
+ max_expansions: Samples::DEFAULT_MAX_EXPANSIONS, max_actions: DEFAULT_MAX_ACTIONS,
59
+ max_stack: DEFAULT_MAX_STACK, coverage_guided: false, path_length: 2,
60
+ algorithms: ALGORITHMS, ielr_strategy: :partition, automata: nil,
61
+ against: nil, against_description: nil)
62
+ raise ArgumentError, "count must be positive" unless count.positive?
63
+ raise ArgumentError, "algorithms must not be empty" if algorithms.empty?
64
+ unless LALR::Builder::IELR_STRATEGIES.include?(ielr_strategy.to_sym)
65
+ raise ArgumentError, "unknown IELR construction strategy #{ielr_strategy.inspect}"
66
+ end
67
+
68
+ @grammar = grammar
69
+ @seed = seed
70
+ @count = count
71
+ @random = Random.new(seed ^ 0x1BE)
72
+ @max_tokens = max_tokens
73
+ @max_depth = max_depth
74
+ @max_expansions = max_expansions
75
+ @max_actions = max_actions
76
+ @max_stack = max_stack
77
+ @coverage_guided = coverage_guided
78
+ @path_length = path_length
79
+ @algorithms = algorithms.map(&:to_sym).freeze
80
+ @ielr_strategy = ielr_strategy.to_sym
81
+ @automata = automata || build_automata
82
+ @against = against
83
+ @against_description = against_description
84
+ end
85
+ # rubocop:enable Metrics/ParameterLists
86
+
87
+ # @rbs () -> Hash[Symbol, Object?]
88
+ def run
89
+ sentences = generate_sentences
90
+ mutation_count = 0
91
+ sentences.each_with_index do |tokens, index|
92
+ compare!(tokens, kind: :generated, sentence: index)
93
+ mutations(tokens).each do |mutation|
94
+ compare!(mutation, kind: :mutation, sentence: index)
95
+ mutation_count += 1
96
+ end
97
+ end
98
+ successful_report(sentences.length, mutation_count)
99
+ rescue Ibex::Error => e
100
+ raise if e.is_a?(Mismatch) || e.is_a?(BudgetExceeded) || !budget_error?(e)
101
+
102
+ raise BudgetExceeded.new(
103
+ message: e.message,
104
+ bounds: { max_expansions: @max_expansions, max_actions: @max_actions, max_stack: @max_stack }
105
+ )
106
+ end
107
+
108
+ # Minimize one observed mismatch without changing its kind or outcomes.
109
+ # @rbs (Mismatch mismatch, ?max_trials: Integer) -> DeltaReducer::Result
110
+ def minimize(mismatch, max_trials: 1_000)
111
+ details = mismatch.details
112
+ original = details[:tokens]
113
+ kind = details[:kind].to_sym
114
+ sentence = details[:sentence]
115
+ outcomes = details[:outcomes]
116
+ DeltaReducer.new(max_trials: max_trials).minimize(original) do |candidate|
117
+ mismatch_reproduced?(candidate, kind: kind, sentence: sentence, outcomes: outcomes)
118
+ end
119
+ end
120
+
121
+ private
122
+
123
+ # @rbs () -> Array[Array[String]]
124
+ def generate_sentences
125
+ Samples.new(
126
+ @grammar, seed: @seed, max_tokens: @max_tokens, max_depth: @max_depth,
127
+ max_expansions: @max_expansions, strategy: @coverage_guided ? :coverage : :random,
128
+ path_length: @path_length
129
+ ).generate(count: @count)
130
+ end
131
+
132
+ # @rbs (Integer sentence_count, Integer mutation_count) -> untyped
133
+ def successful_report(sentence_count, mutation_count)
134
+ report = {
135
+ ibex_report: "fuzz", schema_version: 1, seed: @seed,
136
+ bounds: {
137
+ sentences: @count, max_tokens: @max_tokens, max_depth: @max_depth,
138
+ max_expansions: @max_expansions, max_actions: @max_actions, max_stack: @max_stack
139
+ },
140
+ strategy: @coverage_guided ? "coverage" : "random",
141
+ path_length: @path_length,
142
+ ielr_strategy: @ielr_strategy.to_s,
143
+ algorithms: @algorithms.map(&:to_s),
144
+ generated_sentences: sentence_count,
145
+ mutated_sentences: mutation_count,
146
+ result: "no_difference_within_bounds"
147
+ }
148
+ report[:external] = @against_description if @against_description
149
+ IR.deep_freeze(report)
150
+ end
151
+
152
+ # @rbs () -> Hash[Symbol, IR::Automaton]
153
+ def build_automata
154
+ @algorithms.to_h do |algorithm|
155
+ builder = LALR::Builder.new(
156
+ @grammar,
157
+ algorithm: algorithm,
158
+ ielr_strategy: @ielr_strategy
159
+ )
160
+ [algorithm, builder.build]
161
+ end
162
+ end
163
+
164
+ # @rbs (Array[String] tokens, kind: Symbol, sentence: Integer) -> void
165
+ def compare!(tokens, kind:, sentence:)
166
+ outcomes = @algorithms.to_h do |algorithm|
167
+ automaton = @automata.fetch(algorithm)
168
+ simulator = TableSimulation::Simulator.new(
169
+ automaton, max_steps: @max_actions, max_stack: @max_stack
170
+ )
171
+ [algorithm, simulator.simulate(tokens).status]
172
+ rescue Ibex::Error => e
173
+ if budget_error?(e)
174
+ raise BudgetExceeded.new(
175
+ message: e.message, tokens: tokens,
176
+ bounds: { max_actions: @max_actions, max_stack: @max_stack }
177
+ )
178
+ end
179
+ [algorithm, [:failure, e.class.name, e.message]]
180
+ end
181
+ outcomes[:external] = @against.call(tokens) if @against
182
+ values = outcomes.values.uniq
183
+ return if values.length == 1 && (kind != :generated || values.first == :accepted)
184
+
185
+ raise Mismatch.new(tokens: tokens, kind: kind, sentence: sentence, outcomes: outcomes,
186
+ bounds: { max_actions: @max_actions, max_stack: @max_stack })
187
+ end
188
+
189
+ # @rbs (Array[String] tokens) -> Array[Array[String]]
190
+ def mutations(tokens)
191
+ terminals = @grammar.terminals.reject(&:reserved).map(&:name)
192
+ replacement = terminals.fetch(@random.rand(terminals.length))
193
+ index = tokens.empty? ? 0 : @random.rand(tokens.length)
194
+ inserted = tokens.dup.insert(index, replacement)
195
+ return [inserted] if tokens.empty?
196
+
197
+ deleted = tokens.dup.tap { |items| items.delete_at(index) }
198
+ replaced = tokens.dup.tap { |items| items[index] = replacement }
199
+ [inserted, deleted, replaced].uniq
200
+ end
201
+
202
+ # @rbs (Array[String] tokens, kind: Symbol, sentence: Integer,
203
+ # outcomes: Hash[Symbol, fuzz_outcome]) -> bool
204
+ def mismatch_reproduced?(tokens, kind:, sentence:, outcomes:)
205
+ compare!(tokens, kind: kind, sentence: sentence)
206
+ false
207
+ rescue Mismatch => e
208
+ e.details[:kind].to_sym == kind &&
209
+ e.details[:outcomes] == outcomes
210
+ end
211
+
212
+ # @rbs (Ibex::Error error) -> bool
213
+ def budget_error?(error)
214
+ error.message.match?(
215
+ /simulation exceeded|(?:limit|maximum|budget).*(?:exceed|exhaust)|exceed.*(?:limit|maximum|budget)/i
216
+ )
217
+ end
218
+ end
219
+ end
@@ -0,0 +1,57 @@
1
+ # frozen_string_literal: true
2
+ # rbs_inline: enabled
3
+
4
+ require "digest"
5
+
6
+ module Ibex
7
+ # Digest and identity of source bytes actually consumed by one generation.
8
+ class GenerationInput
9
+ attr_reader :path #: String
10
+ attr_reader :sha256 #: String
11
+ attr_reader :bytesize #: Integer
12
+ attr_reader :dev #: Integer
13
+ attr_reader :ino #: Integer
14
+ attr_reader :access_paths #: Array[String]
15
+
16
+ # @rbs (String path, String source, ?access_path: String) -> void
17
+ def initialize(path, source, access_path: path)
18
+ @path = File.realpath(path).freeze
19
+ stat = File.stat(@path)
20
+ @sha256 = Digest::SHA256.hexdigest(source).freeze
21
+ @bytesize = source.bytesize
22
+ @dev = stat.dev
23
+ @ino = stat.ino
24
+ @access_paths = [File.expand_path(access_path)] #: Array[String]
25
+ end
26
+
27
+ # Record another lexical path that supplied these canonical bytes.
28
+ # @rbs (String path) -> void
29
+ def add_access_path(path)
30
+ expanded = File.expand_path(path)
31
+ @access_paths << expanded unless @access_paths.include?(expanded)
32
+ end
33
+
34
+ # @rbs () -> Hash[String, String | Integer]
35
+ def to_h
36
+ { "path" => @path, "sha256" => @sha256, "bytesize" => @bytesize }
37
+ end
38
+
39
+ # Detect content changes even when size and timestamps are preserved.
40
+ # @rbs () -> bool
41
+ def current?
42
+ source = File.binread(@path)
43
+ stat = File.stat(@path)
44
+ lexical_paths_current? && stat.dev == @dev && stat.ino == @ino &&
45
+ source.bytesize == @bytesize && Digest::SHA256.hexdigest(source) == @sha256
46
+ rescue SystemCallError
47
+ false
48
+ end
49
+
50
+ private
51
+
52
+ # @rbs () -> bool
53
+ def lexical_paths_current?
54
+ @access_paths.all? { |path| File.realpath(path) == @path }
55
+ end
56
+ end
57
+ end
@@ -0,0 +1,211 @@
1
+ # frozen_string_literal: true
2
+ # rbs_inline: enabled
3
+
4
+ require "digest"
5
+ require "json"
6
+
7
+ module Ibex
8
+ # Deterministic description of one published parser generation.
9
+ # rubocop:disable Metrics/ModuleLength -- manifest schema validation is kept in one auditable unit.
10
+ module GenerationManifest
11
+ # @rbs!
12
+ # type json_value = String | Integer | Float | bool | nil | Array[json_value] | Hash[String, json_value]
13
+
14
+ SCHEMA_VERSION = 1 #: Integer
15
+ IDENTIFIER = "generation" #: String
16
+ ROOT_KEYS = %w[ibex_manifest schema_version input options artifacts].freeze #: Array[String]
17
+ INPUT_KEYS = %w[root sha256 files].freeze #: Array[String]
18
+ FILE_KEYS = %w[path sha256 bytesize].freeze #: Array[String]
19
+ ARTIFACT_KEYS = %w[kind path sha256 bytesize].freeze #: Array[String]
20
+
21
+ module_function
22
+
23
+ # @rbs (ArtifactSet artifacts, source_records: Array[GenerationInput], options: Hash[String, Object?]) -> String
24
+ def render(artifacts, source_records:, options:)
25
+ files = source_records.map(&:to_h)
26
+ document = {
27
+ "ibex_manifest" => IDENTIFIER,
28
+ "schema_version" => SCHEMA_VERSION,
29
+ "input" => {
30
+ "root" => files.fetch(0).fetch("path"),
31
+ "sha256" => input_digest(files),
32
+ "files" => files
33
+ },
34
+ "options" => sorted_hash(options),
35
+ "artifacts" => artifacts.map { |artifact| artifact_entry(artifact) }
36
+ }
37
+ "#{JSON.pretty_generate(document)}\n"
38
+ end
39
+
40
+ # Validate the manifest shape and, by default, every published artifact digest.
41
+ # @rbs (String source, ?verify_artifacts: bool) -> Hash[String, json_value]
42
+ def validate(source, verify_artifacts: true)
43
+ document = JSON.parse(source) #: Object?
44
+ validate_document(document)
45
+ document_hash = document #: Hash[String, json_value]
46
+ artifacts = document_hash.fetch("artifacts") #: Array[Object?]
47
+ verify_entries(artifacts) if verify_artifacts
48
+ document_hash
49
+ rescue JSON::ParserError, KeyError, TypeError, ArgumentError => e
50
+ raise Ibex::Error, "(manifest):1:1: invalid generation manifest: #{e.message}"
51
+ end
52
+
53
+ # @rbs (String path, ?verify_artifacts: bool) -> Hash[String, json_value]
54
+ def validate_file(path, verify_artifacts: true)
55
+ validate(File.binread(path), verify_artifacts: verify_artifacts)
56
+ rescue SystemCallError, ArgumentError => e
57
+ raise Ibex::Error, "#{path}:1:1: cannot read generation manifest: #{e.message}"
58
+ end
59
+
60
+ # @rbs (Artifact artifact) -> Hash[String, json_value]
61
+ def artifact_entry(artifact)
62
+ {
63
+ "kind" => artifact.kind.to_s,
64
+ "path" => File.expand_path(artifact.path),
65
+ "sha256" => Digest::SHA256.hexdigest(artifact.content),
66
+ "bytesize" => artifact.content.bytesize
67
+ }
68
+ end
69
+ private_class_method :artifact_entry
70
+
71
+ # @rbs (Array[Hash[String, json_value]] files) -> String
72
+ def input_digest(files)
73
+ digest = Digest::SHA256.new
74
+ files.each do |entry|
75
+ path = entry.fetch("path") # @type var path: String
76
+ sha256 = entry.fetch("sha256") # @type var sha256: String
77
+ bytesize = entry.fetch("bytesize") # @type var bytesize: Integer
78
+ digest << path << "\0" << sha256 << "\0" << bytesize.to_s << "\0"
79
+ end
80
+ digest.hexdigest
81
+ end
82
+ private_class_method :input_digest
83
+
84
+ # @rbs (Hash[String, Object?] value) -> Hash[String, Object?]
85
+ def sorted_hash(value)
86
+ value.keys.sort.to_h do |key|
87
+ child = value.fetch(key)
88
+ normalized = child.is_a?(Hash) ? sorted_hash(child) : child
89
+ [key, normalized]
90
+ end
91
+ end
92
+ private_class_method :sorted_hash
93
+
94
+ # @rbs (Object? document) -> void
95
+ def validate_document(document)
96
+ raise TypeError, "document must be an object" unless document.is_a?(Hash)
97
+
98
+ exact_keys!(document, ROOT_KEYS, "document")
99
+ validate_identity(document)
100
+ validate_input(document.fetch("input"))
101
+ artifacts = document.fetch("artifacts")
102
+ raise TypeError, "options must be an object" unless document.fetch("options").is_a?(Hash)
103
+ raise TypeError, "artifacts must be an array" unless artifacts.is_a?(Array)
104
+ raise TypeError, "artifacts must not be empty" if artifacts.empty?
105
+
106
+ validate_entries(artifacts, "artifact", require_kind: true)
107
+ end
108
+ private_class_method :validate_document
109
+
110
+ # @rbs (Hash[Object?, Object?] document) -> void
111
+ def validate_identity(document)
112
+ raise TypeError, "unsupported manifest identifier" unless document["ibex_manifest"] == IDENTIFIER
113
+ raise TypeError, "unsupported schema_version" unless document["schema_version"] == SCHEMA_VERSION
114
+ end
115
+ private_class_method :validate_identity
116
+
117
+ # @rbs (Object? input) -> void
118
+ def validate_input(input)
119
+ raise TypeError, "input must be an object" unless input.is_a?(Hash)
120
+
121
+ exact_keys!(input, INPUT_KEYS, "input")
122
+
123
+ files = input.fetch("files")
124
+ raise TypeError, "input files must be a non-empty array" unless files.is_a?(Array) && !files.empty?
125
+
126
+ validate_entries(files, "input file")
127
+ expected = input_digest(files)
128
+ root = input.fetch("root")
129
+ raise TypeError, "input root must be a non-empty string" unless root.is_a?(String) && !root.empty?
130
+ raise TypeError, "input digest mismatch" unless input.fetch("sha256") == expected
131
+ raise TypeError, "input root mismatch" unless root == files.fetch(0).fetch("path")
132
+ end
133
+ private_class_method :validate_input
134
+
135
+ # @rbs (Array[Object?] entries, String label, ?require_kind: bool) -> void
136
+ def validate_entries(entries, label, require_kind: false)
137
+ entries.each { |entry| validate_entry(entry, label, require_kind) }
138
+ end
139
+ private_class_method :validate_entries
140
+
141
+ # @rbs (Object? entry, String label, bool require_kind) -> void
142
+ def validate_entry(entry, label, require_kind)
143
+ raise TypeError, "#{label} must be an object" unless entry.is_a?(Hash)
144
+
145
+ exact_keys!(entry, require_kind ? ARTIFACT_KEYS : FILE_KEYS, label)
146
+ validate_path(entry.fetch("path"), label)
147
+ validate_kind(entry.fetch("kind"), label) if require_kind
148
+ validate_digest(entry.fetch("sha256"), label)
149
+ bytesize = entry.fetch("bytesize")
150
+ return if bytesize.is_a?(Integer) && bytesize >= 0
151
+
152
+ raise TypeError, "#{label} bytesize must be non-negative"
153
+ end
154
+ private_class_method :validate_entry
155
+
156
+ # @rbs (Object? path, String label) -> void
157
+ def validate_path(path, label)
158
+ raise TypeError, "#{label} path must be a non-empty string" unless path.is_a?(String) && !path.empty?
159
+ return unless path.include?("\0") || path.match?(/[[:cntrl:]]/)
160
+
161
+ raise TypeError, "#{label} path must not contain control characters"
162
+ end
163
+ private_class_method :validate_path
164
+
165
+ # @rbs (Object? kind, String label) -> void
166
+ def validate_kind(kind, label)
167
+ return if kind.is_a?(String) && !kind.empty?
168
+
169
+ raise TypeError, "#{label} kind must be a non-empty string"
170
+ end
171
+ private_class_method :validate_kind
172
+
173
+ # @rbs (Object? digest, String label) -> void
174
+ def validate_digest(digest, label)
175
+ return if digest.is_a?(String) && digest.match?(/\A[0-9a-f]{64}\z/)
176
+
177
+ raise TypeError, "#{label} digest must be SHA-256"
178
+ end
179
+ private_class_method :validate_digest
180
+
181
+ # @rbs (Hash[Object?, Object?] value, Array[String] expected, String label) -> void
182
+ def exact_keys!(value, expected, label)
183
+ keys = value.keys
184
+ raise TypeError, "#{label} keys must be strings" unless keys.all?(String)
185
+
186
+ extras = keys - expected
187
+ missing = expected - keys
188
+ raise TypeError, "#{label} has unknown property #{extras.fetch(0)}" unless extras.empty?
189
+ raise KeyError, "#{label} is missing #{missing.fetch(0)}" unless missing.empty?
190
+ end
191
+ private_class_method :exact_keys!
192
+
193
+ # @rbs (Array[Object?] entries) -> void
194
+ def verify_entries(entries)
195
+ entries.each do |entry|
196
+ entry_hash = entry #: Hash[String, json_value]
197
+ path = entry_hash.fetch("path") #: String
198
+ source = File.binread(path)
199
+ bytesize = entry_hash.fetch("bytesize") #: Integer
200
+ sha256 = entry_hash.fetch("sha256") #: String
201
+ raise TypeError, "artifact bytesize mismatch for #{path}" unless source.bytesize == bytesize
202
+ raise TypeError, "artifact digest mismatch for #{path}" unless
203
+ Digest::SHA256.hexdigest(source) == sha256
204
+ end
205
+ rescue SystemCallError => e
206
+ raise TypeError, "cannot read artifact: #{e.message}"
207
+ end
208
+ private_class_method :verify_entries
209
+ end
210
+ # rubocop:enable Metrics/ModuleLength
211
+ end