rattler 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (329) hide show
  1. data/README.rdoc +3 -175
  2. data/features/README.markdown +27 -0
  3. data/features/Tutorial.md +224 -0
  4. data/features/command_line/output_option.feature +2 -1
  5. data/features/command_line/{parser_generator.feature → rtlr.feature} +43 -15
  6. data/features/error_reporting/automatic_error_messages.feature +40 -0
  7. data/features/error_reporting/custom_error_messages.feature +28 -0
  8. data/features/examples/json_parser.markdown +88 -0
  9. data/features/{grammar → extended_matching_syntax}/back_reference.feature +5 -3
  10. data/features/{grammar → extended_matching_syntax}/e_symbol.feature +2 -2
  11. data/features/{grammar → extended_matching_syntax}/eof.feature +4 -3
  12. data/features/{grammar → extended_matching_syntax}/fail.feature +8 -6
  13. data/features/extended_matching_syntax/fragments.feature +29 -0
  14. data/features/extended_matching_syntax/include.feature +42 -0
  15. data/features/{grammar → extended_matching_syntax}/list_matching.feature +7 -6
  16. data/features/extended_matching_syntax/posix_class.feature +127 -0
  17. data/features/{grammar → extended_matching_syntax}/repeat.feature +29 -3
  18. data/features/{grammar → extended_matching_syntax}/skip_operator.feature +2 -1
  19. data/features/extended_matching_syntax/super.feature +24 -0
  20. data/features/{grammar → extended_matching_syntax}/token.feature +6 -5
  21. data/features/{grammar → extended_matching_syntax}/whitespace.feature +7 -6
  22. data/features/{grammar → extended_matching_syntax}/word_literal.feature +10 -6
  23. data/features/grammar_heading/explicit_start_rule.feature +20 -0
  24. data/features/grammar_heading/grammar_declaration.feature +60 -0
  25. data/features/grammar_heading/include.feature +19 -0
  26. data/features/grammar_heading/require.feature +27 -0
  27. data/features/{grammar → peg_syntax}/any_character.feature +1 -1
  28. data/features/peg_syntax/character_class.feature +25 -0
  29. data/features/{grammar → peg_syntax}/comments.feature +1 -1
  30. data/features/{grammar → peg_syntax}/literal.feature +5 -3
  31. data/features/{grammar → peg_syntax}/negative_lookahead.feature +5 -3
  32. data/features/peg_syntax/nonterminal.feature +46 -0
  33. data/features/peg_syntax/one_or_more.feature +59 -0
  34. data/features/{grammar → peg_syntax}/optional.feature +2 -2
  35. data/features/peg_syntax/ordered_choice.feature +24 -0
  36. data/features/{grammar → peg_syntax}/positive_lookahead.feature +6 -4
  37. data/features/peg_syntax/sequence.feature +23 -0
  38. data/features/{grammar → peg_syntax}/start_rule.feature +1 -1
  39. data/features/peg_syntax/zero_or_more.feature +59 -0
  40. data/features/{grammar → semantics}/labels.feature +0 -0
  41. data/features/{grammar → semantics}/negative_semantic_predicate.feature +30 -9
  42. data/features/{grammar → semantics}/node_action.feature +0 -0
  43. data/features/{grammar → semantics}/positive_semantic_predicate.feature +29 -8
  44. data/features/{grammar/symantic_action.feature → semantics/semantic_action.feature} +2 -2
  45. data/features/semantics/semantic_result.feature +86 -0
  46. data/features/{grammar → semantics}/side_effect.feature +33 -21
  47. data/features/step_definitions/cli_steps.rb +1 -1
  48. data/features/step_definitions/grammar_steps.rb +19 -5
  49. data/features/support/env.rb +5 -0
  50. data/lib/rattler.rb +21 -44
  51. data/lib/rattler/compiler.rb +69 -0
  52. data/lib/rattler/{grammar → compiler}/grammar_parser.rb +58 -24
  53. data/lib/rattler/compiler/metagrammar.rb +1570 -0
  54. data/lib/rattler/compiler/optimizer.rb +77 -0
  55. data/lib/rattler/{back_end → compiler}/optimizer/composite_reducing.rb +2 -2
  56. data/lib/rattler/{back_end → compiler}/optimizer/flatten_choice.rb +3 -12
  57. data/lib/rattler/{back_end → compiler}/optimizer/flatten_sequence.rb +4 -16
  58. data/lib/rattler/{back_end → compiler}/optimizer/flattening.rb +2 -2
  59. data/lib/rattler/compiler/optimizer/inline_regular_rules.rb +24 -0
  60. data/lib/rattler/{back_end → compiler}/optimizer/join_match_capturing_sequence.rb +16 -14
  61. data/lib/rattler/{back_end → compiler}/optimizer/join_match_choice.rb +4 -13
  62. data/lib/rattler/{back_end → compiler}/optimizer/join_match_matching_sequence.rb +4 -13
  63. data/lib/rattler/compiler/optimizer/join_match_sequence.rb +7 -0
  64. data/lib/rattler/{back_end → compiler}/optimizer/join_predicate_bare_match.rb +3 -12
  65. data/lib/rattler/compiler/optimizer/join_predicate_match.rb +7 -0
  66. data/lib/rattler/{back_end → compiler}/optimizer/join_predicate_nested_match.rb +4 -13
  67. data/lib/rattler/{back_end → compiler}/optimizer/join_predicate_or_bare_match.rb +3 -12
  68. data/lib/rattler/compiler/optimizer/join_predicate_or_match.rb +7 -0
  69. data/lib/rattler/{back_end → compiler}/optimizer/join_predicate_or_nested_match.rb +4 -13
  70. data/lib/rattler/{back_end → compiler}/optimizer/match_joining.rb +2 -2
  71. data/lib/rattler/{back_end → compiler}/optimizer/optimization.rb +12 -34
  72. data/lib/rattler/compiler/optimizer/optimization_context.rb +83 -0
  73. data/lib/rattler/{back_end → compiler}/optimizer/optimization_sequence.rb +3 -11
  74. data/lib/rattler/compiler/optimizer/optimizations.rb +27 -0
  75. data/lib/rattler/{back_end → compiler}/optimizer/optimize_children.rb +6 -14
  76. data/lib/rattler/{back_end → compiler}/optimizer/reduce_repeat_match.rb +4 -13
  77. data/lib/rattler/compiler/optimizer/remove_meaningless_wrapper.rb +22 -0
  78. data/lib/rattler/{back_end → compiler}/optimizer/simplify_redundant_repeat.rb +4 -13
  79. data/lib/rattler/{back_end → compiler}/optimizer/simplify_token_match.rb +4 -13
  80. data/lib/rattler/compiler/parser_generator.rb +108 -0
  81. data/lib/rattler/{back_end → compiler}/parser_generator/apply_generator.rb +7 -21
  82. data/lib/rattler/{back_end → compiler}/parser_generator/assert_generator.rb +11 -19
  83. data/lib/rattler/compiler/parser_generator/attributed_sequence_generator.rb +37 -0
  84. data/lib/rattler/{back_end → compiler}/parser_generator/back_reference_generator.rb +10 -10
  85. data/lib/rattler/{back_end → compiler}/parser_generator/choice_generator.rb +10 -22
  86. data/lib/rattler/{back_end → compiler}/parser_generator/delegating_generator.rb +2 -2
  87. data/lib/rattler/{back_end → compiler}/parser_generator/disallow_generator.rb +11 -19
  88. data/lib/rattler/{back_end → compiler}/parser_generator/e_symbol_generator.rb +2 -10
  89. data/lib/rattler/{back_end → compiler}/parser_generator/eof_generator.rb +2 -10
  90. data/lib/rattler/{back_end → compiler}/parser_generator/expr_generator.rb +9 -35
  91. data/lib/rattler/{back_end → compiler}/parser_generator/fail_generator.rb +7 -3
  92. data/lib/rattler/{back_end → compiler}/parser_generator/gen_method_names.rb +3 -5
  93. data/lib/rattler/{back_end → compiler}/parser_generator/general_list_generator.rb +6 -18
  94. data/lib/rattler/{back_end → compiler}/parser_generator/general_repeat_generator.rb +16 -22
  95. data/lib/rattler/{back_end/parser_generator/rule_set_generator.rb → compiler/parser_generator/grammar_generator.rb} +24 -33
  96. data/lib/rattler/compiler/parser_generator/group_match.rb +33 -0
  97. data/lib/rattler/{back_end → compiler}/parser_generator/group_match_generator.rb +4 -17
  98. data/lib/rattler/compiler/parser_generator/label_generator.rb +37 -0
  99. data/lib/rattler/{back_end → compiler}/parser_generator/list0_generating.rb +5 -5
  100. data/lib/rattler/{back_end → compiler}/parser_generator/list1_generating.rb +3 -3
  101. data/lib/rattler/{back_end → compiler}/parser_generator/list_generator.rb +2 -2
  102. data/lib/rattler/{back_end → compiler}/parser_generator/match_generator.rb +10 -10
  103. data/lib/rattler/{back_end → compiler}/parser_generator/nested.rb +2 -2
  104. data/lib/rattler/compiler/parser_generator/node_action_generator.rb +49 -0
  105. data/lib/rattler/{back_end → compiler}/parser_generator/one_or_more_generating.rb +3 -3
  106. data/lib/rattler/{back_end → compiler}/parser_generator/optional_generating.rb +6 -22
  107. data/lib/rattler/compiler/parser_generator/predicate_propogating.rb +24 -0
  108. data/lib/rattler/{back_end → compiler}/parser_generator/repeat_generator.rb +2 -2
  109. data/lib/rattler/compiler/parser_generator/rule_generator.rb +66 -0
  110. data/lib/rattler/compiler/parser_generator/rule_set_generator.rb +33 -0
  111. data/lib/rattler/compiler/parser_generator/semantic_action_generator.rb +58 -0
  112. data/lib/rattler/compiler/parser_generator/sequence_generating.rb +138 -0
  113. data/lib/rattler/compiler/parser_generator/sequence_generator.rb +57 -0
  114. data/lib/rattler/{back_end → compiler}/parser_generator/skip_generator.rb +4 -18
  115. data/lib/rattler/compiler/parser_generator/skip_propogating.rb +16 -0
  116. data/lib/rattler/{back_end → compiler}/parser_generator/sub_generating.rb +19 -11
  117. data/lib/rattler/compiler/parser_generator/super_generator.rb +54 -0
  118. data/lib/rattler/{back_end → compiler}/parser_generator/token_generator.rb +3 -3
  119. data/lib/rattler/compiler/parser_generator/token_propogating.rb +10 -0
  120. data/lib/rattler/{back_end → compiler}/parser_generator/top_level.rb +2 -2
  121. data/lib/rattler/{back_end → compiler}/parser_generator/zero_or_more_generating.rb +5 -5
  122. data/lib/rattler/compiler/rattler.rtlr +201 -0
  123. data/lib/rattler/{back_end → compiler}/ruby_generator.rb +16 -25
  124. data/lib/rattler/parsers.rb +12 -33
  125. data/lib/rattler/parsers/action_code.rb +25 -16
  126. data/lib/rattler/{grammar → parsers}/analysis.rb +32 -11
  127. data/lib/rattler/parsers/apply.rb +10 -19
  128. data/lib/rattler/parsers/assert.rb +4 -14
  129. data/lib/rattler/parsers/atomic.rb +3 -10
  130. data/lib/rattler/parsers/attributed_sequence.rb +50 -0
  131. data/lib/rattler/parsers/back_reference.rb +19 -14
  132. data/lib/rattler/parsers/choice.rb +11 -12
  133. data/lib/rattler/parsers/combinator_parser.rb +15 -7
  134. data/lib/rattler/parsers/combining.rb +15 -9
  135. data/lib/rattler/parsers/disallow.rb +5 -12
  136. data/lib/rattler/parsers/e_symbol.rb +5 -14
  137. data/lib/rattler/parsers/eof.rb +10 -15
  138. data/lib/rattler/parsers/fail.rb +16 -26
  139. data/lib/rattler/{grammar → parsers}/grammar.rb +15 -20
  140. data/lib/rattler/parsers/label.rb +10 -16
  141. data/lib/rattler/parsers/list_parser.rb +14 -14
  142. data/lib/rattler/parsers/match.rb +5 -17
  143. data/lib/rattler/parsers/node_action.rb +72 -0
  144. data/lib/rattler/parsers/node_code.rb +47 -30
  145. data/lib/rattler/parsers/parser.rb +63 -32
  146. data/lib/rattler/parsers/parser_scope.rb +88 -0
  147. data/lib/rattler/parsers/predicate.rb +12 -10
  148. data/lib/rattler/parsers/repeat.rb +15 -8
  149. data/lib/rattler/parsers/rule.rb +8 -23
  150. data/lib/rattler/parsers/rule_set.rb +67 -12
  151. data/lib/rattler/parsers/semantic.rb +36 -0
  152. data/lib/rattler/parsers/semantic_action.rb +39 -0
  153. data/lib/rattler/parsers/sequence.rb +25 -40
  154. data/lib/rattler/parsers/sequencing.rb +40 -0
  155. data/lib/rattler/parsers/skip.rb +11 -12
  156. data/lib/rattler/parsers/super.rb +33 -0
  157. data/lib/rattler/parsers/token.rb +3 -13
  158. data/lib/rattler/rake_task.rb +50 -0
  159. data/lib/rattler/runner.rb +19 -22
  160. data/lib/rattler/runtime.rb +0 -10
  161. data/lib/rattler/runtime/extended_packrat_parser.rb +40 -45
  162. data/lib/rattler/runtime/packrat_parser.rb +17 -31
  163. data/lib/rattler/runtime/parse_failure.rb +16 -26
  164. data/lib/rattler/runtime/parse_node.rb +8 -18
  165. data/lib/rattler/runtime/parser.rb +6 -18
  166. data/lib/rattler/runtime/parser_helper.rb +3 -10
  167. data/lib/rattler/runtime/recursive_descent_parser.rb +26 -23
  168. data/lib/rattler/runtime/syntax_error.rb +0 -10
  169. data/lib/rattler/util.rb +2 -6
  170. data/lib/rattler/util/grammar_cli.rb +19 -0
  171. data/lib/rattler/util/graphviz.rb +6 -17
  172. data/lib/rattler/util/graphviz/digraph_builder.rb +10 -17
  173. data/lib/rattler/util/graphviz/node_builder.rb +45 -31
  174. data/lib/rattler/util/line_counter.rb +11 -20
  175. data/lib/rattler/util/node.rb +52 -30
  176. data/lib/rattler/util/parser_cli.rb +84 -0
  177. data/lib/rattler/util/parser_spec_helper.rb +8 -12
  178. data/spec/rattler/compiler/assert_compiler_examples.rb +284 -0
  179. data/spec/rattler/compiler/attributed_sequence_compiler_examples.rb +154 -0
  180. data/spec/rattler/compiler/disallow_compiler_examples.rb +293 -0
  181. data/spec/rattler/compiler/grammar_parser_spec.rb +700 -0
  182. data/spec/rattler/{back_end → compiler}/optimizer/flatten_choice_spec.rb +1 -1
  183. data/spec/rattler/{back_end → compiler}/optimizer/flatten_sequence_spec.rb +1 -1
  184. data/spec/rattler/compiler/optimizer/inline_regular_rules_spec.rb +50 -0
  185. data/spec/rattler/{back_end → compiler}/optimizer/join_match_capturing_sequence_spec.rb +106 -22
  186. data/spec/rattler/{back_end → compiler}/optimizer/join_match_choice_spec.rb +1 -1
  187. data/spec/rattler/{back_end → compiler}/optimizer/join_match_matching_sequence_spec.rb +1 -1
  188. data/spec/rattler/{back_end → compiler}/optimizer/join_predicate_bare_match_spec.rb +1 -1
  189. data/spec/rattler/{back_end → compiler}/optimizer/join_predicate_nested_match_spec.rb +1 -1
  190. data/spec/rattler/{back_end → compiler}/optimizer/join_predicate_or_bare_match_spec.rb +1 -1
  191. data/spec/rattler/{back_end → compiler}/optimizer/join_predicate_or_nested_match_spec.rb +1 -1
  192. data/spec/rattler/{back_end → compiler}/optimizer/reduce_repeat_match_spec.rb +1 -1
  193. data/spec/rattler/compiler/optimizer/remove_meaningless_wrapper_spec.rb +82 -0
  194. data/spec/rattler/{back_end → compiler}/optimizer/simplify_redundant_repeat_spec.rb +1 -1
  195. data/spec/rattler/{back_end → compiler}/optimizer/simplify_token_match_spec.rb +1 -1
  196. data/spec/rattler/{back_end → compiler}/optimizer_spec.rb +1 -1
  197. data/spec/rattler/{back_end → compiler}/parser_generator/apply_generator_spec.rb +1 -39
  198. data/spec/rattler/{back_end → compiler}/parser_generator/assert_generator_spec.rb +1 -55
  199. data/spec/rattler/compiler/parser_generator/attributed_sequence_generator_spec.rb +699 -0
  200. data/spec/rattler/{back_end → compiler}/parser_generator/back_reference_generator_spec.rb +3 -56
  201. data/spec/rattler/{back_end → compiler}/parser_generator/choice_generator_spec.rb +1 -63
  202. data/spec/rattler/{back_end → compiler}/parser_generator/disallow_generator_spec.rb +1 -55
  203. data/spec/rattler/{back_end → compiler}/parser_generator/e_symbol_generator_spec.rb +1 -39
  204. data/spec/rattler/{back_end → compiler}/parser_generator/eof_generator_spec.rb +1 -39
  205. data/spec/rattler/{back_end → compiler}/parser_generator/fail_generator_spec.rb +94 -23
  206. data/spec/rattler/compiler/parser_generator/grammar_generator_spec.rb +98 -0
  207. data/spec/rattler/compiler/parser_generator/group_match_generator_spec.rb +67 -0
  208. data/spec/rattler/{back_end → compiler}/parser_generator/group_match_spec.rb +1 -1
  209. data/spec/rattler/{back_end → compiler}/parser_generator/label_generator_spec.rb +1 -55
  210. data/spec/rattler/{back_end → compiler}/parser_generator/list0_generator_examples.rb +0 -88
  211. data/spec/rattler/{back_end → compiler}/parser_generator/list1_generator_examples.rb +0 -88
  212. data/spec/rattler/{back_end → compiler}/parser_generator/list_generator_spec.rb +1 -227
  213. data/spec/rattler/{back_end → compiler}/parser_generator/match_generator_spec.rb +1 -55
  214. data/spec/rattler/compiler/parser_generator/node_action_generator_spec.rb +135 -0
  215. data/spec/rattler/{back_end → compiler}/parser_generator/one_or_more_generator_examples.rb +0 -74
  216. data/spec/rattler/{back_end → compiler}/parser_generator/optional_generator_examples.rb +0 -62
  217. data/spec/rattler/{back_end → compiler}/parser_generator/repeat_generator_spec.rb +66 -1
  218. data/spec/rattler/{back_end → compiler}/parser_generator/rule_generator_spec.rb +3 -2
  219. data/spec/rattler/{back_end → compiler}/parser_generator/rule_set_generator_spec.rb +9 -27
  220. data/spec/rattler/compiler/parser_generator/semantic_action_generator_spec.rb +437 -0
  221. data/spec/rattler/{back_end → compiler}/parser_generator/sequence_generator_spec.rb +234 -68
  222. data/spec/rattler/{back_end → compiler}/parser_generator/skip_generator_spec.rb +1 -55
  223. data/spec/rattler/compiler/parser_generator/super_generator_spec.rb +93 -0
  224. data/spec/rattler/{back_end → compiler}/parser_generator/token_generator_spec.rb +1 -55
  225. data/spec/rattler/{back_end → compiler}/parser_generator/zero_or_more_generator_examples.rb +0 -74
  226. data/spec/rattler/{back_end → compiler}/ruby_generator_spec.rb +13 -13
  227. data/spec/rattler/compiler/semantic_action_compiler_examples.rb +57 -0
  228. data/spec/rattler/{back_end → compiler}/shared_compiler_examples.rb +111 -140
  229. data/spec/rattler/{back_end → compiler}/skip_compiler_examples.rb +60 -57
  230. data/spec/rattler/{back_end → compiler}/token_compiler_examples.rb +99 -104
  231. data/spec/rattler/compiler_spec.rb +67 -0
  232. data/spec/rattler/parsers/action_code_spec.rb +34 -18
  233. data/spec/rattler/{grammar → parsers}/analysis_spec.rb +13 -67
  234. data/spec/rattler/parsers/apply_spec.rb +6 -0
  235. data/spec/rattler/parsers/assert_spec.rb +38 -2
  236. data/spec/rattler/parsers/attributed_sequence_spec.rb +204 -0
  237. data/spec/rattler/parsers/back_reference_spec.rb +6 -0
  238. data/spec/rattler/parsers/choice_spec.rb +38 -1
  239. data/spec/rattler/parsers/combinator_parser_spec.rb +2 -1
  240. data/spec/rattler/parsers/disallow_spec.rb +38 -2
  241. data/spec/rattler/parsers/e_symbol_spec.rb +6 -0
  242. data/spec/rattler/parsers/eof_spec.rb +6 -0
  243. data/spec/rattler/parsers/fail_spec.rb +6 -0
  244. data/spec/rattler/{grammar → parsers}/grammar_spec.rb +10 -15
  245. data/spec/rattler/parsers/label_spec.rb +30 -0
  246. data/spec/rattler/parsers/list_parser_spec.rb +31 -2
  247. data/spec/rattler/parsers/match_spec.rb +6 -0
  248. data/spec/rattler/parsers/node_action_spec.rb +121 -0
  249. data/spec/rattler/parsers/parser_scope_spec.rb +105 -0
  250. data/spec/rattler/parsers/repeat_spec.rb +56 -0
  251. data/spec/rattler/parsers/rule_set_spec.rb +42 -0
  252. data/spec/rattler/parsers/semantic_action_spec.rb +89 -0
  253. data/spec/rattler/parsers/sequence_spec.rb +156 -12
  254. data/spec/rattler/parsers/skip_spec.rb +21 -0
  255. data/spec/rattler/parsers/super_spec.rb +45 -0
  256. data/spec/rattler/parsers/token_spec.rb +33 -14
  257. data/spec/rattler/runtime/extended_packrat_parser_spec.rb +10 -8
  258. data/spec/rattler/runtime/recursive_descent_parser_spec.rb +26 -0
  259. data/spec/rattler/runtime/shared_parser_examples.rb +22 -16
  260. data/spec/rattler/util/graphviz/node_builder_spec.rb +33 -17
  261. data/spec/rattler/util/line_counter_spec.rb +21 -21
  262. data/spec/rattler/util/node_spec.rb +62 -0
  263. data/spec/rattler_spec.rb +7 -41
  264. data/spec/spec_helper.rb +1 -2
  265. data/spec/support/combinator_parser_spec_helper.rb +1 -1
  266. data/spec/support/compiler_spec_helper.rb +0 -4
  267. data/spec/support/parser_generator_spec_helper.rb +7 -7
  268. data/spec/support/runtime_parser_spec_helper.rb +57 -3
  269. metadata +447 -303
  270. data/features/grammar/character_class.feature +0 -20
  271. data/features/grammar/nonterminal.feature +0 -24
  272. data/features/grammar/one_or_more.feature +0 -34
  273. data/features/grammar/ordered_choice.feature +0 -21
  274. data/features/grammar/posix_class.feature +0 -70
  275. data/features/grammar/sequence.feature +0 -20
  276. data/features/grammar/zero_or_more.feature +0 -34
  277. data/lib/rattler/back_end.rb +0 -22
  278. data/lib/rattler/back_end/compiler.rb +0 -128
  279. data/lib/rattler/back_end/optimizer.rb +0 -101
  280. data/lib/rattler/back_end/optimizer/inline_regular_rules.rb +0 -46
  281. data/lib/rattler/back_end/optimizer/join_match_sequence.rb +0 -17
  282. data/lib/rattler/back_end/optimizer/join_predicate_match.rb +0 -17
  283. data/lib/rattler/back_end/optimizer/join_predicate_or_match.rb +0 -17
  284. data/lib/rattler/back_end/optimizer/optimization_context.rb +0 -72
  285. data/lib/rattler/back_end/optimizer/remove_meaningless_wrapper.rb +0 -32
  286. data/lib/rattler/back_end/optimizer/specialize_repeat.rb +0 -40
  287. data/lib/rattler/back_end/parser_generator.rb +0 -113
  288. data/lib/rattler/back_end/parser_generator/direct_action_generator.rb +0 -45
  289. data/lib/rattler/back_end/parser_generator/dispatch_action_generator.rb +0 -45
  290. data/lib/rattler/back_end/parser_generator/group_match.rb +0 -26
  291. data/lib/rattler/back_end/parser_generator/label_generator.rb +0 -64
  292. data/lib/rattler/back_end/parser_generator/predicate_propogating.rb +0 -24
  293. data/lib/rattler/back_end/parser_generator/rule_generator.rb +0 -53
  294. data/lib/rattler/back_end/parser_generator/sequence_generator.rb +0 -190
  295. data/lib/rattler/back_end/parser_generator/skip_propogating.rb +0 -16
  296. data/lib/rattler/back_end/parser_generator/token_propogating.rb +0 -10
  297. data/lib/rattler/grammar.rb +0 -43
  298. data/lib/rattler/grammar/grammar_dsl.rb +0 -51
  299. data/lib/rattler/grammar/metagrammar.rb +0 -990
  300. data/lib/rattler/grammar/rattler.rtlr +0 -183
  301. data/lib/rattler/parsers/assert_code.rb +0 -31
  302. data/lib/rattler/parsers/direct_action.rb +0 -85
  303. data/lib/rattler/parsers/disallow_code.rb +0 -31
  304. data/lib/rattler/parsers/dispatch_action.rb +0 -121
  305. data/lib/rattler/parsers/effect_code.rb +0 -31
  306. data/lib/rattler/parsers/parser_dsl.rb +0 -414
  307. data/lib/rattler/parsers/semantic_assert.rb +0 -19
  308. data/lib/rattler/parsers/semantic_disallow.rb +0 -19
  309. data/lib/rattler/parsers/side_effect.rb +0 -19
  310. data/spec/rattler/back_end/assert_compiler_examples.rb +0 -187
  311. data/spec/rattler/back_end/compiler_spec.rb +0 -43
  312. data/spec/rattler/back_end/direct_action_compiler_examples.rb +0 -227
  313. data/spec/rattler/back_end/disallow_compiler_examples.rb +0 -187
  314. data/spec/rattler/back_end/dispatch_action_compiler_examples.rb +0 -225
  315. data/spec/rattler/back_end/optimizer/inline_regular_rules_spec.rb +0 -80
  316. data/spec/rattler/back_end/parser_generator/direct_action_generator_spec.rb +0 -204
  317. data/spec/rattler/back_end/parser_generator/dispatch_action_generator_spec.rb +0 -204
  318. data/spec/rattler/back_end/parser_generator/group_match_generator_spec.rb +0 -185
  319. data/spec/rattler/back_end/semantic_assert_compiler_examples.rb +0 -152
  320. data/spec/rattler/back_end/semantic_disallow_compiler_examples.rb +0 -152
  321. data/spec/rattler/back_end/side_effect_compiler_examples.rb +0 -227
  322. data/spec/rattler/grammar/grammar_parser_spec.rb +0 -626
  323. data/spec/rattler/parsers/direct_action_spec.rb +0 -224
  324. data/spec/rattler/parsers/dispatch_action_spec.rb +0 -209
  325. data/spec/rattler/parsers/node_code_spec.rb +0 -59
  326. data/spec/rattler/parsers/parser_dsl_spec.rb +0 -334
  327. data/spec/rattler/parsers/semantic_assert_spec.rb +0 -83
  328. data/spec/rattler/parsers/semantic_disallow_spec.rb +0 -83
  329. data/spec/rattler/parsers/side_effect_spec.rb +0 -214
@@ -0,0 +1,77 @@
1
+ require 'rattler/compiler'
2
+
3
+ module Rattler::Compiler
4
+
5
+ # The +Optimizer+ transforms parser models into equivalent models that can
6
+ # result in more efficient parsing code. This is primarily achieved by
7
+ # converting regular expressions into equivalent Regexp patterns, thus
8
+ # reducing object instantiation and method dispatch and having StringScanner
9
+ # do as much of the parsing work as possible.
10
+ module Optimizer
11
+
12
+ class << self
13
+
14
+ include Rattler::Parsers
15
+
16
+ # @param [Rattler::Parsers::Grammar, Rattler::Parsers::RuleSet, Rattler::Parsers::Rule, Rattler::Parsers::Parser]
17
+ # model the model to be optimized
18
+ # @param [Hash] opts options for the optimizer
19
+ # @return an optimized parser model
20
+ def optimize(model, opts={})
21
+ case model
22
+ when Grammar then optimize_grammar model, opts
23
+ when RuleSet then optimize_rule_set model, opts
24
+ else optimizations.apply model, default_context(opts)
25
+ end
26
+ end
27
+
28
+ private
29
+
30
+ def default_context(opts)
31
+ OptimizationContext[opts.merge :type => :capturing]
32
+ end
33
+
34
+ def optimize_grammar(grammar, opts)
35
+ grammar.with_rules optimize_rule_set(grammar.rules, opts)
36
+ end
37
+
38
+ def optimize_rule_set(rule_set, opts)
39
+ context = default_context(opts).with(:rules => rule_set)
40
+ rule_set = rule_set.map_rules {|_| optimizations.apply _, context }
41
+ context = context.with(:rules => rule_set)
42
+ rule_set.select_rules {|_| context.relavent? _ }
43
+ end
44
+
45
+ end
46
+
47
+ autoload :OptimizationContext, 'rattler/compiler/optimizer/optimization_context'
48
+ autoload :Optimization, 'rattler/compiler/optimizer/optimization'
49
+ autoload :OptimizationSequence, 'rattler/compiler/optimizer/optimization_sequence'
50
+ autoload :Optimizations, 'rattler/compiler/optimizer/optimizations'
51
+ autoload :OptimizeChildren, 'rattler/compiler/optimizer/optimize_children'
52
+ autoload :InlineRegularRules, 'rattler/compiler/optimizer/inline_regular_rules'
53
+ autoload :SimplifyRedundantRepeat, 'rattler/compiler/optimizer/simplify_redundant_repeat'
54
+ autoload :RemoveMeaninglessWrapper, 'rattler/compiler/optimizer/remove_meaningless_wrapper'
55
+ autoload :SimplifyTokenMatch, 'rattler/compiler/optimizer/simplify_token_match'
56
+ autoload :FlattenSequence, 'rattler/compiler/optimizer/flatten_sequence'
57
+ autoload :FlattenChoice, 'rattler/compiler/optimizer/flatten_choice'
58
+ autoload :ReduceRepeatMatch, 'rattler/compiler/optimizer/reduce_repeat_match'
59
+ autoload :JoinPredicateMatch, 'rattler/compiler/optimizer/join_predicate_match'
60
+ autoload :JoinPredicateBareMatch, 'rattler/compiler/optimizer/join_predicate_bare_match'
61
+ autoload :JoinPredicateNestedMatch, 'rattler/compiler/optimizer/join_predicate_nested_match'
62
+ autoload :JoinPredicateOrMatch, 'rattler/compiler/optimizer/join_predicate_or_match'
63
+ autoload :JoinPredicateOrBareMatch, 'rattler/compiler/optimizer/join_predicate_or_bare_match'
64
+ autoload :JoinPredicateOrNestedMatch, 'rattler/compiler/optimizer/join_predicate_or_nested_match'
65
+ autoload :JoinMatchSequence, 'rattler/compiler/optimizer/join_match_sequence'
66
+ autoload :JoinMatchCapturingSequence, 'rattler/compiler/optimizer/join_match_capturing_sequence'
67
+ autoload :JoinMatchMatchingSequence, 'rattler/compiler/optimizer/join_match_matching_sequence'
68
+ autoload :JoinMatchChoice, 'rattler/compiler/optimizer/join_match_choice'
69
+ autoload :MatchJoining, 'rattler/compiler/optimizer/match_joining'
70
+ autoload :Flattening, 'rattler/compiler/optimizer/flattening'
71
+ autoload :CompositeReducing, 'rattler/compiler/optimizer/composite_reducing'
72
+
73
+ end
74
+
75
+ Optimizer.extend Optimizer::Optimizations
76
+
77
+ end
@@ -1,6 +1,6 @@
1
- require 'rattler'
1
+ require 'rattler/compiler/optimizer'
2
2
 
3
- module Rattler::BackEnd::Optimizer
3
+ module Rattler::Compiler::Optimizer
4
4
  # @private
5
5
  module CompositeReducing #:nodoc:
6
6
 
@@ -1,17 +1,8 @@
1
- #
2
- # = rattler/back_end/optimizer/flatten_choice.rb
3
- #
4
- # Author:: Jason Arhart
5
- # Documentation:: Author
6
- #
7
- require 'rattler'
1
+ require 'rattler/compiler/optimizer'
2
+
3
+ module Rattler::Compiler::Optimizer
8
4
 
9
- module Rattler::BackEnd::Optimizer
10
- #
11
5
  # Nested choice expressions can be flattened without affecting how they parse.
12
- #
13
- # @author Jason Arhart
14
- #
15
6
  class FlattenChoice < Optimization
16
7
  include Flattening
17
8
 
@@ -1,19 +1,9 @@
1
- #
2
- # = rattler/back_end/optimizer/flatten_sequence.rb
3
- #
4
- # Author:: Jason Arhart
5
- # Documentation:: Author
6
- #
7
- require 'rattler'
1
+ require 'rattler/compiler/optimizer'
8
2
 
9
- module Rattler::BackEnd::Optimizer
3
+ module Rattler::Compiler::Optimizer
10
4
 
11
- #
12
5
  # Nested sequence expressions can be flattened without affecting how they
13
6
  # match.
14
- #
15
- # @author Jason Arhart
16
- #
17
7
  class FlattenMatchingSequence < Optimization
18
8
  include Flattening
19
9
  include Rattler::Parsers
@@ -31,12 +21,8 @@ module Rattler::BackEnd::Optimizer
31
21
  end
32
22
  end
33
23
 
34
- #
35
24
  # Nested sequence expressions can be flattened without affecting how they
36
25
  # parse if the nested sequence expressions are not multi-capturing.
37
- #
38
- # @author Jason Arhart
39
- #
40
26
  class FlattenCapturingSequence < Optimization
41
27
  include Flattening
42
28
  include Rattler::Parsers
@@ -54,6 +40,8 @@ module Rattler::BackEnd::Optimizer
54
40
  end
55
41
  end
56
42
 
43
+ # Nested sequence expressions can be flattened without affecting how they
44
+ # parse given certain conditions
57
45
  FlattenSequence = FlattenMatchingSequence >> FlattenCapturingSequence
58
46
 
59
47
  end
@@ -1,6 +1,6 @@
1
- require 'rattler'
1
+ require 'rattler/compiler/optimizer'
2
2
 
3
- module Rattler::BackEnd::Optimizer
3
+ module Rattler::Compiler::Optimizer
4
4
  # @private
5
5
  module Flattening #:nodoc:
6
6
 
@@ -0,0 +1,24 @@
1
+ require 'rattler/compiler/optimizer'
2
+
3
+ module Rattler::Compiler::Optimizer
4
+
5
+ # References to regular parse rules can be inlined without affecting how they
6
+ # parse, assuming the referenced rule does not change. This optimization is
7
+ # only applied if the referenced rule is regular and marked for inlining.
8
+ class InlineRegularRules < Optimization
9
+
10
+ include Rattler::Parsers
11
+
12
+ protected
13
+
14
+ def _applies_to?(parser, context)
15
+ parser.is_a? Apply and
16
+ context.inlineable? parser.rule_name
17
+ end
18
+
19
+ def _apply(parser, context)
20
+ context.rules[parser.rule_name].expr
21
+ end
22
+
23
+ end
24
+ end
@@ -1,29 +1,21 @@
1
- #
2
- # = rattler/back_end/optimizer/join_match_capturing_sequence.rb
3
- #
4
- # Author:: Jason Arhart
5
- # Documentation:: Author
6
- #
7
- require 'rattler'
8
-
9
- module Rattler::BackEnd::Optimizer
10
- #
1
+ require 'rattler/compiler/optimizer'
2
+
3
+ module Rattler::Compiler::Optimizer
4
+
11
5
  # Sequences of Regexp matches can be joined into a single Regexp match using
12
6
  # capturing groups if necessary.
13
- #
14
- # @author Jason Arhart
15
- #
16
7
  class JoinMatchCapturingSequence < Optimization
17
8
  include MatchJoining
18
9
 
19
10
  include Rattler::Parsers
20
- include Rattler::BackEnd::ParserGenerator
11
+ include Rattler::Compiler::ParserGenerator
21
12
 
22
13
  protected
23
14
 
24
15
  def _applies_to?(parser, context)
25
16
  context.capturing? and
26
17
  parser.is_a?(Sequence) and
18
+ not disqualifying_captures?(parser) and
27
19
  any_neighbors?(parser) {|_| eligible_child? _ }
28
20
  end
29
21
 
@@ -33,6 +25,16 @@ module Rattler::BackEnd::Optimizer
33
25
  (child.is_a? Skip and child.child.is_a? Match)
34
26
  end
35
27
 
28
+ def disqualifying_captures?(parser)
29
+ parser.any? {|_| _.capturing? and eligible_child? _ } and
30
+ parser.any? {|_| capture_incompatible? _ }
31
+ end
32
+
33
+ def capture_incompatible?(child)
34
+ (child.capturing? and not eligible_child? child) or
35
+ child.semantic?
36
+ end
37
+
36
38
  def create_patterns(parsers)
37
39
  num_groups = 0
38
40
  patterns = parsers.map do |parser|
@@ -1,18 +1,9 @@
1
- #
2
- # = rattler/back_end/optimizer/join_match_choice.rb
3
- #
4
- # Author:: Jason Arhart
5
- # Documentation:: Author
6
- #
7
- require 'rattler'
8
-
9
- module Rattler::BackEnd::Optimizer
10
- #
1
+ require 'rattler/compiler/optimizer'
2
+
3
+ module Rattler::Compiler::Optimizer
4
+
11
5
  # A choice of Regexp matches can be joined into a single Regexp match without
12
6
  # affecting how it parses.
13
- #
14
- # @author Jason Arhart
15
- #
16
7
  class JoinMatchChoice < Optimization
17
8
  include MatchJoining
18
9
 
@@ -1,18 +1,9 @@
1
- #
2
- # = rattler/back_end/optimizer/join_match_matching_sequence.rb
3
- #
4
- # Author:: Jason Arhart
5
- # Documentation:: Author
6
- #
7
- require 'rattler'
8
-
9
- module Rattler::BackEnd::Optimizer
10
- #
1
+ require 'rattler/compiler/optimizer'
2
+
3
+ module Rattler::Compiler::Optimizer
4
+
11
5
  # Sequences of Regexp matches can be joined into a single Regexp without
12
6
  # affecting how they match.
13
- #
14
- # @author Jason Arhart
15
- #
16
7
  class JoinMatchMatchingSequence < Optimization
17
8
  include MatchJoining
18
9
 
@@ -0,0 +1,7 @@
1
+ require 'rattler/compiler/optimizer'
2
+
3
+ module Rattler::Compiler::Optimizer
4
+ # Sequences of Regexp matches can be joined into a single Regexp match using
5
+ # capturing groups if necessary.
6
+ JoinMatchSequence = JoinMatchMatchingSequence >> JoinMatchCapturingSequence
7
+ end
@@ -1,18 +1,9 @@
1
- #
2
- # = rattler/back_end/optimizer/join_predicate_bare_match.rb
3
- #
4
- # Author:: Jason Arhart
5
- # Documentation:: Author
6
- #
7
- require 'rattler'
1
+ require 'rattler/compiler/optimizer'
2
+
3
+ module Rattler::Compiler::Optimizer
8
4
 
9
- module Rattler::BackEnd::Optimizer
10
- #
11
5
  # A predicate and an adjacent Regexp match in a Sequence can be joined into a
12
6
  # single Regexp match.
13
- #
14
- # @author Jason Arhart
15
- #
16
7
  class JoinPredicateBareMatch < Optimization
17
8
  include CompositeReducing
18
9
 
@@ -0,0 +1,7 @@
1
+ require 'rattler/compiler/optimizer'
2
+
3
+ module Rattler::Compiler::Optimizer
4
+ # A predicate and an adjacent Regexp match in a Sequence can be joined into a
5
+ # single Regexp match.
6
+ JoinPredicateMatch = JoinPredicateBareMatch >> JoinPredicateNestedMatch
7
+ end
@@ -1,18 +1,9 @@
1
- #
2
- # = rattler/back_end/optimizer/join_predicate_nested_match.rb
3
- #
4
- # Author:: Jason Arhart
5
- # Documentation:: Author
6
- #
7
- require 'rattler'
8
-
9
- module Rattler::BackEnd::Optimizer
10
- #
1
+ require 'rattler/compiler/optimizer'
2
+
3
+ module Rattler::Compiler::Optimizer
4
+
11
5
  # A predicate and an adjacent skip of a Regexp match in a Sequence can be
12
6
  # joined into a single skip of a Regexp match.
13
- #
14
- # @author Jason Arhart
15
- #
16
7
  class JoinPredicateNestedMatch < JoinPredicateBareMatch
17
8
  include CompositeReducing
18
9
 
@@ -1,18 +1,9 @@
1
- #
2
- # = rattler/back_end/optimizer/join_predicate_or_bare_match.rb
3
- #
4
- # Author:: Jason Arhart
5
- # Documentation:: Author
6
- #
7
- require 'rattler'
1
+ require 'rattler/compiler/optimizer'
2
+
3
+ module Rattler::Compiler::Optimizer
8
4
 
9
- module Rattler::BackEnd::Optimizer
10
- #
11
5
  # A predicate and an adjacent Regexp match in a Choice can be joined into a
12
6
  # single Regexp match.
13
- #
14
- # @author Jason Arhart
15
- #
16
7
  class JoinPredicateOrBareMatch < Optimization
17
8
  include CompositeReducing
18
9
 
@@ -0,0 +1,7 @@
1
+ require 'rattler/compiler/optimizer'
2
+
3
+ module Rattler::Compiler::Optimizer
4
+ # A predicate and an adjacent Regexp match in a Choice can be joined into a
5
+ # single Regexp match.
6
+ JoinPredicateOrMatch = JoinPredicateOrBareMatch >> JoinPredicateOrNestedMatch
7
+ end
@@ -1,18 +1,9 @@
1
- #
2
- # = rattler/back_end/optimizer/join_predicate_or_nested_match.rb
3
- #
4
- # Author:: Jason Arhart
5
- # Documentation:: Author
6
- #
7
- require 'rattler'
8
-
9
- module Rattler::BackEnd::Optimizer
10
- #
1
+ require 'rattler/compiler/optimizer'
2
+
3
+ module Rattler::Compiler::Optimizer
4
+
11
5
  # A predicate and an adjacent skip of a Regexp match in a Choice can be
12
6
  # joined into a single skip of a Regexp match.
13
- #
14
- # @author Jason Arhart
15
- #
16
7
  class JoinPredicateOrNestedMatch < JoinPredicateOrBareMatch
17
8
  include CompositeReducing
18
9
 
@@ -1,6 +1,6 @@
1
- require 'rattler'
1
+ require 'rattler/compiler/optimizer'
2
2
 
3
- module Rattler::BackEnd::Optimizer
3
+ module Rattler::Compiler::Optimizer
4
4
  # @private
5
5
  module MatchJoining #:nodoc:
6
6
 
@@ -1,39 +1,30 @@
1
- #
2
- # = rattler/back_end/optimizer/optimization.rb
3
- #
4
- # Author:: Jason Arhart
5
- # Documentation:: Author
6
- #
7
- require 'rattler'
8
- require 'singleton'
1
+ require 'rattler/compiler/optimizer'
2
+
3
+ module Rattler::Compiler::Optimizer
9
4
 
10
- module Rattler::BackEnd::Optimizer
11
- #
12
5
  # An +Optimization+ represents a simple transformation of a parser model into
13
6
  # an equivalent model that can result in more efficient parsing code.
14
7
  # Subclasses override <tt>#_applies_to?</tt> and <tt>#_apply</tt> to define
15
8
  # an optimization.
16
- #
17
- # @author Jason Arhart
18
- #
19
9
  class Optimization
20
10
 
21
11
  class <<self
12
+ # @return a lazy singleton instance
22
13
  def instance
23
14
  @instance ||= self.new
24
15
  end
25
16
 
26
- # @see Rattler::BackEnd::Optimizer::Optimization#>>
17
+ # (see Rattler::Compiler::Optimizer::Optimization#>>)
27
18
  def >>(other)
28
19
  instance >> (other.respond_to?(:instance) ? other.instance : other)
29
20
  end
30
21
 
31
- # @see Rattler::BackEnd::Optimizer::Optimization#match
22
+ # (see Rattler::Compiler::Optimizer::Optimization#apply)
32
23
  def apply(*args)
33
24
  instance.apply(*args)
34
25
  end
35
26
 
36
- # @see Rattler::BackEnd::Optimizer::Optimization#applies_to?
27
+ # (see Rattler::Compiler::Optimizer::Optimization#applies_to?)
37
28
  def applies_to?(*args)
38
29
  instance.applies_to?(*args)
39
30
  end
@@ -47,10 +38,10 @@ module Rattler::BackEnd::Optimizer
47
38
  # i.e. the new optimzation applies this optimzation, then applies +other+
48
39
  # to the result.
49
40
  #
50
- # @param [Rattler::BackEnd::Optimizer::Optimization] other the optimization
41
+ # @param [Rattler::Compiler::Optimizer::Optimization] other the optimization
51
42
  # to apply after this one
52
43
  #
53
- # @return [Rattler::BackEnd::Optimizer::Optimization] a new optimzation
44
+ # @return [Rattler::Compiler::Optimizer::Optimization] a new optimzation
54
45
  # that sequences this optimzation and +other+
55
46
  def >>(other)
56
47
  OptimizationSequence.new(self, other)
@@ -60,35 +51,22 @@ module Rattler::BackEnd::Optimizer
60
51
  # <tt>#applies_to?(parser, context)</tt> is +true+.
61
52
  #
62
53
  # @param [Rattler::Parsers::Parser] parser the parser to be optimized
63
- # @param [Rattler::BackEnd::Optimizer::OptimizationContext] context
54
+ # @param [Rattler::Compiler::Optimizer::OptimizationContext] context
64
55
  #
65
56
  # @return [Rattler::Parsers::Parser] the optimized parser
66
57
  def apply(parser, context)
67
58
  applies_to?(parser, context) ? _apply(parser, context) : parser
68
59
  end
69
60
 
70
- # Return +true+ if this optimzation applies to +parser+ in +context+.
71
- #
72
61
  # @param [Rattler::Parsers::Parser] parser a parser model
73
- # @param [Rattler::BackEnd::Optimizer::OptimizationContext] context
62
+ # @param [Rattler::Compiler::Optimizer::OptimizationContext] context
74
63
  #
75
- # @return [Boolean] +true+ if this optimzation applies to +parser+ in
76
- # +context+
64
+ # @return +true+ if this optimzation applies to +parser+ in +context+
77
65
  def applies_to?(parser, context)
78
66
  @applies_to_cache[context].fetch(parser) do
79
67
  @applies_to_cache[context][parser] = _applies_to?(parser, context)
80
68
  end
81
69
  end
82
70
 
83
- private
84
-
85
- def optimize(parser, context)
86
- optimizations.apply(parser, context)
87
- end
88
-
89
- def optimizations
90
- ::Rattler::BackEnd::Optimizer.optimizations
91
- end
92
-
93
71
  end
94
72
  end