rattler 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. data/README.rdoc +57 -37
  2. data/features/command_line/dest_option.feature +8 -21
  3. data/features/command_line/lib_option.feature +37 -0
  4. data/features/command_line/parser_generator.feature +7 -4
  5. data/features/grammar/back_reference.feature +37 -0
  6. data/features/grammar/fail.feature +3 -3
  7. data/features/grammar/labels.feature +11 -3
  8. data/features/grammar/list_matching.feature +14 -5
  9. data/features/grammar/literal.feature +30 -4
  10. data/features/grammar/nonterminal.feature +1 -1
  11. data/features/grammar/ordered_choice.feature +2 -2
  12. data/features/grammar/skip_operator.feature +1 -1
  13. data/features/grammar/symantic_action.feature +7 -7
  14. data/features/grammar/whitespace.feature +2 -2
  15. data/features/step_definitions/grammar_steps.rb +2 -2
  16. data/lib/rattler/back_end.rb +1 -0
  17. data/lib/rattler/back_end/compiler.rb +19 -20
  18. data/lib/rattler/back_end/optimizer.rb +100 -0
  19. data/lib/rattler/back_end/optimizer/composite_reducing.rb +18 -0
  20. data/lib/rattler/back_end/optimizer/flatten_choice.rb +31 -0
  21. data/lib/rattler/back_end/optimizer/flatten_sequence.rb +59 -0
  22. data/lib/rattler/back_end/optimizer/flattening.rb +17 -0
  23. data/lib/rattler/back_end/optimizer/inline_regular_rules.rb +46 -0
  24. data/lib/rattler/back_end/optimizer/join_match_capturing_sequence.rb +71 -0
  25. data/lib/rattler/back_end/optimizer/join_match_choice.rb +37 -0
  26. data/lib/rattler/back_end/optimizer/join_match_matching_sequence.rb +38 -0
  27. data/lib/rattler/back_end/optimizer/join_match_sequence.rb +17 -0
  28. data/lib/rattler/back_end/optimizer/join_predicate_bare_match.rb +68 -0
  29. data/lib/rattler/back_end/optimizer/join_predicate_match.rb +17 -0
  30. data/lib/rattler/back_end/optimizer/join_predicate_nested_match.rb +37 -0
  31. data/lib/rattler/back_end/optimizer/join_predicate_or_bare_match.rb +68 -0
  32. data/lib/rattler/back_end/optimizer/join_predicate_or_match.rb +17 -0
  33. data/lib/rattler/back_end/optimizer/join_predicate_or_nested_match.rb +36 -0
  34. data/lib/rattler/back_end/optimizer/match_joining.rb +60 -0
  35. data/lib/rattler/back_end/optimizer/optimization.rb +94 -0
  36. data/lib/rattler/back_end/optimizer/optimization_context.rb +72 -0
  37. data/lib/rattler/back_end/optimizer/optimization_sequence.rb +37 -0
  38. data/lib/rattler/back_end/optimizer/optimize_children.rb +46 -0
  39. data/lib/rattler/back_end/optimizer/reduce_repeat_match.rb +44 -0
  40. data/lib/rattler/back_end/optimizer/remove_meaningless_wrapper.rb +32 -0
  41. data/lib/rattler/back_end/optimizer/simplify_redundant_repeat.rb +43 -0
  42. data/lib/rattler/back_end/optimizer/simplify_token_match.rb +38 -0
  43. data/lib/rattler/back_end/parser_generator.rb +21 -14
  44. data/lib/rattler/back_end/parser_generator/apply_generator.rb +35 -35
  45. data/lib/rattler/back_end/parser_generator/assert_generator.rb +29 -30
  46. data/lib/rattler/back_end/parser_generator/back_reference_generator.rb +93 -0
  47. data/lib/rattler/back_end/parser_generator/choice_generator.rb +33 -49
  48. data/lib/rattler/back_end/parser_generator/direct_action_generator.rb +14 -14
  49. data/lib/rattler/back_end/parser_generator/disallow_generator.rb +29 -30
  50. data/lib/rattler/back_end/parser_generator/dispatch_action_generator.rb +11 -13
  51. data/lib/rattler/back_end/parser_generator/expr_generator.rb +36 -56
  52. data/lib/rattler/back_end/parser_generator/fail_generator.rb +18 -18
  53. data/lib/rattler/back_end/parser_generator/group_match.rb +18 -0
  54. data/lib/rattler/back_end/parser_generator/group_match_generator.rb +76 -0
  55. data/lib/rattler/back_end/parser_generator/label_generator.rb +25 -6
  56. data/lib/rattler/back_end/parser_generator/list1_generator.rb +7 -7
  57. data/lib/rattler/back_end/parser_generator/list_generating.rb +19 -20
  58. data/lib/rattler/back_end/parser_generator/list_generator.rb +5 -5
  59. data/lib/rattler/back_end/parser_generator/match_generator.rb +52 -52
  60. data/lib/rattler/back_end/parser_generator/one_or_more_generator.rb +6 -6
  61. data/lib/rattler/back_end/parser_generator/optional_generator.rb +30 -29
  62. data/lib/rattler/back_end/parser_generator/predicate_propogating.rb +8 -8
  63. data/lib/rattler/back_end/parser_generator/repeat_generating.rb +23 -25
  64. data/lib/rattler/back_end/parser_generator/rule_generator.rb +27 -79
  65. data/lib/rattler/back_end/parser_generator/rule_set_generator.rb +102 -0
  66. data/lib/rattler/back_end/parser_generator/sequence_generator.rb +49 -41
  67. data/lib/rattler/back_end/parser_generator/skip_generator.rb +14 -20
  68. data/lib/rattler/back_end/parser_generator/skip_propogating.rb +4 -4
  69. data/lib/rattler/back_end/parser_generator/sub_generating.rb +6 -0
  70. data/lib/rattler/back_end/parser_generator/token_generator.rb +12 -12
  71. data/lib/rattler/back_end/parser_generator/token_propogating.rb +2 -2
  72. data/lib/rattler/back_end/parser_generator/zero_or_more_generator.rb +4 -4
  73. data/lib/rattler/grammar.rb +4 -3
  74. data/lib/rattler/grammar/analysis.rb +91 -0
  75. data/lib/rattler/grammar/grammar.rb +37 -25
  76. data/lib/rattler/grammar/grammar_parser.rb +19 -11
  77. data/lib/rattler/grammar/metagrammar.rb +569 -800
  78. data/lib/rattler/grammar/rattler.rtlr +162 -144
  79. data/lib/rattler/parsers.rb +5 -1
  80. data/lib/rattler/parsers/action_code.rb +29 -15
  81. data/lib/rattler/parsers/apply.rb +5 -5
  82. data/lib/rattler/parsers/assert.rb +4 -18
  83. data/lib/rattler/parsers/back_reference.rb +46 -0
  84. data/lib/rattler/parsers/choice.rb +6 -39
  85. data/lib/rattler/parsers/combinator_parser.rb +32 -0
  86. data/lib/rattler/parsers/combining.rb +3 -29
  87. data/lib/rattler/parsers/direct_action.rb +27 -30
  88. data/lib/rattler/parsers/disallow.rb +4 -18
  89. data/lib/rattler/parsers/dispatch_action.rb +30 -25
  90. data/lib/rattler/parsers/label.rb +9 -18
  91. data/lib/rattler/parsers/list.rb +3 -34
  92. data/lib/rattler/parsers/list1.rb +4 -36
  93. data/lib/rattler/parsers/list_parser.rb +64 -0
  94. data/lib/rattler/parsers/match.rb +7 -42
  95. data/lib/rattler/parsers/node_code.rb +44 -0
  96. data/lib/rattler/parsers/one_or_more.rb +7 -27
  97. data/lib/rattler/parsers/optional.rb +5 -25
  98. data/lib/rattler/parsers/parser.rb +16 -44
  99. data/lib/rattler/parsers/parser_dsl.rb +13 -3
  100. data/lib/rattler/parsers/predicate.rb +4 -12
  101. data/lib/rattler/parsers/rule.rb +18 -19
  102. data/lib/rattler/parsers/rule_set.rb +63 -0
  103. data/lib/rattler/parsers/sequence.rb +12 -46
  104. data/lib/rattler/parsers/skip.rb +12 -26
  105. data/lib/rattler/parsers/token.rb +6 -21
  106. data/lib/rattler/parsers/zero_or_more.rb +6 -26
  107. data/lib/rattler/runner.rb +66 -28
  108. data/lib/rattler/runtime/extended_packrat_parser.rb +26 -20
  109. data/lib/rattler/runtime/packrat_parser.rb +17 -21
  110. data/lib/rattler/runtime/parser.rb +12 -2
  111. data/lib/rattler/runtime/recursive_descent_parser.rb +3 -11
  112. data/lib/rattler/util.rb +2 -1
  113. data/lib/rattler/util/graphviz.rb +29 -0
  114. data/lib/rattler/util/graphviz/digraph_builder.rb +71 -0
  115. data/lib/rattler/util/graphviz/node_builder.rb +84 -0
  116. data/lib/rattler/util/node.rb +37 -19
  117. data/lib/rattler/util/parser_spec_helper.rb +61 -35
  118. data/spec/rattler/back_end/compiler_spec.rb +6 -860
  119. data/spec/rattler/back_end/optimizer/flatten_choice_spec.rb +70 -0
  120. data/spec/rattler/back_end/optimizer/flatten_sequence_spec.rb +130 -0
  121. data/spec/rattler/back_end/optimizer/inline_regular_rules_spec.rb +80 -0
  122. data/spec/rattler/back_end/optimizer/join_match_capturing_sequence_spec.rb +241 -0
  123. data/spec/rattler/back_end/optimizer/join_match_choice_spec.rb +100 -0
  124. data/spec/rattler/back_end/optimizer/join_match_matching_sequence_spec.rb +112 -0
  125. data/spec/rattler/back_end/optimizer/join_predicate_bare_match_spec.rb +194 -0
  126. data/spec/rattler/back_end/optimizer/join_predicate_nested_match_spec.rb +180 -0
  127. data/spec/rattler/back_end/optimizer/join_predicate_or_bare_match_spec.rb +153 -0
  128. data/spec/rattler/back_end/optimizer/join_predicate_or_nested_match_spec.rb +153 -0
  129. data/spec/rattler/back_end/optimizer/reduce_repeat_match_spec.rb +98 -0
  130. data/spec/rattler/back_end/optimizer/simplify_redundant_repeat_spec.rb +226 -0
  131. data/spec/rattler/back_end/optimizer/simplify_token_match_spec.rb +85 -0
  132. data/spec/rattler/back_end/parser_generator/apply_generator_spec.rb +38 -33
  133. data/spec/rattler/back_end/parser_generator/assert_generator_spec.rb +38 -33
  134. data/spec/rattler/back_end/parser_generator/back_reference_generator_spec.rb +181 -0
  135. data/spec/rattler/back_end/parser_generator/choice_generator_spec.rb +38 -33
  136. data/spec/rattler/back_end/parser_generator/direct_action_generator_spec.rb +38 -33
  137. data/spec/rattler/back_end/parser_generator/disallow_generator_spec.rb +38 -33
  138. data/spec/rattler/back_end/parser_generator/dispatch_action_generator_spec.rb +38 -33
  139. data/spec/rattler/back_end/parser_generator/group_match_generator_spec.rb +185 -0
  140. data/spec/rattler/back_end/parser_generator/label_generator_spec.rb +38 -33
  141. data/spec/rattler/back_end/parser_generator/list1_generator_spec.rb +10 -5
  142. data/spec/rattler/back_end/parser_generator/list_generator_spec.rb +10 -5
  143. data/spec/rattler/back_end/parser_generator/match_generator_spec.rb +38 -33
  144. data/spec/rattler/back_end/parser_generator/one_or_more_generator_spec.rb +38 -33
  145. data/spec/rattler/back_end/parser_generator/optional_generator_spec.rb +38 -33
  146. data/spec/rattler/back_end/parser_generator/rule_generator_spec.rb +13 -46
  147. data/spec/rattler/back_end/parser_generator/rule_set_generator_spec.rb +97 -0
  148. data/spec/rattler/back_end/parser_generator/sequence_generator_spec.rb +38 -33
  149. data/spec/rattler/back_end/parser_generator/skip_generator_spec.rb +38 -33
  150. data/spec/rattler/back_end/parser_generator/token_generator_spec.rb +38 -33
  151. data/spec/rattler/back_end/parser_generator/zero_or_more_generator_spec.rb +39 -34
  152. data/spec/rattler/back_end/shared_compiler_examples.rb +885 -0
  153. data/spec/rattler/grammar/analysis_spec.rb +167 -0
  154. data/spec/rattler/grammar/grammar_parser_spec.rb +169 -179
  155. data/spec/rattler/grammar/grammar_spec.rb +24 -21
  156. data/spec/rattler/parsers/action_code_spec.rb +64 -19
  157. data/spec/rattler/parsers/apply_spec.rb +9 -9
  158. data/spec/rattler/parsers/back_reference_spec.rb +38 -0
  159. data/spec/rattler/parsers/combinator_parser_spec.rb +14 -0
  160. data/spec/rattler/parsers/direct_action_spec.rb +16 -2
  161. data/spec/rattler/parsers/dispatch_action_spec.rb +15 -32
  162. data/spec/rattler/parsers/fail_spec.rb +6 -4
  163. data/spec/rattler/parsers/label_spec.rb +10 -28
  164. data/spec/rattler/parsers/node_code_spec.rb +48 -0
  165. data/spec/rattler/parsers/parser_dsl_spec.rb +1 -1
  166. data/spec/rattler/parsers/rule_set_spec.rb +35 -0
  167. data/spec/rattler/parsers/sequence_spec.rb +15 -24
  168. data/spec/rattler/runtime/extended_packrat_parser_spec.rb +22 -17
  169. data/spec/rattler/runtime/packrat_parser_spec.rb +1 -1
  170. data/spec/rattler/runtime/parse_node_spec.rb +15 -19
  171. data/spec/rattler/runtime/recursive_descent_parser_spec.rb +1 -1
  172. data/spec/rattler/runtime/shared_parser_examples.rb +61 -28
  173. data/spec/rattler/util/graphviz/node_builder_spec.rb +84 -0
  174. data/spec/rattler/util/node_spec.rb +92 -65
  175. data/spec/rattler_spec.rb +16 -16
  176. data/spec/support/combinator_parser_spec_helper.rb +19 -18
  177. data/spec/support/compiler_spec_helper.rb +56 -87
  178. data/spec/support/runtime_parser_spec_helper.rb +6 -14
  179. metadata +117 -22
  180. data/features/grammar/regex.feature +0 -24
  181. data/lib/rattler/parsers/match_joining.rb +0 -67
  182. data/lib/rattler/parsers/rules.rb +0 -43
@@ -22,7 +22,7 @@ Feature: Nonterminals
22
22
  """
23
23
  expr <- as "b"
24
24
  as <- "a" as
25
- | "a"
25
+ / "a"
26
26
  """
27
27
  When I parse "aaab"
28
28
  Then the parse result should be [["a", ["a", "a"]], "b"]
@@ -1,6 +1,6 @@
1
1
  Feature: Ordered Choice Expressions
2
2
 
3
- An ordered choice expression is a series of sub-expressions separated by "|"
3
+ An ordered choice expression is a series of sub-expressions separated by "/"
4
4
  and it means to try each sub-expression in order until one matches, and fail
5
5
  if none of the sub-expressions match.
6
6
 
@@ -11,7 +11,7 @@ Feature: Ordered Choice Expressions
11
11
  Scenario Outline: Parsing
12
12
  Given a grammar with:
13
13
  """
14
- expr <- "A" | "B"
14
+ expr <- "A" / "B"
15
15
  """
16
16
  When I parse <input>
17
17
  Then the parse result should be <result>
@@ -9,7 +9,7 @@ Feature: Skip Operator
9
9
  Scenario: Sequence with skipped sub-expressions
10
10
  Given a grammar with:
11
11
  """
12
- sum <- ~"(" /\d+/ ~"+" /\d+/ ~")"
12
+ sum <- ~"(" @DIGIT+ ~"+" @DIGIT+ ~")"
13
13
  """
14
14
  When I parse "(23+45)"
15
15
  Then the parse result should be ["23", "45"]
@@ -15,7 +15,7 @@ Feature: Symantic Actions
15
15
  Scenario: Single token
16
16
  Given a grammar with:
17
17
  """
18
- integer <- /\d+/ {|s| s.to_i }
18
+ integer <- @DIGIT+ {|s| s.to_i }
19
19
  """
20
20
  When I parse "42"
21
21
  Then the parse result should be 42
@@ -24,7 +24,7 @@ Feature: Symantic Actions
24
24
  Given a grammar with:
25
25
  """
26
26
  %whitespace SPACE*
27
- product <- /\d+/ /\d+/ {|a,b| a.to_i * b.to_i }
27
+ product <- @DIGIT+ @DIGIT+ {|a,b| a.to_i * b.to_i }
28
28
  """
29
29
  When I parse "3 16"
30
30
  Then the parse result should be 48
@@ -32,7 +32,7 @@ Feature: Symantic Actions
32
32
  Scenario: Sequence with non-capturing expressions
33
33
  Given a grammar with:
34
34
  """
35
- sum <- ~"(" /\d+/ ~"+" /\d+/ ~")" {|a,b| a.to_i + b.to_i }
35
+ sum <- ~"(" @DIGIT+ ~"+" @DIGIT+ ~")" {|a,b| a.to_i + b.to_i }
36
36
  """
37
37
  When I parse "(23+17)"
38
38
  Then the parse result should be 40
@@ -40,7 +40,7 @@ Feature: Symantic Actions
40
40
  Scenario: Sequence with labeled expressions
41
41
  Given a grammar with:
42
42
  """
43
- sum <- "(" left:/\d+/ "+" right:/\d+/ ")" { left.to_i + right.to_i }
43
+ sum <- "(" left:@DIGIT+ "+" right:@DIGIT+ ")" { left.to_i + right.to_i }
44
44
  """
45
45
  When I parse "(17+29)"
46
46
  Then the parse result should be 46
@@ -48,7 +48,7 @@ Feature: Symantic Actions
48
48
  Scenario: Single token using "_"
49
49
  Given a grammar with:
50
50
  """
51
- integer <- /\d+/ { _.to_i }
51
+ integer <- @DIGIT+ { _.to_i }
52
52
  """
53
53
  When I parse "23"
54
54
  Then the parse result should be 23
@@ -57,7 +57,7 @@ Feature: Symantic Actions
57
57
  Given a grammar with:
58
58
  """
59
59
  %whitespace SPACE*
60
- ints <- /\d+/ /\d+/ { _.reverse }
60
+ ints <- @DIGIT+ @DIGIT+ { _.reverse }
61
61
  """
62
62
  When I parse "3 16"
63
63
  Then the parse result should be ["16", "3"]
@@ -66,7 +66,7 @@ Feature: Symantic Actions
66
66
  Given a grammar with:
67
67
  """
68
68
  %whitespace SPACE*
69
- ints <- /\d+/ /\d+/ {|_| _.to_i }
69
+ ints <- @DIGIT+ @DIGIT+ {|_| _.to_i }
70
70
  """
71
71
  When I parse "3 16"
72
72
  Then the parse result should be 3
@@ -14,7 +14,7 @@ Feature: Whitespace
14
14
  Given a grammar with:
15
15
  """
16
16
  %whitespace SPACE* {
17
- word <- /\w+/
17
+ word <- @WORD+
18
18
  }
19
19
  """
20
20
  When I parse " foo"
@@ -24,7 +24,7 @@ Feature: Whitespace
24
24
  Given a grammar with:
25
25
  """
26
26
  %whitespace SPACE*
27
- word <- /\w+/
27
+ word <- @WORD+
28
28
  """
29
29
  When I parse " foo"
30
30
  Then the parse result should be "foo"
@@ -1,8 +1,8 @@
1
- Given /grammar with:/ do |source|
1
+ Given /grammar with\:/ do |source|
2
2
  @parser_class = Rattler.compile_parser(source)
3
3
  end
4
4
 
5
- Given /class definition:/ do |source|
5
+ Given /class definition\:/ do |source|
6
6
  eval(source, TOPLEVEL_BINDING)
7
7
  end
8
8
 
@@ -17,5 +17,6 @@ module Rattler
17
17
  autoload :Compiler, 'rattler/back_end/compiler'
18
18
  autoload :ParserGenerator, 'rattler/back_end/parser_generator'
19
19
  autoload :RubyGenerator, 'rattler/back_end/ruby_generator'
20
+ autoload :Optimizer, 'rattler/back_end/optimizer'
20
21
  end
21
22
  end
@@ -13,15 +13,16 @@ module Rattler::BackEnd
13
13
  # @author Jason Arhart
14
14
  #
15
15
  class Compiler
16
-
16
+
17
+ include Rattler::Grammar
17
18
  include Rattler::Parsers
18
-
19
+
19
20
  # Compile rules or grammar source into a new parser subclass of +base+.
20
21
  #
21
22
  # @overload compile_parser(base, rules)
22
23
  # Compile +rules+ into a new parser subclass of +base+.
23
24
  # @param [Class] base the base class for the new parser class
24
- # @param [Rattler::Parser::Rules] rules the rules to compile
25
+ # @param [Rattler::Parser::RuleSet] rules the rules to compile
25
26
  # @return [Class] a new parser class
26
27
  #
27
28
  # @overload compile_parser(base, rule)
@@ -40,33 +41,33 @@ module Rattler::BackEnd
40
41
  parser_class = Class.new(base)
41
42
  compile(parser_class, rules_or_grammar)
42
43
  end
43
-
44
+
44
45
  # Compile rules or grammar source into match methods in the module +mod+.
45
46
  def self.compile(mod, rules_or_grammar)
46
47
  self.new(mod).compile(rules_or_grammar)
47
48
  end
48
-
49
+
49
50
  # Compile +rules+ into match methods in the module +mod+.
50
51
  def self.compile_rules(mod, rules)
51
52
  self.new(mod).compile_rules(rules)
52
53
  end
53
-
54
+
54
55
  # Compile grammar +source+ into a match method in the module +mod+.
55
56
  def compile_grammar(mod, source)
56
57
  self.new(mod).compiler_grammar(source)
57
58
  end
58
-
59
+
59
60
  # Create a new compiler that compiles rules into match methods in the
60
61
  # given module.
61
62
  def initialize(mod)
62
63
  @mod = mod
63
64
  end
64
-
65
+
65
66
  # Compile the rules or grammar source into match methods in the module.
66
67
  #
67
68
  # @overload compile(rules)
68
69
  # Compile +rules+ into match methods in the module.
69
- # @param [Rattler::Parser::Rules] rules the rules to compile
70
+ # @param [Rattler::Parser::RuleSet] rules the rules to compile
70
71
  # @return [Module] the module
71
72
  #
72
73
  # @overload compile(rule)
@@ -81,17 +82,16 @@ module Rattler::BackEnd
81
82
  #
82
83
  def compile(_)
83
84
  case _
84
- when Rules then compile_rules(_)
85
- when Rule then compile_rules(_)
85
+ when Grammar, RuleSet, Rule then compile_rules(_)
86
86
  else compile_grammar(_.to_s)
87
87
  end
88
88
  end
89
-
89
+
90
90
  # Compile +rules+ into match methods in the module.
91
91
  #
92
92
  # @overload compile_rules(rules)
93
93
  # Compile +rules+ into match methods in the module.
94
- # @param [Rattler::Parser::Rules] rules the rules to compile
94
+ # @param [Rattler::Parser::RuleSet] rules the rules to compile
95
95
  # @return [Module] the module
96
96
  #
97
97
  # @overload compile_rules(rule)
@@ -100,10 +100,9 @@ module Rattler::BackEnd
100
100
  # @return [Module] the module
101
101
  #
102
102
  def compile_rules(rules)
103
- assert_kind_of([Rules, Rule], rules)
104
- compile_model(rules.optimized)
103
+ compile_model(rules)
105
104
  end
106
-
105
+
107
106
  # Compile +grammar+ into match methods in the module.
108
107
  # @param [String] grammar the grammar source to compile
109
108
  # @return [Module] the module
@@ -111,19 +110,19 @@ module Rattler::BackEnd
111
110
  result = Rattler::Grammar.parse!(source)
112
111
  compile_model(result.rules)
113
112
  end
114
-
113
+
115
114
  private
116
-
115
+
117
116
  def assert_kind_of(kinds, model) #:nodoc:
118
117
  unless kinds.any? {|kind| model.kind_of?(kind) }
119
118
  raise TypeError, "Expected #{model.inspect} to be a kind of #{klass}", caller
120
119
  end
121
120
  end
122
-
121
+
123
122
  def compile_model(model) #:nodoc:
124
123
  @mod.module_eval ParserGenerator.code_for(model)
125
124
  @mod
126
125
  end
127
-
126
+
128
127
  end
129
128
  end
@@ -0,0 +1,100 @@
1
+ #
2
+ # = rattler/back_end/optimizer.rb
3
+ #
4
+ # Author:: Jason Arhart
5
+ # Documentation:: Author
6
+ #
7
+ require 'rattler'
8
+
9
+ module Rattler::BackEnd
10
+ #
11
+ # The +Optimizer+ transforms parser models into equivalent models that can
12
+ # result in more efficient parsing code. This is primarily achieved by
13
+ # converting regular expressions into equivalent Regexp patterns, thus
14
+ # reducing object instantiation and method dispatch and having StringScanner
15
+ # do as much of the parsing work as possible.
16
+ #
17
+ # @author Jason Arhart
18
+ #
19
+ module Optimizer
20
+
21
+ class << self
22
+ def optimizations
23
+ @optimizations ||=
24
+ OptimizeChildren >>
25
+ InlineRegularRules >>
26
+ SimplifyRedundantRepeat >>
27
+ RemoveMeaninglessWrapper >>
28
+ SimplifyTokenMatch >>
29
+ FlattenSequence >>
30
+ FlattenChoice >>
31
+ ReduceRepeatMatch >>
32
+ JoinPredicateMatch >>
33
+ JoinPredicateOrMatch >>
34
+ JoinMatchSequence >>
35
+ JoinMatchChoice
36
+ end
37
+
38
+ def optimize(model, opts={})
39
+ case model
40
+ when ::Rattler::Grammar::Grammar then optimize_grammar model, opts
41
+ when ::Rattler::Parsers::RuleSet then optimize_rule_set model, opts
42
+ when ::Rattler::Parsers::Rule then optimize_rule model, default_context(opts)
43
+ else optimize_expr model, default_context(opts)
44
+ end
45
+ end
46
+
47
+ private
48
+
49
+ def default_context(opts)
50
+ OptimizationContext[opts.merge :type => :capturing]
51
+ end
52
+
53
+ def optimize_grammar(grammar, opts)
54
+ grammar.with_rules optimize_rule_set(grammar.rules, opts)
55
+ end
56
+
57
+ def optimize_rule_set(rule_set, opts)
58
+ context = default_context(opts).with(:rules => rule_set)
59
+ rule_set = rule_set.map_rules {|_| optimize_rule _, context }
60
+ context = context.with(:rules => rule_set)
61
+ rule_set.select_rules {|_| context.relavent? _ }
62
+ end
63
+
64
+ def optimize_rule(rule, context)
65
+ rule.with_expr optimizations.apply(rule.expr, context)
66
+ end
67
+
68
+ def optimize_expr(expr, context)
69
+ optimizations.apply rule.expr, context
70
+ end
71
+
72
+ end
73
+
74
+ autoload :OptimizationContext, 'rattler/back_end/optimizer/optimization_context'
75
+ autoload :Optimization, 'rattler/back_end/optimizer/optimization'
76
+ autoload :OptimizationSequence, 'rattler/back_end/optimizer/optimization_sequence'
77
+ autoload :OptimizeChildren, 'rattler/back_end/optimizer/optimize_children'
78
+ autoload :InlineRegularRules, 'rattler/back_end/optimizer/inline_regular_rules'
79
+ autoload :SimplifyRedundantRepeat, 'rattler/back_end/optimizer/simplify_redundant_repeat'
80
+ autoload :RemoveMeaninglessWrapper, 'rattler/back_end/optimizer/remove_meaningless_wrapper'
81
+ autoload :SimplifyTokenMatch, 'rattler/back_end/optimizer/simplify_token_match'
82
+ autoload :FlattenSequence, 'rattler/back_end/optimizer/flatten_sequence'
83
+ autoload :FlattenChoice, 'rattler/back_end/optimizer/flatten_choice'
84
+ autoload :ReduceRepeatMatch, 'rattler/back_end/optimizer/reduce_repeat_match'
85
+ autoload :JoinPredicateMatch, 'rattler/back_end/optimizer/join_predicate_match'
86
+ autoload :JoinPredicateBareMatch, 'rattler/back_end/optimizer/join_predicate_bare_match'
87
+ autoload :JoinPredicateNestedMatch, 'rattler/back_end/optimizer/join_predicate_nested_match'
88
+ autoload :JoinPredicateOrMatch, 'rattler/back_end/optimizer/join_predicate_or_match'
89
+ autoload :JoinPredicateOrBareMatch, 'rattler/back_end/optimizer/join_predicate_or_bare_match'
90
+ autoload :JoinPredicateOrNestedMatch, 'rattler/back_end/optimizer/join_predicate_or_nested_match'
91
+ autoload :JoinMatchSequence, 'rattler/back_end/optimizer/join_match_sequence'
92
+ autoload :JoinMatchCapturingSequence, 'rattler/back_end/optimizer/join_match_capturing_sequence'
93
+ autoload :JoinMatchMatchingSequence, 'rattler/back_end/optimizer/join_match_matching_sequence'
94
+ autoload :JoinMatchChoice, 'rattler/back_end/optimizer/join_match_choice'
95
+ autoload :MatchJoining, 'rattler/back_end/optimizer/match_joining'
96
+ autoload :Flattening, 'rattler/back_end/optimizer/flattening'
97
+ autoload :CompositeReducing, 'rattler/back_end/optimizer/composite_reducing'
98
+
99
+ end
100
+ end
@@ -0,0 +1,18 @@
1
+ require 'rattler'
2
+
3
+ module Rattler::BackEnd::Optimizer
4
+ # @private
5
+ module CompositeReducing #:nodoc:
6
+
7
+ protected
8
+
9
+ def finish_reduce(parser, new_children)
10
+ if new_children.size == 1
11
+ new_children[0]
12
+ else
13
+ parser.with_children new_children
14
+ end
15
+ end
16
+
17
+ end
18
+ end
@@ -0,0 +1,31 @@
1
+ #
2
+ # = rattler/back_end/optimizer/flatten_choice.rb
3
+ #
4
+ # Author:: Jason Arhart
5
+ # Documentation:: Author
6
+ #
7
+ require 'rattler'
8
+
9
+ module Rattler::BackEnd::Optimizer
10
+ #
11
+ # Nested choice expressions can be flattened without affecting how they parse.
12
+ #
13
+ # @author Jason Arhart
14
+ #
15
+ class FlattenChoice < Optimization
16
+ include Flattening
17
+
18
+ include Rattler::Parsers
19
+
20
+ protected
21
+
22
+ def _applies_to?(parser, context)
23
+ parser.is_a?(Choice) and super
24
+ end
25
+
26
+ def eligible_child?(child)
27
+ child.is_a? Choice
28
+ end
29
+
30
+ end
31
+ end
@@ -0,0 +1,59 @@
1
+ #
2
+ # = rattler/back_end/optimizer/flatten_sequence.rb
3
+ #
4
+ # Author:: Jason Arhart
5
+ # Documentation:: Author
6
+ #
7
+ require 'rattler'
8
+
9
+ module Rattler::BackEnd::Optimizer
10
+
11
+ #
12
+ # Nested sequence expressions can be flattened without affecting how they
13
+ # match.
14
+ #
15
+ # @author Jason Arhart
16
+ #
17
+ class FlattenMatchingSequence < Optimization
18
+ include Flattening
19
+ include Rattler::Parsers
20
+
21
+ protected
22
+
23
+ def _applies_to?(parser, context)
24
+ context.matching? and
25
+ parser.is_a?(Sequence) and
26
+ super
27
+ end
28
+
29
+ def eligible_child?(child)
30
+ child.is_a? Sequence
31
+ end
32
+ end
33
+
34
+ #
35
+ # Nested sequence expressions can be flattened without affecting how they
36
+ # parse if the nested sequence expressions are not multi-capturing.
37
+ #
38
+ # @author Jason Arhart
39
+ #
40
+ class FlattenCapturingSequence < Optimization
41
+ include Flattening
42
+ include Rattler::Parsers
43
+
44
+ protected
45
+
46
+ def _applies_to?(parser, context)
47
+ context.capturing? and
48
+ parser.is_a?(Sequence) and
49
+ super
50
+ end
51
+
52
+ def eligible_child?(child)
53
+ child.is_a?(Sequence) and child.capture_count <= 1
54
+ end
55
+ end
56
+
57
+ FlattenSequence = FlattenMatchingSequence >> FlattenCapturingSequence
58
+
59
+ end