rattler 0.2.2 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. data/README.rdoc +83 -64
  2. data/features/grammar/comments.feature +24 -0
  3. data/features/grammar/list_matching.feature +41 -0
  4. data/features/grammar/symantic_action.feature +30 -12
  5. data/lib/rattler/back_end/parser_generator/assert_generator.rb +27 -27
  6. data/lib/rattler/back_end/parser_generator/choice_generator.rb +29 -29
  7. data/lib/rattler/back_end/parser_generator/direct_action_generator.rb +17 -17
  8. data/lib/rattler/back_end/parser_generator/disallow_generator.rb +27 -27
  9. data/lib/rattler/back_end/parser_generator/dispatch_action_generator.rb +17 -17
  10. data/lib/rattler/back_end/parser_generator/expr_generator.rb +129 -40
  11. data/lib/rattler/back_end/parser_generator/label_generator.rb +15 -15
  12. data/lib/rattler/back_end/parser_generator/list1_generator.rb +61 -0
  13. data/lib/rattler/back_end/parser_generator/list_generating.rb +71 -0
  14. data/lib/rattler/back_end/parser_generator/list_generator.rb +57 -0
  15. data/lib/rattler/back_end/parser_generator/one_or_more_generator.rb +14 -15
  16. data/lib/rattler/back_end/parser_generator/optional_generator.rb +24 -24
  17. data/lib/rattler/back_end/parser_generator/predicate_propogating.rb +9 -9
  18. data/lib/rattler/back_end/parser_generator/repeat_generating.rb +16 -16
  19. data/lib/rattler/back_end/parser_generator/sequence_generator.rb +40 -40
  20. data/lib/rattler/back_end/parser_generator/skip_generator.rb +18 -18
  21. data/lib/rattler/back_end/parser_generator/skip_propogating.rb +5 -5
  22. data/lib/rattler/back_end/parser_generator/sub_generating.rb +128 -0
  23. data/lib/rattler/back_end/parser_generator/token_generator.rb +15 -15
  24. data/lib/rattler/back_end/parser_generator/token_propogating.rb +1 -1
  25. data/lib/rattler/back_end/parser_generator/zero_or_more_generator.rb +12 -13
  26. data/lib/rattler/back_end/parser_generator.rb +10 -7
  27. data/lib/rattler/grammar/grammar_parser.rb +16 -21
  28. data/lib/rattler/grammar/metagrammar.rb +1039 -1035
  29. data/lib/rattler/grammar/rattler.rtlr +28 -28
  30. data/lib/rattler/parsers/action_code.rb +20 -9
  31. data/lib/rattler/parsers/fail.rb +7 -1
  32. data/lib/rattler/parsers/list.rb +57 -0
  33. data/lib/rattler/parsers/list1.rb +58 -0
  34. data/lib/rattler/parsers/parser_dsl.rb +60 -38
  35. data/lib/rattler/parsers.rb +5 -3
  36. data/lib/rattler/runtime/extended_packrat_parser.rb +88 -20
  37. data/lib/rattler/runtime/packrat_parser.rb +21 -14
  38. data/lib/rattler/runtime/parser.rb +74 -18
  39. data/lib/rattler/runtime/recursive_descent_parser.rb +15 -46
  40. data/spec/rattler/back_end/compiler_spec.rb +173 -107
  41. data/spec/rattler/back_end/parser_generator/list1_generator_spec.rb +304 -0
  42. data/spec/rattler/back_end/parser_generator/list_generator_spec.rb +288 -0
  43. data/spec/rattler/grammar/grammar_parser_spec.rb +65 -76
  44. data/spec/rattler/parsers/action_code_spec.rb +84 -34
  45. data/spec/rattler/parsers/direct_action_spec.rb +56 -34
  46. data/spec/rattler/parsers/fail_spec.rb +20 -0
  47. data/spec/rattler/parsers/list1_spec.rb +82 -0
  48. data/spec/rattler/parsers/list_spec.rb +82 -0
  49. data/spec/rattler/parsers/parser_dsl_spec.rb +48 -19
  50. data/spec/rattler/runtime/extended_packrat_parser_spec.rb +0 -1
  51. metadata +92 -173
  52. data/bin/rtlr.bat +0 -3
  53. data/lib/rattler/back_end/parser_generator/generator_helper.rb +0 -130
  54. data/lib/rattler/back_end/parser_generator/generators.rb +0 -86
  55. data/lib/rattler/back_end/parser_generator/nested_generators.rb +0 -15
  56. data/lib/rattler/back_end/parser_generator/top_level_generators.rb +0 -15
@@ -11,52 +11,120 @@ module Rattler::Runtime
11
11
  #
12
12
  # +ExtendedPackratParser+ implements the algorithm described by Alessandro
13
13
  # Warth, James R. Douglass, and Todd Millstein for extending packrat parsing
14
- # to support left-recursive grammars. It currently only implements the first
15
- # part to support direct left recursion.
14
+ # to support left-recursive grammars.
16
15
  #
17
16
  # @author Jason Arhart
18
17
  #
19
18
  class ExtendedPackratParser < PackratParser
20
19
 
20
+ # Create a new extended packrat parser to parse +source+.
21
+ #
22
+ # @param (see PackratParser#initialize)
23
+ # @option (see PackratParser#initialize)
24
+ #
25
+ def initialize(source, options={})
26
+ super
27
+ @heads = {}
28
+ @lr_stack = []
29
+ end
30
+
21
31
  private
22
-
32
+
23
33
  # @private
24
- def apply!(rule_name, key, start_pos) #:nodoc:
25
- lr = LR.new
26
- m = @memo[key] = MemoEntry.new(lr, start_pos, nil, nil)
34
+ def apply!(rule_name, start_pos) #:nodoc:
35
+ lr = LR.new(false, rule_name, nil)
36
+ @lr_stack.push lr
37
+ m = inject_memo rule_name, start_pos, lr, start_pos, nil, nil
27
38
  result = eval_rule rule_name
28
- memorize m, result
29
- result = grow_lr(rule_name, start_pos, m) if result and lr.detected
30
- result
39
+ @lr_stack.pop
40
+ if lr.head
41
+ m.end_pos = @scanner.pos
42
+ lr.seed = result
43
+ lr_answer rule_name, start_pos, m
44
+ else
45
+ memorize m, result
46
+ end
47
+ end
48
+
49
+ # @private
50
+ def memo(rule_name, start_pos) #:nodoc:
51
+ m = super
52
+ head = @heads[start_pos] or return m
53
+ if !m && !head.involves?(rule_name)
54
+ return inject_memo rule_name, start_pos, false, start_pos, nil, nil
55
+ end
56
+ if head.eval_set.delete(rule_name)
57
+ memorize m, eval_rule(rule_name)
58
+ end
59
+ return m
31
60
  end
32
61
 
33
- def recall(m)
62
+ # @private
63
+ def recall(m, rule_name) #:nodoc:
34
64
  if (result = m.result).is_a? LR
35
- result.detected = true
36
- false
65
+ setup_lr rule_name, result
66
+ result.seed
37
67
  else
38
68
  super
39
69
  end
40
70
  end
41
71
 
42
72
  # @private
43
- def grow_lr(rule_name, start_pos, m) #:nodoc:
73
+ def setup_lr(rule_name, lr) #:nodoc:
74
+ lr.head ||= Head.new(rule_name)
75
+ @lr_stack.reverse_each do |_|
76
+ return if _.head == lr.head
77
+ lr.head.involved_set[_.rule_name] = _.rule_name
78
+ end
79
+ end
80
+
81
+ # @private
82
+ def lr_answer(rule_name, start_pos, m) #:nodoc:
83
+ head = m.result.head
84
+ if head.rule_name == rule_name
85
+ grow_lr(rule_name, start_pos, m, head) if m.result = m.result.seed
86
+ else
87
+ memorize m, m.result.seed
88
+ end
89
+ end
90
+
91
+ # @private
92
+ def grow_lr(rule_name, start_pos, m, head) #:nodoc:
93
+ @heads[start_pos] = head
44
94
  loop do
45
95
  @scanner.pos = start_pos
96
+ head.eval_set.replace(head.involved_set)
46
97
  result = eval_rule(rule_name)
47
- return recall(m) if !result or @scanner.pos <= m.end_pos
98
+ if !result or @scanner.pos <= m.end_pos
99
+ @heads.delete(start_pos)
100
+ return recall m, rule_name
101
+ end
48
102
  memorize m, result
49
103
  end
50
104
  end
51
-
105
+
52
106
  # @private
53
- class LR
54
- def initialize(detected = false)
55
- @detected = detected
107
+ class LR #:nodoc:
108
+ def initialize(seed, rule_name, head)
109
+ @seed = seed
110
+ @rule_name = rule_name
111
+ @head = head
56
112
  end
57
- attr_accessor :detected
113
+ attr_accessor :seed, :rule_name, :head
58
114
  end
59
115
 
60
- end
116
+ # @private
117
+ class Head #:nodoc:
118
+ def initialize(rule_name)
119
+ @rule_name = rule_name
120
+ @involved_set = {}
121
+ @eval_set = {}
122
+ end
123
+ attr_accessor :rule_name, :involved_set, :eval_set
124
+ def involves?(rule_name)
125
+ rule_name == self.rule_name or involved_set.has_key? rule_name
126
+ end
127
+ end
61
128
 
129
+ end
62
130
  end
@@ -16,7 +16,7 @@ module Rattler::Runtime
16
16
  # @author Jason Arhart
17
17
  #
18
18
  class PackratParser < RecursiveDescentParser
19
-
19
+
20
20
  # Create a new packrat parser to parse +source+.
21
21
  #
22
22
  # @param (see RecursiveDescentParser#initialize)
@@ -24,15 +24,15 @@ module Rattler::Runtime
24
24
  #
25
25
  def initialize(source, options={})
26
26
  super
27
- @memo = {}
27
+ @memo = Hash.new {|h, rule_name| h[rule_name] = {} }
28
28
  end
29
-
29
+
30
30
  # @private
31
31
  alias_method :eval_rule, :apply
32
32
  private :eval_rule
33
-
33
+
34
34
  protected
35
-
35
+
36
36
  # Apply a rule by dispatching to the method associated with the given rule
37
37
  # name, which is named by <tt>"match_#{rule_name}"<tt>, and if the match
38
38
  # fails set a parse error. The result of applying the rule is memoized
@@ -44,22 +44,29 @@ module Rattler::Runtime
44
44
  #
45
45
  def apply(rule_name)
46
46
  start_pos = @scanner.pos
47
- key = [rule_name, start_pos]
48
- if @memo.has_key? key
49
- recall @memo[key]
47
+ if m = memo(rule_name, start_pos)
48
+ recall m, rule_name
50
49
  else
51
- apply! rule_name, key, start_pos
50
+ apply! rule_name, start_pos
52
51
  end
53
52
  end
54
-
53
+
55
54
  private
56
55
 
57
56
  # @private
58
- def apply!(rule_name, key, start_pos) #:nodoc:
59
- m = @memo[key] = MemoEntry.new(false, start_pos, start_pos, 'left-recursion detected')
57
+ def apply!(rule_name, start_pos) #:nodoc:
58
+ m = inject_memo rule_name, start_pos, false, start_pos, start_pos, 'left-recursion detected'
60
59
  memorize m, eval_rule(rule_name)
61
60
  end
62
61
 
62
+ def memo(rule_name, start_pos)
63
+ @memo[rule_name][start_pos]
64
+ end
65
+
66
+ def inject_memo(rule_name, start_pos, result, end_pos, failure_pos, failure_msg)
67
+ @memo[rule_name][start_pos] = MemoEntry.new(result, end_pos, failure_pos, failure_msg)
68
+ end
69
+
63
70
  # @private
64
71
  def memorize(m, result) #:nodoc:
65
72
  m.end_pos = @scanner.pos
@@ -69,7 +76,7 @@ module Rattler::Runtime
69
76
  end
70
77
 
71
78
  # @private
72
- def recall(m) #:nodoc:
79
+ def recall(m, rule_name) #:nodoc:
73
80
  @scanner.pos = m.end_pos
74
81
  @failure_pos = m.failure_pos
75
82
  @failure_msg = m.failure_msg
@@ -77,7 +84,7 @@ module Rattler::Runtime
77
84
  end
78
85
 
79
86
  # @private
80
- class MemoEntry
87
+ class MemoEntry #:nodoc:
81
88
  def initialize(result, end_pos, failure_pos, failure_msg)
82
89
  @result = result
83
90
  @end_pos = end_pos
@@ -15,7 +15,16 @@ module Rattler::Runtime
15
15
  # @author Jason Arhart
16
16
  #
17
17
  class Parser
18
-
18
+
19
+ # Parse +source+ and raise a {SyntaxError} if the parse fails.
20
+ #
21
+ # @param (see #initialize)
22
+ # @raise (see #parse!)
23
+ # @return (see #parse!)
24
+ def self.parse!(source, options={})
25
+ self.new(source, options).parse!
26
+ end
27
+
19
28
  # Create a new parser to parse +source+.
20
29
  #
21
30
  # @param [String] source the source to parse
@@ -26,24 +35,58 @@ module Rattler::Runtime
26
35
  @scanner = StringScanner.new(source)
27
36
  @tab_size = options[:tab_size]
28
37
  end
29
-
38
+
30
39
  # The source that this parser parses
31
40
  # @return [String] the source that this parser parses
32
41
  attr_reader :source
33
-
42
+
43
+ # Parse or register a parse failure
44
+ #
45
+ # @return the parse result
46
+ def parse
47
+ catch(:parse_failed) { return finish __parse__ }
48
+ false
49
+ end
50
+
51
+ # Parse or raise a {SyntaxError}
52
+ #
53
+ # @raise [SyntaxError] a {SyntaxError} if the parse fails
54
+ #
55
+ # @return (see #parse)
56
+ def parse!
57
+ parse or raise_error
58
+ end
59
+
60
+ # Parse the entire source or register a parse failure
61
+ #
62
+ # @return the parse result if the entire source was matched
63
+ def parse_fully
64
+ (result = parse) && (@scanner.eos? || fail { :EOF }) && result
65
+ end
66
+
67
+ # Parse the entire source or raise a {SyntaxError}
68
+ #
69
+ # @raise [SyntaxError] a {SyntaxError} if the parse fails or the entire
70
+ # source is not matched
71
+ #
72
+ # @return (see #parse_fully)
73
+ def parse_fully!
74
+ parse_full or raise_error
75
+ end
76
+
34
77
  # The current parse position
35
78
  # @return [Integer] the current parse position
36
79
  def pos
37
80
  @scanner.pos
38
81
  end
39
-
82
+
40
83
  # Set the current parse position
41
84
  # @param [Integer] n the new parse position
42
85
  # @return [Integer] n
43
86
  def pos=(n)
44
87
  @scanner.pos = n
45
88
  end
46
-
89
+
47
90
  # Fail and register a parse failure, unless a failure has already
48
91
  # occurred at the same or later position in the source.
49
92
  #
@@ -58,28 +101,41 @@ module Rattler::Runtime
58
101
  register_failure pos, (block_given? ? yield : nil)
59
102
  end
60
103
  end
61
-
104
+
62
105
  # Fail and register a parse failure, unless a failure has already
63
106
  # occurred at a later position in the source.
64
107
  #
65
- # @yieldreturn [String, Symbol] a failure message or rule name
66
- #
67
- # @see ParseFailure
108
+ # @yieldreturn (see #fail)
68
109
  #
69
- # @return [false]
110
+ # @return (see #fail)
70
111
  def fail! # :yield:
71
112
  pos = @scanner.pos
72
113
  unless failure? and @failure_pos > pos
73
114
  register_failure pos, (block_given? ? yield : nil)
74
115
  end
75
116
  end
76
-
117
+
118
+ # Fail the same as <tt>#fail</tt> but cause the entire parse to fail
119
+ # immediately.
120
+ #
121
+ # @yieldreturn (see #fail)
122
+ #
123
+ # @return (see #fail)
124
+ def fail_parse
125
+ if block_given?
126
+ fail! { yield }
127
+ else
128
+ fail!
129
+ end
130
+ throw :parse_failed
131
+ end
132
+
77
133
  # Return true if there is a parse failure
78
134
  # @return [Boolean] true if there is a parse failure
79
135
  def failure?
80
136
  !@failure_pos.nil?
81
137
  end
82
-
138
+
83
139
  # Return the last parse failure
84
140
  # @return [ParseFailure] the last parse failure
85
141
  def failure
@@ -87,9 +143,9 @@ module Rattler::Runtime
87
143
  @__failure__ ||= ParseFailure.new(source, @failure_pos, @failure_msg)
88
144
  end
89
145
  end
90
-
146
+
91
147
  protected
92
-
148
+
93
149
  # Finish any necessary clean-up based on the final parse result.
94
150
  # @param final_result the final parse result
95
151
  # @return final_result
@@ -97,7 +153,7 @@ module Rattler::Runtime
97
153
  clear_failure if final_result
98
154
  final_result
99
155
  end
100
-
156
+
101
157
  # Register a parse failure
102
158
  #
103
159
  # @param [Integer] position the position of the failure
@@ -110,20 +166,20 @@ module Rattler::Runtime
110
166
  @__failure__ = nil
111
167
  false
112
168
  end
113
-
169
+
114
170
  # Clear the registered failure
115
171
  def clear_failure
116
172
  @failure_pos = nil
117
173
  @failure_msg = nil
118
174
  @__failure__ = nil
119
175
  end
120
-
176
+
121
177
  # Raise a {SyntaxError} for the last parse failure
122
178
  # @raise [SyntaxError] a {SyntaxError} for the last parse failure
123
179
  # @return [nothing]
124
180
  def raise_error
125
181
  raise SyntaxError, failure.to_s
126
182
  end
127
-
183
+
128
184
  end
129
185
  end
@@ -19,17 +19,7 @@ module Rattler::Runtime
19
19
  class RecursiveDescentParser < Parser
20
20
  include ParserHelper
21
21
  include Rattler::Grammar::GrammarDSL
22
-
23
- # Parse +source+ by matching the start rule and raise a {SyntaxError} if
24
- # the parse fails.
25
- #
26
- # @param (see #initialize)
27
- # @raise (see #parse!)
28
- # @return (see #parse!)
29
- def self.parse!(source, options={})
30
- self.new(source, options).parse!
31
- end
32
-
22
+
33
23
  # Create a new recursive descent parser to parse +source+.
34
24
  #
35
25
  # @param (see Parser#initialize)
@@ -39,26 +29,7 @@ module Rattler::Runtime
39
29
  super
40
30
  @rule_method_names = Hash.new {|h, name| h[name] = :"match_#{name}" }
41
31
  end
42
-
43
- # Parse by matching the rule returned by <tt>#start_rule</tt> or
44
- # <tt>:start</tt> if <tt>#start_rule</tt> is not defined.
45
- #
46
- # @return the result of applying the start rule
47
- def parse
48
- catch(:parse_failed) { return finish(match(start_rule)) }
49
- false
50
- end
51
-
52
- # Parse by matching the start rule and raise a {SyntaxError} if the parse
53
- # fails.
54
- #
55
- # @raise [SyntaxError] a {SyntaxError} if the parse fails
56
- #
57
- # @return the result of applying the start rule if successful
58
- def parse!
59
- parse or raise_error
60
- end
61
-
32
+
62
33
  # Apply a rule by dispatching to the method associated with +rule_name+
63
34
  # which is named by <tt>"match_#{rule_name}"<tt>, and if the match fails
64
35
  # register a parse failure.
@@ -69,17 +40,25 @@ module Rattler::Runtime
69
40
  def match(rule_name)
70
41
  apply(rule_name) or fail { rule_name }
71
42
  end
72
-
43
+
73
44
  def method_missing(symbol, *args)
74
45
  (symbol == :start_rule) ? :start : super
75
46
  end
76
-
47
+
77
48
  def respond_to?(symbol)
78
49
  super or (symbol == :start_rule)
79
50
  end
80
-
51
+
81
52
  protected
82
-
53
+
54
+ # Parse by matching the rule returned by <tt>#start_rule</tt> or
55
+ # <tt>:start</tt> if <tt>#start_rule</tt> is not defined.
56
+ #
57
+ # @return the result of applying the start rule
58
+ def __parse__
59
+ match start_rule
60
+ end
61
+
83
62
  # Apply a rule by dispatching to the method associated with the given rule
84
63
  # name, which is named by <tt>"match_#{rule_name}"<tt>. This method is
85
64
  # called by +match+ and should not be called directly.
@@ -91,16 +70,6 @@ module Rattler::Runtime
91
70
  def apply(rule_name)
92
71
  send @rule_method_names[rule_name]
93
72
  end
94
-
95
- # Fail the same as <tt>#fail</tt> but cause the entire parse to fail.
96
- def fail_parse
97
- if block_given?
98
- fail! { yield }
99
- else
100
- fail!
101
- end
102
- throw :parse_failed
103
- end
104
-
73
+
105
74
  end
106
75
  end