rattler 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.rdoc +83 -64
- data/features/grammar/comments.feature +24 -0
- data/features/grammar/list_matching.feature +41 -0
- data/features/grammar/symantic_action.feature +30 -12
- data/lib/rattler/back_end/parser_generator/assert_generator.rb +27 -27
- data/lib/rattler/back_end/parser_generator/choice_generator.rb +29 -29
- data/lib/rattler/back_end/parser_generator/direct_action_generator.rb +17 -17
- data/lib/rattler/back_end/parser_generator/disallow_generator.rb +27 -27
- data/lib/rattler/back_end/parser_generator/dispatch_action_generator.rb +17 -17
- data/lib/rattler/back_end/parser_generator/expr_generator.rb +129 -40
- data/lib/rattler/back_end/parser_generator/label_generator.rb +15 -15
- data/lib/rattler/back_end/parser_generator/list1_generator.rb +61 -0
- data/lib/rattler/back_end/parser_generator/list_generating.rb +71 -0
- data/lib/rattler/back_end/parser_generator/list_generator.rb +57 -0
- data/lib/rattler/back_end/parser_generator/one_or_more_generator.rb +14 -15
- data/lib/rattler/back_end/parser_generator/optional_generator.rb +24 -24
- data/lib/rattler/back_end/parser_generator/predicate_propogating.rb +9 -9
- data/lib/rattler/back_end/parser_generator/repeat_generating.rb +16 -16
- data/lib/rattler/back_end/parser_generator/sequence_generator.rb +40 -40
- data/lib/rattler/back_end/parser_generator/skip_generator.rb +18 -18
- data/lib/rattler/back_end/parser_generator/skip_propogating.rb +5 -5
- data/lib/rattler/back_end/parser_generator/sub_generating.rb +128 -0
- data/lib/rattler/back_end/parser_generator/token_generator.rb +15 -15
- data/lib/rattler/back_end/parser_generator/token_propogating.rb +1 -1
- data/lib/rattler/back_end/parser_generator/zero_or_more_generator.rb +12 -13
- data/lib/rattler/back_end/parser_generator.rb +10 -7
- data/lib/rattler/grammar/grammar_parser.rb +16 -21
- data/lib/rattler/grammar/metagrammar.rb +1039 -1035
- data/lib/rattler/grammar/rattler.rtlr +28 -28
- data/lib/rattler/parsers/action_code.rb +20 -9
- data/lib/rattler/parsers/fail.rb +7 -1
- data/lib/rattler/parsers/list.rb +57 -0
- data/lib/rattler/parsers/list1.rb +58 -0
- data/lib/rattler/parsers/parser_dsl.rb +60 -38
- data/lib/rattler/parsers.rb +5 -3
- data/lib/rattler/runtime/extended_packrat_parser.rb +88 -20
- data/lib/rattler/runtime/packrat_parser.rb +21 -14
- data/lib/rattler/runtime/parser.rb +74 -18
- data/lib/rattler/runtime/recursive_descent_parser.rb +15 -46
- data/spec/rattler/back_end/compiler_spec.rb +173 -107
- data/spec/rattler/back_end/parser_generator/list1_generator_spec.rb +304 -0
- data/spec/rattler/back_end/parser_generator/list_generator_spec.rb +288 -0
- data/spec/rattler/grammar/grammar_parser_spec.rb +65 -76
- data/spec/rattler/parsers/action_code_spec.rb +84 -34
- data/spec/rattler/parsers/direct_action_spec.rb +56 -34
- data/spec/rattler/parsers/fail_spec.rb +20 -0
- data/spec/rattler/parsers/list1_spec.rb +82 -0
- data/spec/rattler/parsers/list_spec.rb +82 -0
- data/spec/rattler/parsers/parser_dsl_spec.rb +48 -19
- data/spec/rattler/runtime/extended_packrat_parser_spec.rb +0 -1
- metadata +92 -173
- data/bin/rtlr.bat +0 -3
- data/lib/rattler/back_end/parser_generator/generator_helper.rb +0 -130
- data/lib/rattler/back_end/parser_generator/generators.rb +0 -86
- data/lib/rattler/back_end/parser_generator/nested_generators.rb +0 -15
- data/lib/rattler/back_end/parser_generator/top_level_generators.rb +0 -15
@@ -11,52 +11,120 @@ module Rattler::Runtime
|
|
11
11
|
#
|
12
12
|
# +ExtendedPackratParser+ implements the algorithm described by Alessandro
|
13
13
|
# Warth, James R. Douglass, and Todd Millstein for extending packrat parsing
|
14
|
-
# to support left-recursive grammars.
|
15
|
-
# part to support direct left recursion.
|
14
|
+
# to support left-recursive grammars.
|
16
15
|
#
|
17
16
|
# @author Jason Arhart
|
18
17
|
#
|
19
18
|
class ExtendedPackratParser < PackratParser
|
20
19
|
|
20
|
+
# Create a new extended packrat parser to parse +source+.
|
21
|
+
#
|
22
|
+
# @param (see PackratParser#initialize)
|
23
|
+
# @option (see PackratParser#initialize)
|
24
|
+
#
|
25
|
+
def initialize(source, options={})
|
26
|
+
super
|
27
|
+
@heads = {}
|
28
|
+
@lr_stack = []
|
29
|
+
end
|
30
|
+
|
21
31
|
private
|
22
|
-
|
32
|
+
|
23
33
|
# @private
|
24
|
-
def apply!(rule_name,
|
25
|
-
lr = LR.new
|
26
|
-
|
34
|
+
def apply!(rule_name, start_pos) #:nodoc:
|
35
|
+
lr = LR.new(false, rule_name, nil)
|
36
|
+
@lr_stack.push lr
|
37
|
+
m = inject_memo rule_name, start_pos, lr, start_pos, nil, nil
|
27
38
|
result = eval_rule rule_name
|
28
|
-
|
29
|
-
|
30
|
-
|
39
|
+
@lr_stack.pop
|
40
|
+
if lr.head
|
41
|
+
m.end_pos = @scanner.pos
|
42
|
+
lr.seed = result
|
43
|
+
lr_answer rule_name, start_pos, m
|
44
|
+
else
|
45
|
+
memorize m, result
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
# @private
|
50
|
+
def memo(rule_name, start_pos) #:nodoc:
|
51
|
+
m = super
|
52
|
+
head = @heads[start_pos] or return m
|
53
|
+
if !m && !head.involves?(rule_name)
|
54
|
+
return inject_memo rule_name, start_pos, false, start_pos, nil, nil
|
55
|
+
end
|
56
|
+
if head.eval_set.delete(rule_name)
|
57
|
+
memorize m, eval_rule(rule_name)
|
58
|
+
end
|
59
|
+
return m
|
31
60
|
end
|
32
61
|
|
33
|
-
|
62
|
+
# @private
|
63
|
+
def recall(m, rule_name) #:nodoc:
|
34
64
|
if (result = m.result).is_a? LR
|
35
|
-
|
36
|
-
|
65
|
+
setup_lr rule_name, result
|
66
|
+
result.seed
|
37
67
|
else
|
38
68
|
super
|
39
69
|
end
|
40
70
|
end
|
41
71
|
|
42
72
|
# @private
|
43
|
-
def
|
73
|
+
def setup_lr(rule_name, lr) #:nodoc:
|
74
|
+
lr.head ||= Head.new(rule_name)
|
75
|
+
@lr_stack.reverse_each do |_|
|
76
|
+
return if _.head == lr.head
|
77
|
+
lr.head.involved_set[_.rule_name] = _.rule_name
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
# @private
|
82
|
+
def lr_answer(rule_name, start_pos, m) #:nodoc:
|
83
|
+
head = m.result.head
|
84
|
+
if head.rule_name == rule_name
|
85
|
+
grow_lr(rule_name, start_pos, m, head) if m.result = m.result.seed
|
86
|
+
else
|
87
|
+
memorize m, m.result.seed
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
# @private
|
92
|
+
def grow_lr(rule_name, start_pos, m, head) #:nodoc:
|
93
|
+
@heads[start_pos] = head
|
44
94
|
loop do
|
45
95
|
@scanner.pos = start_pos
|
96
|
+
head.eval_set.replace(head.involved_set)
|
46
97
|
result = eval_rule(rule_name)
|
47
|
-
|
98
|
+
if !result or @scanner.pos <= m.end_pos
|
99
|
+
@heads.delete(start_pos)
|
100
|
+
return recall m, rule_name
|
101
|
+
end
|
48
102
|
memorize m, result
|
49
103
|
end
|
50
104
|
end
|
51
|
-
|
105
|
+
|
52
106
|
# @private
|
53
|
-
class LR
|
54
|
-
def initialize(
|
55
|
-
@
|
107
|
+
class LR #:nodoc:
|
108
|
+
def initialize(seed, rule_name, head)
|
109
|
+
@seed = seed
|
110
|
+
@rule_name = rule_name
|
111
|
+
@head = head
|
56
112
|
end
|
57
|
-
attr_accessor :
|
113
|
+
attr_accessor :seed, :rule_name, :head
|
58
114
|
end
|
59
115
|
|
60
|
-
|
116
|
+
# @private
|
117
|
+
class Head #:nodoc:
|
118
|
+
def initialize(rule_name)
|
119
|
+
@rule_name = rule_name
|
120
|
+
@involved_set = {}
|
121
|
+
@eval_set = {}
|
122
|
+
end
|
123
|
+
attr_accessor :rule_name, :involved_set, :eval_set
|
124
|
+
def involves?(rule_name)
|
125
|
+
rule_name == self.rule_name or involved_set.has_key? rule_name
|
126
|
+
end
|
127
|
+
end
|
61
128
|
|
129
|
+
end
|
62
130
|
end
|
@@ -16,7 +16,7 @@ module Rattler::Runtime
|
|
16
16
|
# @author Jason Arhart
|
17
17
|
#
|
18
18
|
class PackratParser < RecursiveDescentParser
|
19
|
-
|
19
|
+
|
20
20
|
# Create a new packrat parser to parse +source+.
|
21
21
|
#
|
22
22
|
# @param (see RecursiveDescentParser#initialize)
|
@@ -24,15 +24,15 @@ module Rattler::Runtime
|
|
24
24
|
#
|
25
25
|
def initialize(source, options={})
|
26
26
|
super
|
27
|
-
@memo = {}
|
27
|
+
@memo = Hash.new {|h, rule_name| h[rule_name] = {} }
|
28
28
|
end
|
29
|
-
|
29
|
+
|
30
30
|
# @private
|
31
31
|
alias_method :eval_rule, :apply
|
32
32
|
private :eval_rule
|
33
|
-
|
33
|
+
|
34
34
|
protected
|
35
|
-
|
35
|
+
|
36
36
|
# Apply a rule by dispatching to the method associated with the given rule
|
37
37
|
# name, which is named by <tt>"match_#{rule_name}"<tt>, and if the match
|
38
38
|
# fails set a parse error. The result of applying the rule is memoized
|
@@ -44,22 +44,29 @@ module Rattler::Runtime
|
|
44
44
|
#
|
45
45
|
def apply(rule_name)
|
46
46
|
start_pos = @scanner.pos
|
47
|
-
|
48
|
-
|
49
|
-
recall @memo[key]
|
47
|
+
if m = memo(rule_name, start_pos)
|
48
|
+
recall m, rule_name
|
50
49
|
else
|
51
|
-
apply! rule_name,
|
50
|
+
apply! rule_name, start_pos
|
52
51
|
end
|
53
52
|
end
|
54
|
-
|
53
|
+
|
55
54
|
private
|
56
55
|
|
57
56
|
# @private
|
58
|
-
def apply!(rule_name,
|
59
|
-
m =
|
57
|
+
def apply!(rule_name, start_pos) #:nodoc:
|
58
|
+
m = inject_memo rule_name, start_pos, false, start_pos, start_pos, 'left-recursion detected'
|
60
59
|
memorize m, eval_rule(rule_name)
|
61
60
|
end
|
62
61
|
|
62
|
+
def memo(rule_name, start_pos)
|
63
|
+
@memo[rule_name][start_pos]
|
64
|
+
end
|
65
|
+
|
66
|
+
def inject_memo(rule_name, start_pos, result, end_pos, failure_pos, failure_msg)
|
67
|
+
@memo[rule_name][start_pos] = MemoEntry.new(result, end_pos, failure_pos, failure_msg)
|
68
|
+
end
|
69
|
+
|
63
70
|
# @private
|
64
71
|
def memorize(m, result) #:nodoc:
|
65
72
|
m.end_pos = @scanner.pos
|
@@ -69,7 +76,7 @@ module Rattler::Runtime
|
|
69
76
|
end
|
70
77
|
|
71
78
|
# @private
|
72
|
-
def recall(m) #:nodoc:
|
79
|
+
def recall(m, rule_name) #:nodoc:
|
73
80
|
@scanner.pos = m.end_pos
|
74
81
|
@failure_pos = m.failure_pos
|
75
82
|
@failure_msg = m.failure_msg
|
@@ -77,7 +84,7 @@ module Rattler::Runtime
|
|
77
84
|
end
|
78
85
|
|
79
86
|
# @private
|
80
|
-
class MemoEntry
|
87
|
+
class MemoEntry #:nodoc:
|
81
88
|
def initialize(result, end_pos, failure_pos, failure_msg)
|
82
89
|
@result = result
|
83
90
|
@end_pos = end_pos
|
@@ -15,7 +15,16 @@ module Rattler::Runtime
|
|
15
15
|
# @author Jason Arhart
|
16
16
|
#
|
17
17
|
class Parser
|
18
|
-
|
18
|
+
|
19
|
+
# Parse +source+ and raise a {SyntaxError} if the parse fails.
|
20
|
+
#
|
21
|
+
# @param (see #initialize)
|
22
|
+
# @raise (see #parse!)
|
23
|
+
# @return (see #parse!)
|
24
|
+
def self.parse!(source, options={})
|
25
|
+
self.new(source, options).parse!
|
26
|
+
end
|
27
|
+
|
19
28
|
# Create a new parser to parse +source+.
|
20
29
|
#
|
21
30
|
# @param [String] source the source to parse
|
@@ -26,24 +35,58 @@ module Rattler::Runtime
|
|
26
35
|
@scanner = StringScanner.new(source)
|
27
36
|
@tab_size = options[:tab_size]
|
28
37
|
end
|
29
|
-
|
38
|
+
|
30
39
|
# The source that this parser parses
|
31
40
|
# @return [String] the source that this parser parses
|
32
41
|
attr_reader :source
|
33
|
-
|
42
|
+
|
43
|
+
# Parse or register a parse failure
|
44
|
+
#
|
45
|
+
# @return the parse result
|
46
|
+
def parse
|
47
|
+
catch(:parse_failed) { return finish __parse__ }
|
48
|
+
false
|
49
|
+
end
|
50
|
+
|
51
|
+
# Parse or raise a {SyntaxError}
|
52
|
+
#
|
53
|
+
# @raise [SyntaxError] a {SyntaxError} if the parse fails
|
54
|
+
#
|
55
|
+
# @return (see #parse)
|
56
|
+
def parse!
|
57
|
+
parse or raise_error
|
58
|
+
end
|
59
|
+
|
60
|
+
# Parse the entire source or register a parse failure
|
61
|
+
#
|
62
|
+
# @return the parse result if the entire source was matched
|
63
|
+
def parse_fully
|
64
|
+
(result = parse) && (@scanner.eos? || fail { :EOF }) && result
|
65
|
+
end
|
66
|
+
|
67
|
+
# Parse the entire source or raise a {SyntaxError}
|
68
|
+
#
|
69
|
+
# @raise [SyntaxError] a {SyntaxError} if the parse fails or the entire
|
70
|
+
# source is not matched
|
71
|
+
#
|
72
|
+
# @return (see #parse_fully)
|
73
|
+
def parse_fully!
|
74
|
+
parse_full or raise_error
|
75
|
+
end
|
76
|
+
|
34
77
|
# The current parse position
|
35
78
|
# @return [Integer] the current parse position
|
36
79
|
def pos
|
37
80
|
@scanner.pos
|
38
81
|
end
|
39
|
-
|
82
|
+
|
40
83
|
# Set the current parse position
|
41
84
|
# @param [Integer] n the new parse position
|
42
85
|
# @return [Integer] n
|
43
86
|
def pos=(n)
|
44
87
|
@scanner.pos = n
|
45
88
|
end
|
46
|
-
|
89
|
+
|
47
90
|
# Fail and register a parse failure, unless a failure has already
|
48
91
|
# occurred at the same or later position in the source.
|
49
92
|
#
|
@@ -58,28 +101,41 @@ module Rattler::Runtime
|
|
58
101
|
register_failure pos, (block_given? ? yield : nil)
|
59
102
|
end
|
60
103
|
end
|
61
|
-
|
104
|
+
|
62
105
|
# Fail and register a parse failure, unless a failure has already
|
63
106
|
# occurred at a later position in the source.
|
64
107
|
#
|
65
|
-
# @yieldreturn
|
66
|
-
#
|
67
|
-
# @see ParseFailure
|
108
|
+
# @yieldreturn (see #fail)
|
68
109
|
#
|
69
|
-
# @return
|
110
|
+
# @return (see #fail)
|
70
111
|
def fail! # :yield:
|
71
112
|
pos = @scanner.pos
|
72
113
|
unless failure? and @failure_pos > pos
|
73
114
|
register_failure pos, (block_given? ? yield : nil)
|
74
115
|
end
|
75
116
|
end
|
76
|
-
|
117
|
+
|
118
|
+
# Fail the same as <tt>#fail</tt> but cause the entire parse to fail
|
119
|
+
# immediately.
|
120
|
+
#
|
121
|
+
# @yieldreturn (see #fail)
|
122
|
+
#
|
123
|
+
# @return (see #fail)
|
124
|
+
def fail_parse
|
125
|
+
if block_given?
|
126
|
+
fail! { yield }
|
127
|
+
else
|
128
|
+
fail!
|
129
|
+
end
|
130
|
+
throw :parse_failed
|
131
|
+
end
|
132
|
+
|
77
133
|
# Return true if there is a parse failure
|
78
134
|
# @return [Boolean] true if there is a parse failure
|
79
135
|
def failure?
|
80
136
|
!@failure_pos.nil?
|
81
137
|
end
|
82
|
-
|
138
|
+
|
83
139
|
# Return the last parse failure
|
84
140
|
# @return [ParseFailure] the last parse failure
|
85
141
|
def failure
|
@@ -87,9 +143,9 @@ module Rattler::Runtime
|
|
87
143
|
@__failure__ ||= ParseFailure.new(source, @failure_pos, @failure_msg)
|
88
144
|
end
|
89
145
|
end
|
90
|
-
|
146
|
+
|
91
147
|
protected
|
92
|
-
|
148
|
+
|
93
149
|
# Finish any necessary clean-up based on the final parse result.
|
94
150
|
# @param final_result the final parse result
|
95
151
|
# @return final_result
|
@@ -97,7 +153,7 @@ module Rattler::Runtime
|
|
97
153
|
clear_failure if final_result
|
98
154
|
final_result
|
99
155
|
end
|
100
|
-
|
156
|
+
|
101
157
|
# Register a parse failure
|
102
158
|
#
|
103
159
|
# @param [Integer] position the position of the failure
|
@@ -110,20 +166,20 @@ module Rattler::Runtime
|
|
110
166
|
@__failure__ = nil
|
111
167
|
false
|
112
168
|
end
|
113
|
-
|
169
|
+
|
114
170
|
# Clear the registered failure
|
115
171
|
def clear_failure
|
116
172
|
@failure_pos = nil
|
117
173
|
@failure_msg = nil
|
118
174
|
@__failure__ = nil
|
119
175
|
end
|
120
|
-
|
176
|
+
|
121
177
|
# Raise a {SyntaxError} for the last parse failure
|
122
178
|
# @raise [SyntaxError] a {SyntaxError} for the last parse failure
|
123
179
|
# @return [nothing]
|
124
180
|
def raise_error
|
125
181
|
raise SyntaxError, failure.to_s
|
126
182
|
end
|
127
|
-
|
183
|
+
|
128
184
|
end
|
129
185
|
end
|
@@ -19,17 +19,7 @@ module Rattler::Runtime
|
|
19
19
|
class RecursiveDescentParser < Parser
|
20
20
|
include ParserHelper
|
21
21
|
include Rattler::Grammar::GrammarDSL
|
22
|
-
|
23
|
-
# Parse +source+ by matching the start rule and raise a {SyntaxError} if
|
24
|
-
# the parse fails.
|
25
|
-
#
|
26
|
-
# @param (see #initialize)
|
27
|
-
# @raise (see #parse!)
|
28
|
-
# @return (see #parse!)
|
29
|
-
def self.parse!(source, options={})
|
30
|
-
self.new(source, options).parse!
|
31
|
-
end
|
32
|
-
|
22
|
+
|
33
23
|
# Create a new recursive descent parser to parse +source+.
|
34
24
|
#
|
35
25
|
# @param (see Parser#initialize)
|
@@ -39,26 +29,7 @@ module Rattler::Runtime
|
|
39
29
|
super
|
40
30
|
@rule_method_names = Hash.new {|h, name| h[name] = :"match_#{name}" }
|
41
31
|
end
|
42
|
-
|
43
|
-
# Parse by matching the rule returned by <tt>#start_rule</tt> or
|
44
|
-
# <tt>:start</tt> if <tt>#start_rule</tt> is not defined.
|
45
|
-
#
|
46
|
-
# @return the result of applying the start rule
|
47
|
-
def parse
|
48
|
-
catch(:parse_failed) { return finish(match(start_rule)) }
|
49
|
-
false
|
50
|
-
end
|
51
|
-
|
52
|
-
# Parse by matching the start rule and raise a {SyntaxError} if the parse
|
53
|
-
# fails.
|
54
|
-
#
|
55
|
-
# @raise [SyntaxError] a {SyntaxError} if the parse fails
|
56
|
-
#
|
57
|
-
# @return the result of applying the start rule if successful
|
58
|
-
def parse!
|
59
|
-
parse or raise_error
|
60
|
-
end
|
61
|
-
|
32
|
+
|
62
33
|
# Apply a rule by dispatching to the method associated with +rule_name+
|
63
34
|
# which is named by <tt>"match_#{rule_name}"<tt>, and if the match fails
|
64
35
|
# register a parse failure.
|
@@ -69,17 +40,25 @@ module Rattler::Runtime
|
|
69
40
|
def match(rule_name)
|
70
41
|
apply(rule_name) or fail { rule_name }
|
71
42
|
end
|
72
|
-
|
43
|
+
|
73
44
|
def method_missing(symbol, *args)
|
74
45
|
(symbol == :start_rule) ? :start : super
|
75
46
|
end
|
76
|
-
|
47
|
+
|
77
48
|
def respond_to?(symbol)
|
78
49
|
super or (symbol == :start_rule)
|
79
50
|
end
|
80
|
-
|
51
|
+
|
81
52
|
protected
|
82
|
-
|
53
|
+
|
54
|
+
# Parse by matching the rule returned by <tt>#start_rule</tt> or
|
55
|
+
# <tt>:start</tt> if <tt>#start_rule</tt> is not defined.
|
56
|
+
#
|
57
|
+
# @return the result of applying the start rule
|
58
|
+
def __parse__
|
59
|
+
match start_rule
|
60
|
+
end
|
61
|
+
|
83
62
|
# Apply a rule by dispatching to the method associated with the given rule
|
84
63
|
# name, which is named by <tt>"match_#{rule_name}"<tt>. This method is
|
85
64
|
# called by +match+ and should not be called directly.
|
@@ -91,16 +70,6 @@ module Rattler::Runtime
|
|
91
70
|
def apply(rule_name)
|
92
71
|
send @rule_method_names[rule_name]
|
93
72
|
end
|
94
|
-
|
95
|
-
# Fail the same as <tt>#fail</tt> but cause the entire parse to fail.
|
96
|
-
def fail_parse
|
97
|
-
if block_given?
|
98
|
-
fail! { yield }
|
99
|
-
else
|
100
|
-
fail!
|
101
|
-
end
|
102
|
-
throw :parse_failed
|
103
|
-
end
|
104
|
-
|
73
|
+
|
105
74
|
end
|
106
75
|
end
|