rattler 0.2.2 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. data/README.rdoc +83 -64
  2. data/features/grammar/comments.feature +24 -0
  3. data/features/grammar/list_matching.feature +41 -0
  4. data/features/grammar/symantic_action.feature +30 -12
  5. data/lib/rattler/back_end/parser_generator/assert_generator.rb +27 -27
  6. data/lib/rattler/back_end/parser_generator/choice_generator.rb +29 -29
  7. data/lib/rattler/back_end/parser_generator/direct_action_generator.rb +17 -17
  8. data/lib/rattler/back_end/parser_generator/disallow_generator.rb +27 -27
  9. data/lib/rattler/back_end/parser_generator/dispatch_action_generator.rb +17 -17
  10. data/lib/rattler/back_end/parser_generator/expr_generator.rb +129 -40
  11. data/lib/rattler/back_end/parser_generator/label_generator.rb +15 -15
  12. data/lib/rattler/back_end/parser_generator/list1_generator.rb +61 -0
  13. data/lib/rattler/back_end/parser_generator/list_generating.rb +71 -0
  14. data/lib/rattler/back_end/parser_generator/list_generator.rb +57 -0
  15. data/lib/rattler/back_end/parser_generator/one_or_more_generator.rb +14 -15
  16. data/lib/rattler/back_end/parser_generator/optional_generator.rb +24 -24
  17. data/lib/rattler/back_end/parser_generator/predicate_propogating.rb +9 -9
  18. data/lib/rattler/back_end/parser_generator/repeat_generating.rb +16 -16
  19. data/lib/rattler/back_end/parser_generator/sequence_generator.rb +40 -40
  20. data/lib/rattler/back_end/parser_generator/skip_generator.rb +18 -18
  21. data/lib/rattler/back_end/parser_generator/skip_propogating.rb +5 -5
  22. data/lib/rattler/back_end/parser_generator/sub_generating.rb +128 -0
  23. data/lib/rattler/back_end/parser_generator/token_generator.rb +15 -15
  24. data/lib/rattler/back_end/parser_generator/token_propogating.rb +1 -1
  25. data/lib/rattler/back_end/parser_generator/zero_or_more_generator.rb +12 -13
  26. data/lib/rattler/back_end/parser_generator.rb +10 -7
  27. data/lib/rattler/grammar/grammar_parser.rb +16 -21
  28. data/lib/rattler/grammar/metagrammar.rb +1039 -1035
  29. data/lib/rattler/grammar/rattler.rtlr +28 -28
  30. data/lib/rattler/parsers/action_code.rb +20 -9
  31. data/lib/rattler/parsers/fail.rb +7 -1
  32. data/lib/rattler/parsers/list.rb +57 -0
  33. data/lib/rattler/parsers/list1.rb +58 -0
  34. data/lib/rattler/parsers/parser_dsl.rb +60 -38
  35. data/lib/rattler/parsers.rb +5 -3
  36. data/lib/rattler/runtime/extended_packrat_parser.rb +88 -20
  37. data/lib/rattler/runtime/packrat_parser.rb +21 -14
  38. data/lib/rattler/runtime/parser.rb +74 -18
  39. data/lib/rattler/runtime/recursive_descent_parser.rb +15 -46
  40. data/spec/rattler/back_end/compiler_spec.rb +173 -107
  41. data/spec/rattler/back_end/parser_generator/list1_generator_spec.rb +304 -0
  42. data/spec/rattler/back_end/parser_generator/list_generator_spec.rb +288 -0
  43. data/spec/rattler/grammar/grammar_parser_spec.rb +65 -76
  44. data/spec/rattler/parsers/action_code_spec.rb +84 -34
  45. data/spec/rattler/parsers/direct_action_spec.rb +56 -34
  46. data/spec/rattler/parsers/fail_spec.rb +20 -0
  47. data/spec/rattler/parsers/list1_spec.rb +82 -0
  48. data/spec/rattler/parsers/list_spec.rb +82 -0
  49. data/spec/rattler/parsers/parser_dsl_spec.rb +48 -19
  50. data/spec/rattler/runtime/extended_packrat_parser_spec.rb +0 -1
  51. metadata +92 -173
  52. data/bin/rtlr.bat +0 -3
  53. data/lib/rattler/back_end/parser_generator/generator_helper.rb +0 -130
  54. data/lib/rattler/back_end/parser_generator/generators.rb +0 -86
  55. data/lib/rattler/back_end/parser_generator/nested_generators.rb +0 -15
  56. data/lib/rattler/back_end/parser_generator/top_level_generators.rb +0 -15
@@ -4,20 +4,20 @@ include Rattler::Parsers
4
4
 
5
5
  describe Rattler::Grammar::GrammarParser do
6
6
  include Rattler::Util::ParserSpecHelper
7
-
7
+
8
8
  let :posix_names do
9
9
  %w{alnum alpha blank cntrl digit graph lower print punct space upper xdigit word}
10
10
  end
11
-
11
+
12
12
  it 'skips normal whitespace' do
13
13
  parsing(" \n\t foo").as(:identifier).should result_in('foo').at(8)
14
14
  end
15
-
15
+
16
16
  it 'skips comments' do
17
17
  parsing("\n# a comment\n\t foo").as(:identifier).
18
18
  should result_in('foo').at(18)
19
19
  end
20
-
20
+
21
21
  describe '#match(:identifier)' do
22
22
  it 'recognizes identifiers' do
23
23
  parsing(' fooBar ').as(:identifier).should result_in('fooBar').at(7)
@@ -25,30 +25,30 @@ describe Rattler::Grammar::GrammarParser do
25
25
  parsing(' EO ').as(:identifier).should result_in('EO').at(3)
26
26
  parsing(' EOFA ').as(:identifier).should result_in('EOFA').at(5)
27
27
  end
28
-
28
+
29
29
  it 'does not recognize EOF as identifier' do
30
30
  parsing('EOF').as(:identifier).should fail
31
31
  end
32
32
  end
33
-
33
+
34
34
  describe '#match(:eol)' do
35
35
  it 'recognizes end-of-line before end-of-file' do
36
36
  parsing("foo\nbar").from(3).as(:eol).should result_in(true).at(4)
37
37
  parsing("foobar").from(3).as(:eol).should fail
38
38
  end
39
-
39
+
40
40
  it 'recognizes end-of-file as end-of-line' do
41
41
  parsing("foo\nbar").from(7).as(:eol).should result_in(true).at(7)
42
42
  end
43
43
  end
44
-
44
+
45
45
  describe '#match(:var_name)' do
46
46
  it 'recognizes variable names' do
47
47
  parsing(' fooBar ').as(:var_name).should result_in('fooBar').at(7)
48
48
  parsing(' FooBar ').as(:var_name).should fail
49
49
  end
50
50
  end
51
-
51
+
52
52
  describe '#match(:const_name)' do
53
53
  it 'recognizes constant names' do
54
54
  parsing(' FooBar ').as(:const_name).should result_in('FooBar').at(7)
@@ -56,14 +56,14 @@ describe Rattler::Grammar::GrammarParser do
56
56
  parsing(' fooBar ').as(:const_name).should fail
57
57
  end
58
58
  end
59
-
59
+
60
60
  describe '#match(:constant)' do
61
61
  it 'recognizes constants' do
62
62
  parsing(' Foo::Bar ').as(:constant).should result_in('Foo::Bar').at(9)
63
63
  parsing(' FooBar ').as(:constant).should result_in('FooBar').at(7)
64
64
  end
65
65
  end
66
-
66
+
67
67
  describe '#match(:literal)' do
68
68
  it 'recognizes string literals' do
69
69
  parsing(%{ "a string" }).as(:literal).should result_in(%{"a string"}).at(11)
@@ -72,38 +72,38 @@ describe Rattler::Grammar::GrammarParser do
72
72
  should result_in(%q{"a \"string\""}).at(15)
73
73
  end
74
74
  end
75
-
75
+
76
76
  describe '#match(:word_literal)' do
77
77
  it 'recognizes word literals' do
78
78
  parsing(' `then` ').as(:word_literal).should result_in("`then`").at(7)
79
79
  end
80
80
  end
81
-
81
+
82
82
  describe '#match(:class_char)' do
83
83
  it 'recognizes normal characters as class characters' do
84
84
  parsing('ab').as(:class_char).should result_in('a').at(1)
85
85
  end
86
-
86
+
87
87
  it 'recognizes octal codes as class characters' do
88
88
  parsing('\\247').as(:class_char).should result_in('\\247').at(4)
89
89
  parsing('\\2474').as(:class_char).should result_in('\\247').at(4)
90
90
  end
91
-
91
+
92
92
  it 'recognizes hex codes as class characters' do
93
93
  parsing('\\x7e').as(:class_char).should result_in('\\x7e').at(4)
94
94
  parsing('\\x7ee').as(:class_char).should result_in('\\x7e').at(4)
95
95
  end
96
-
96
+
97
97
  it 'recognizes the "any" character as a class character' do
98
98
  parsing('.').as(:class_char).should result_in('.').at(1)
99
99
  end
100
-
100
+
101
101
  it 'recognizes escaped characters as class characters' do
102
102
  parsing('\\]').as(:class_char).should result_in('\\]').at(2)
103
103
  parsing('\\\\').as(:class_char).should result_in('\\\\').at(2)
104
104
  end
105
105
  end
106
-
106
+
107
107
  describe '#match(:posix_name)' do
108
108
  it 'recognizes posix names' do
109
109
  for name in posix_names
@@ -114,7 +114,7 @@ describe Rattler::Grammar::GrammarParser do
114
114
  end
115
115
  end
116
116
  end
117
-
117
+
118
118
  describe '#match(:range)' do
119
119
  it 'recognizes posix character classes as class ranges' do
120
120
  for name in posix_names
@@ -123,38 +123,38 @@ describe Rattler::Grammar::GrammarParser do
123
123
  end
124
124
  parsing('[:foo:]').as(:range).should result_in('[').at(1)
125
125
  end
126
-
126
+
127
127
  it 'recognizes normal character ranges as class ranges' do
128
128
  parsing('A-Z ').as(:range).should result_in('A-Z').at(3)
129
129
  end
130
-
130
+
131
131
  it 'recognizes class characters as class ranges' do
132
132
  parsing('ab').as(:range).should result_in('a').at(1)
133
133
  end
134
134
  end
135
-
135
+
136
136
  describe '#match(:class)' do
137
137
  it 'recognizes character classes' do
138
138
  parsing(' [A-Za-z_] ').as(:class).should result_in('[A-Za-z_]').at(10)
139
139
  end
140
140
  end
141
-
141
+
142
142
  describe '#match(:regexp)' do
143
143
  it 'recognizes regexps' do
144
144
  parsing(' /\\d+(?:\\.\\d+)?/ ').as(:regexp).
145
145
  should result_in('/\\d+(?:\\.\\d+)?/').at(16)
146
146
  end
147
147
  end
148
-
148
+
149
149
  describe '#match(:atom)' do
150
150
  it 'recognizes EOF as an eof atom' do
151
151
  parsing(' EOF ').as(:atom).should result_in(Eof[]).at(4)
152
152
  end
153
-
153
+
154
154
  it 'recognizes "." as a regexp atom' do
155
155
  parsing(' . ').as(:atom).should result_in(Match[/./]).at(2)
156
156
  end
157
-
157
+
158
158
  it 'recognizes uppercase posix character class names as regexp atoms' do
159
159
  for name in posix_names.map {|_| _.upcase } - ['WORD']
160
160
  parsing(" #{name} ").as(:atom).
@@ -162,76 +162,89 @@ describe Rattler::Grammar::GrammarParser do
162
162
  at(name.length + 1)
163
163
  end
164
164
  end
165
-
165
+
166
166
  it 'recognizes WORD as syntactic sugar for [[:alnum:]_]' do
167
167
  parsing(' WORD ').as(:atom).should result_in(Match[/[[:alnum:]_]/]).at(5)
168
168
  end
169
-
169
+
170
170
  it 'recognizes identifiers as apply atoms' do
171
171
  parsing(' expr ').as(:atom).should result_in(Apply[:expr]).at(5)
172
172
  end
173
-
173
+
174
174
  it 'recognizes string literals as match atoms' do
175
175
  parsing(%{ "a string" }).as(:atom).should result_in(Match[/a\ string/]).at(11)
176
176
  end
177
-
177
+
178
178
  it 'recognizes word literals as word atoms' do
179
179
  parsing(' `then` ').as(:atom).
180
180
  should result_in(Token[Sequence[Match[/then/], Disallow[Match[/[[:alnum:]_]/]]]]).at(7)
181
181
  end
182
-
182
+
183
183
  it 'recognizes character classes as match atoms' do
184
184
  parsing(' [A-Za-z_] ').as(:atom).should result_in(Match[/[A-Za-z_]/]).at(10)
185
185
  end
186
186
  end
187
-
187
+
188
+ describe '#match(:attributed)' do
189
+ it 'recognizes dispatch-action-attributed expressions' do
190
+ parsing(' expr <Expr> ').as(:attributed).
191
+ should result_in(DispatchAction[Apply[:expr], {:target => 'Expr', :method => 'parsed'}]).
192
+ at(12)
193
+ end
194
+
195
+ it 'recognizes direct-action-attributed expressions' do
196
+ parsing(' digits {|_| _.to_i} ').as(:attributed).
197
+ should result_in(DirectAction[Apply[:digits], '|_| _.to_i']).at(20)
198
+ end
199
+ end
200
+
188
201
  describe '#match(:term)' do
189
202
  it 'recognizes optional terms' do
190
203
  parsing(' expr? ').as(:term).should result_in(Optional[Apply[:expr]]).at(6)
191
204
  end
192
-
205
+
193
206
  it 'recognizes zero-or-more terms' do
194
207
  parsing(' expr* ').as(:term).should result_in(ZeroOrMore[Apply[:expr]]).at(6)
195
208
  end
196
-
209
+
197
210
  it 'recognizes one-or-more terms' do
198
211
  parsing(' expr+ ').as(:term).should result_in(OneOrMore[Apply[:expr]]).at(6)
199
212
  end
200
-
213
+
201
214
  it 'recognizes assert terms' do
202
215
  parsing(' &expr ').as(:term).should result_in(Assert[Apply[:expr]]).at(6)
203
216
  end
204
-
217
+
205
218
  it 'recognizes disallow terms' do
206
219
  parsing(' !expr ').as(:term).should result_in(Disallow[Apply[:expr]]).at(6)
207
220
  end
208
-
221
+
209
222
  it 'recognizes skip terms' do
210
223
  parsing(' ~expr ').as(:term).should result_in(Skip[Apply[:expr]]).at(6)
211
224
  end
212
-
225
+
213
226
  it 'recognizes token terms' do
214
227
  parsing(' @expr ').as(:term).should result_in(Token[Apply[:expr]]).at(6)
215
228
  end
216
-
229
+
217
230
  it 'recognizes labeled terms' do
218
231
  parsing(' val:expr ').as(:term).should result_in(Label[:val, Apply[:expr]]).at(9)
219
232
  end
220
-
233
+
221
234
  it 'recognizes fail expressions' do
222
235
  parsing(' fail("bad!") ').as(:term).
223
236
  should result_in(Fail[:expr, 'bad!']).at(13)
224
237
  parsing(' fail "bad!" ').as(:term).
225
238
  should result_in(Fail[:expr, 'bad!']).at(12)
226
239
  end
227
-
240
+
228
241
  it 'recognizes fail_rule expressions' do
229
242
  parsing(' fail_rule("bad!") ').as(:term).
230
243
  should result_in(Fail[:rule, 'bad!']).at(18)
231
244
  parsing(' fail_rule "bad!" ').as(:term).
232
245
  should result_in(Fail[:rule, 'bad!']).at(17)
233
246
  end
234
-
247
+
235
248
  it 'recognizes fail_parse expressions' do
236
249
  parsing(' fail_parse("bad!") ').as(:term).
237
250
  should result_in(Fail[:parse, 'bad!']).at(19)
@@ -239,60 +252,36 @@ describe Rattler::Grammar::GrammarParser do
239
252
  should result_in(Fail[:parse, 'bad!']).at(18)
240
253
  end
241
254
  end
242
-
243
- describe '#match(:attribute)' do
244
- it 'recognizes dispatch-action attributes with variables as targets' do
245
- parsing(' <expr> ').as(:attribute).should result_in(['expr']).at(7)
246
- end
247
-
248
- it 'recognizes dispatch-action attributes with constants as targets' do
249
- parsing(' <Foo::Bar> ').as(:attribute).should result_in(['Foo::Bar']).at(11)
250
- end
251
-
252
- it 'recognizes dispatch-action attributes with empty targets' do
253
- parsing(' <> ').as(:attribute).should result_in([]).at(3)
254
- end
255
- end
256
-
257
- describe '#match(:action)' do
258
- it 'recognizes symantic actions' do
259
- parsing(' {|_| _.to_f } ').as(:action).should result_in('|_| _.to_f ').at(14)
260
- end
261
-
262
- it 'recognizes shorcut symantic attributes' do
263
- parsing(' <.expr> ').as(:action).should result_in('|_| _.expr').at(8)
264
- end
265
- end
266
-
255
+
267
256
  describe '#match(:expression)' do
268
- it 'recognizes dispatch-action-attributed term expressions' do
257
+ it 'recognizes dispatch-action-attributed expressions' do
269
258
  parsing(' expr <Expr> ').as(:expression).
270
259
  should result_in(DispatchAction[Apply[:expr], {:target => 'Expr', :method => 'parsed'}]).
271
260
  at(12)
272
261
  end
273
-
274
- it 'recognizes direct-action-attributed term expressions' do
262
+
263
+ it 'recognizes direct-action-attributed expressions' do
275
264
  parsing(' digits {|_| _.to_i} ').as(:expression).
276
265
  should result_in(DirectAction[Apply[:digits], '|_| _.to_i']).at(20)
277
266
  end
278
-
267
+
279
268
  it 'recognizes sequence expressions' do
280
269
  parsing(' name "=" value ').as(:expression).
281
- should result_in(Sequence[Apply[:name], Match[/=/], Apply[:value]]).at(15)
270
+ should result_in(Sequence[Apply[:name], Match[%r{=}], Apply[:value]]).at(15)
282
271
  end
283
-
272
+
284
273
  it 'recognizes attributed sequence expressions' do
285
274
  parsing(' name "=" value <Assign>').as(:expression).
286
275
  should result_in(DispatchAction[
287
- Sequence[Apply[:name], Match[/=/], Apply[:value]],
276
+ Sequence[Apply[:name], Match[%r{=}], Apply[:value]],
288
277
  {:target => 'Assign', :method => 'parsed'}]).
289
278
  at(24)
290
279
  end
291
-
280
+
292
281
  it 'recognizes choice expressions' do
293
282
  parsing(' string | number ').as(:expression).
294
283
  should result_in(Choice[Apply[:string], Apply[:number]]).at(16)
295
284
  end
296
285
  end
297
-
286
+
298
287
  end
@@ -1,101 +1,151 @@
1
1
  require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
2
2
 
3
3
  describe Rattler::Parsers::ActionCode do
4
-
4
+
5
5
  describe '#param_names' do
6
6
  context 'when the code has block parameters' do
7
-
7
+
8
8
  subject { ActionCode.new('|a, b| a + b') }
9
-
9
+
10
10
  it 'returns an array of the parameter names' do
11
11
  subject.param_names.should == ['a', 'b']
12
12
  end
13
13
  end
14
-
14
+
15
15
  context 'when the code has no block paratmers' do
16
-
16
+
17
17
  subject { ActionCode.new('l + r') }
18
-
18
+
19
19
  it 'returns an empty array' do
20
20
  subject.param_names.should == []
21
21
  end
22
22
  end
23
23
  end
24
-
24
+
25
25
  describe '#body' do
26
26
  context 'when the code has block parameters' do
27
-
27
+
28
28
  subject { ActionCode.new('|a, b| a + b') }
29
-
29
+
30
30
  it 'returns the code after the block parameters' do
31
31
  subject.body.should == 'a + b'
32
32
  end
33
33
  end
34
-
34
+
35
35
  context 'when the code has no block paratmers' do
36
-
36
+
37
37
  subject { ActionCode.new('l + r') }
38
-
38
+
39
39
  it 'returns the code' do
40
40
  subject.body.should == 'l + r'
41
41
  end
42
42
  end
43
43
  end
44
-
44
+
45
+ describe '#blank_binding' do
46
+
47
+ subject { ActionCode.new('') }
48
+
49
+ context 'given no arguments' do
50
+ it 'returns an empty hash' do
51
+ subject.blank_binding([]).should == {}
52
+ end
53
+ end
54
+
55
+ context 'given one argument' do
56
+ it 'binds "_" to the argument' do
57
+ subject.blank_binding(['r0']).should == { '_' => 'r0' }
58
+ end
59
+ end
60
+
61
+ context 'given more than one argument' do
62
+ it 'binds "_" to the array of arguments' do
63
+ subject.blank_binding(['r0_0', 'r0_1']).
64
+ should == { '_' => '[r0_0, r0_1]' }
65
+ end
66
+ end
67
+ end
68
+
45
69
  describe '#arg_bindings' do
46
70
  context 'when the code has block parameters' do
47
-
71
+
48
72
  subject { ActionCode.new('|a, b| a + b') }
49
-
73
+
50
74
  it 'associates the parameter names with arguments' do
51
75
  subject.arg_bindings(['r0_0', 'r0_1']).
52
76
  should == {'a' => 'r0_0', 'b' => 'r0_1'}
53
77
  end
54
-
78
+
55
79
  it 'allows more args than param names' do
56
80
  subject.arg_bindings(['r0_0', 'r0_1', 'r0_2']).
57
81
  should == {'a' => 'r0_0', 'b' => 'r0_1'}
58
82
  end
59
83
  end
60
-
84
+
61
85
  context 'when the code has no block paratmers' do
62
-
86
+
63
87
  subject { ActionCode.new('l + r') }
64
-
88
+
65
89
  it 'returns an empty hash' do
66
90
  subject.arg_bindings(['r0_0', 'r0_1']).should == {}
67
91
  end
68
92
  end
69
93
  end
70
-
94
+
71
95
  describe '#bind' do
72
- context 'given an array' do
73
-
96
+ context 'when the code uses block parameters' do
97
+
74
98
  subject { ActionCode.new('|a, b| a + b') }
75
-
76
- it 'replaces block parameter names with names from the array' do
99
+
100
+ it 'replaces block parameter names with corresponding arguments' do
77
101
  subject.bind(['r0_0', 'r0_1']).should == 'r0_0 + r0_1'
78
102
  end
79
103
  end
80
-
81
- context 'given a hash' do
82
-
104
+
105
+ context 'when the code refers to labels' do
106
+
83
107
  subject { ActionCode.new('l + r') }
84
-
85
- it 'replaces label names with associated names from the hash' do
108
+
109
+ it 'replaces label names with associated arguments' do
86
110
  subject.bind({:l => 'r0_3', :r => 'r0_5'}).should == 'r0_3 + r0_5'
87
111
  end
88
112
  end
89
-
90
- context 'given an array and a hash' do
91
-
113
+
114
+ context 'when the code uses block parameters and label names' do
115
+
92
116
  subject { ActionCode.new('|a, b| a * c + b * d') }
93
-
117
+
94
118
  it 'replaces both block parameter names and label names' do
95
119
  subject.bind(['r0_0', 'r0_1'], {:c => 'r0_3', :d => 'r0_5'}).
96
120
  should == 'r0_0 * r0_3 + r0_1 * r0_5'
97
121
  end
98
122
  end
123
+
124
+ context 'when the code uses "_"' do
125
+
126
+ subject { ActionCode.new('_.to_s') }
127
+
128
+ context 'given one argument' do
129
+ it 'replaces "_" with the argument' do
130
+ subject.bind(['r0']).should == 'r0.to_s'
131
+ end
132
+ end
133
+
134
+ context 'given multiple arguments' do
135
+ it 'replaces "_" with the array of arguments' do
136
+ subject.bind(['r0_0', 'r0_1']).should == '[r0_0, r0_1].to_s'
137
+ end
138
+ end
139
+ end
140
+
141
+ context 'when the code uses "_" as a block parameter' do
142
+
143
+ subject { ActionCode.new('|_| _.to_f') }
144
+
145
+ it 'shadows the default "_" binding' do
146
+ subject.bind(['r0_0', 'r0_1']).should == 'r0_0.to_f'
147
+ end
148
+ end
99
149
  end
100
-
150
+
101
151
  end