rattler 0.2.2 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. data/README.rdoc +83 -64
  2. data/features/grammar/comments.feature +24 -0
  3. data/features/grammar/list_matching.feature +41 -0
  4. data/features/grammar/symantic_action.feature +30 -12
  5. data/lib/rattler/back_end/parser_generator/assert_generator.rb +27 -27
  6. data/lib/rattler/back_end/parser_generator/choice_generator.rb +29 -29
  7. data/lib/rattler/back_end/parser_generator/direct_action_generator.rb +17 -17
  8. data/lib/rattler/back_end/parser_generator/disallow_generator.rb +27 -27
  9. data/lib/rattler/back_end/parser_generator/dispatch_action_generator.rb +17 -17
  10. data/lib/rattler/back_end/parser_generator/expr_generator.rb +129 -40
  11. data/lib/rattler/back_end/parser_generator/label_generator.rb +15 -15
  12. data/lib/rattler/back_end/parser_generator/list1_generator.rb +61 -0
  13. data/lib/rattler/back_end/parser_generator/list_generating.rb +71 -0
  14. data/lib/rattler/back_end/parser_generator/list_generator.rb +57 -0
  15. data/lib/rattler/back_end/parser_generator/one_or_more_generator.rb +14 -15
  16. data/lib/rattler/back_end/parser_generator/optional_generator.rb +24 -24
  17. data/lib/rattler/back_end/parser_generator/predicate_propogating.rb +9 -9
  18. data/lib/rattler/back_end/parser_generator/repeat_generating.rb +16 -16
  19. data/lib/rattler/back_end/parser_generator/sequence_generator.rb +40 -40
  20. data/lib/rattler/back_end/parser_generator/skip_generator.rb +18 -18
  21. data/lib/rattler/back_end/parser_generator/skip_propogating.rb +5 -5
  22. data/lib/rattler/back_end/parser_generator/sub_generating.rb +128 -0
  23. data/lib/rattler/back_end/parser_generator/token_generator.rb +15 -15
  24. data/lib/rattler/back_end/parser_generator/token_propogating.rb +1 -1
  25. data/lib/rattler/back_end/parser_generator/zero_or_more_generator.rb +12 -13
  26. data/lib/rattler/back_end/parser_generator.rb +10 -7
  27. data/lib/rattler/grammar/grammar_parser.rb +16 -21
  28. data/lib/rattler/grammar/metagrammar.rb +1039 -1035
  29. data/lib/rattler/grammar/rattler.rtlr +28 -28
  30. data/lib/rattler/parsers/action_code.rb +20 -9
  31. data/lib/rattler/parsers/fail.rb +7 -1
  32. data/lib/rattler/parsers/list.rb +57 -0
  33. data/lib/rattler/parsers/list1.rb +58 -0
  34. data/lib/rattler/parsers/parser_dsl.rb +60 -38
  35. data/lib/rattler/parsers.rb +5 -3
  36. data/lib/rattler/runtime/extended_packrat_parser.rb +88 -20
  37. data/lib/rattler/runtime/packrat_parser.rb +21 -14
  38. data/lib/rattler/runtime/parser.rb +74 -18
  39. data/lib/rattler/runtime/recursive_descent_parser.rb +15 -46
  40. data/spec/rattler/back_end/compiler_spec.rb +173 -107
  41. data/spec/rattler/back_end/parser_generator/list1_generator_spec.rb +304 -0
  42. data/spec/rattler/back_end/parser_generator/list_generator_spec.rb +288 -0
  43. data/spec/rattler/grammar/grammar_parser_spec.rb +65 -76
  44. data/spec/rattler/parsers/action_code_spec.rb +84 -34
  45. data/spec/rattler/parsers/direct_action_spec.rb +56 -34
  46. data/spec/rattler/parsers/fail_spec.rb +20 -0
  47. data/spec/rattler/parsers/list1_spec.rb +82 -0
  48. data/spec/rattler/parsers/list_spec.rb +82 -0
  49. data/spec/rattler/parsers/parser_dsl_spec.rb +48 -19
  50. data/spec/rattler/runtime/extended_packrat_parser_spec.rb +0 -1
  51. metadata +92 -173
  52. data/bin/rtlr.bat +0 -3
  53. data/lib/rattler/back_end/parser_generator/generator_helper.rb +0 -130
  54. data/lib/rattler/back_end/parser_generator/generators.rb +0 -86
  55. data/lib/rattler/back_end/parser_generator/nested_generators.rb +0 -15
  56. data/lib/rattler/back_end/parser_generator/top_level_generators.rb +0 -15
@@ -0,0 +1,304 @@
1
+ require File.expand_path(File.dirname(__FILE__) + '/../../../spec_helper')
2
+
3
+ include Rattler::Parsers
4
+
5
+ describe Rattler::BackEnd::ParserGenerator::List1Generator do
6
+
7
+ include ParserGeneratorSpecHelper
8
+
9
+ let(:list) { List1[Match[/w+/], Match[/[,;]/]] }
10
+
11
+ describe '#gen_basic' do
12
+
13
+ let :list do
14
+ List1[Choice[Match[/[[:alpha:]]/], Match[/[[:digit:]]/]], Match[/[,;]/]]
15
+ end
16
+
17
+ context 'when nested' do
18
+ it 'generates nested list1 matching code' do
19
+ nested_code {|g| g.gen_basic list }.
20
+ should == (<<-CODE).strip
21
+ begin
22
+ a = []
23
+ lp = nil
24
+ while r = begin
25
+ @scanner.scan(/[[:alpha:]]/) ||
26
+ @scanner.scan(/[[:digit:]]/)
27
+ end
28
+ lp = @scanner.pos
29
+ a << r
30
+ break unless @scanner.skip(/[,;]/)
31
+ end
32
+ @scanner.pos = lp unless lp.nil?
33
+ a unless a.empty?
34
+ end
35
+ CODE
36
+ end
37
+ end
38
+
39
+ context 'when top-level' do
40
+ it 'generates top-level list1 matching code' do
41
+ top_level_code {|g| g.gen_basic list }.
42
+ should == (<<-CODE).strip
43
+ a = []
44
+ lp = nil
45
+ while r = begin
46
+ @scanner.scan(/[[:alpha:]]/) ||
47
+ @scanner.scan(/[[:digit:]]/)
48
+ end
49
+ lp = @scanner.pos
50
+ a << r
51
+ break unless @scanner.skip(/[,;]/)
52
+ end
53
+ @scanner.pos = lp unless lp.nil?
54
+ a unless a.empty?
55
+ CODE
56
+ end
57
+ end
58
+
59
+ context 'with a non-capturing parser' do
60
+
61
+ let(:list) { List1[Skip[Match[/w+/]], Match[/[,;]/]] }
62
+
63
+ context 'when nested' do
64
+ it 'generates nested list skipping code' do
65
+ nested_code {|g| g.gen_basic list }.
66
+ should == (<<-CODE).strip
67
+ begin
68
+ r = false
69
+ lp = nil
70
+ while @scanner.skip(/w+/)
71
+ r = true
72
+ lp = @scanner.pos
73
+ break unless @scanner.skip(/[,;]/)
74
+ end
75
+ @scanner.pos = lp unless lp.nil?
76
+ r
77
+ end
78
+ CODE
79
+ end
80
+ end
81
+
82
+ context 'when top-level' do
83
+ it 'generates nested list skipping code' do
84
+ top_level_code {|g| g.gen_basic list }.
85
+ should == (<<-CODE).strip
86
+ r = false
87
+ lp = nil
88
+ while @scanner.skip(/w+/)
89
+ r = true
90
+ lp = @scanner.pos
91
+ break unless @scanner.skip(/[,;]/)
92
+ end
93
+ @scanner.pos = lp unless lp.nil?
94
+ r
95
+ CODE
96
+ end
97
+ end
98
+ end
99
+ end
100
+
101
+ describe '#gen_assert' do
102
+
103
+ context 'when nested' do
104
+ it 'generates nested positive lookahead code' do
105
+ nested_code {|g| g.gen_assert list }.
106
+ should == '(@scanner.skip(/(?=w+)/) && true)'
107
+ end
108
+ end
109
+
110
+ context 'when top-level' do
111
+ it 'generates top-level positive lookahead code' do
112
+ top_level_code {|g| g.gen_assert list }.
113
+ should == '@scanner.skip(/(?=w+)/) && true'
114
+ end
115
+ end
116
+ end
117
+
118
+ describe '#gen_disallow' do
119
+
120
+ context 'when nested' do
121
+ it 'generates nested one-or-more negative lookahead code' do
122
+ nested_code {|g| g.gen_disallow list }.
123
+ should == '(@scanner.skip(/(?!w+)/) && true)'
124
+ end
125
+ end
126
+
127
+ context 'when top-level' do
128
+ it 'generates top-level one-or-more negative lookahead code' do
129
+ top_level_code {|g| g.gen_disallow list }.
130
+ should == '@scanner.skip(/(?!w+)/) && true'
131
+ end
132
+ end
133
+ end
134
+
135
+ describe '#gen_dispatch_action' do
136
+
137
+ context 'when nested' do
138
+ it 'generates nested list matching code with a dispatch action' do
139
+ nested_code {|g| g.gen_dispatch_action list, 'Word', 'parsed' }.
140
+ should == (<<-CODE).strip
141
+ begin
142
+ a = []
143
+ lp = nil
144
+ while r = @scanner.scan(/w+/)
145
+ lp = @scanner.pos
146
+ a << r
147
+ break unless @scanner.skip(/[,;]/)
148
+ end
149
+ @scanner.pos = lp unless lp.nil?
150
+ Word.parsed(select_captures(a)) unless a.empty?
151
+ end
152
+ CODE
153
+ end
154
+ end
155
+
156
+ context 'when top-level' do
157
+ it 'generates top level list matching code with a dispatch action' do
158
+ top_level_code {|g| g.gen_dispatch_action list, 'Word', 'parsed' }.
159
+ should == (<<-CODE).strip
160
+ a = []
161
+ lp = nil
162
+ while r = @scanner.scan(/w+/)
163
+ lp = @scanner.pos
164
+ a << r
165
+ break unless @scanner.skip(/[,;]/)
166
+ end
167
+ @scanner.pos = lp unless lp.nil?
168
+ Word.parsed(select_captures(a)) unless a.empty?
169
+ CODE
170
+ end
171
+ end
172
+ end
173
+
174
+ describe '#gen_direct_action' do
175
+
176
+ context 'when nested' do
177
+ it 'generates nested list matching code with a dispatch action' do
178
+ nested_code {|g| g.gen_direct_action list, ActionCode.new('|_| _.size') }.
179
+ should == (<<-CODE).strip
180
+ begin
181
+ a = []
182
+ lp = nil
183
+ while r = @scanner.scan(/w+/)
184
+ lp = @scanner.pos
185
+ a << r
186
+ break unless @scanner.skip(/[,;]/)
187
+ end
188
+ @scanner.pos = lp unless lp.nil?
189
+ (select_captures(a).size) unless a.empty?
190
+ end
191
+ CODE
192
+ end
193
+ end
194
+
195
+ context 'when top-level' do
196
+ it 'generates top level list matching code with a dispatch action' do
197
+ top_level_code {|g| g.gen_direct_action list, ActionCode.new('|_| _.size') }.
198
+ should == (<<-CODE).strip
199
+ a = []
200
+ lp = nil
201
+ while r = @scanner.scan(/w+/)
202
+ lp = @scanner.pos
203
+ a << r
204
+ break unless @scanner.skip(/[,;]/)
205
+ end
206
+ @scanner.pos = lp unless lp.nil?
207
+ (select_captures(a).size) unless a.empty?
208
+ CODE
209
+ end
210
+ end
211
+ end
212
+
213
+ describe '#gen_skip' do
214
+
215
+ context 'when nested' do
216
+ it 'generates nested list skipping code' do
217
+ nested_code {|g| g.gen_skip list }.
218
+ should == (<<-CODE).strip
219
+ begin
220
+ r = false
221
+ lp = nil
222
+ while @scanner.skip(/w+/)
223
+ r = true
224
+ lp = @scanner.pos
225
+ break unless @scanner.skip(/[,;]/)
226
+ end
227
+ @scanner.pos = lp unless lp.nil?
228
+ r
229
+ end
230
+ CODE
231
+ end
232
+ end
233
+
234
+ context 'when top-level' do
235
+ it 'generates nested list skipping code' do
236
+ top_level_code {|g| g.gen_skip list }.
237
+ should == (<<-CODE).strip
238
+ r = false
239
+ lp = nil
240
+ while @scanner.skip(/w+/)
241
+ r = true
242
+ lp = @scanner.pos
243
+ break unless @scanner.skip(/[,;]/)
244
+ end
245
+ @scanner.pos = lp unless lp.nil?
246
+ r
247
+ CODE
248
+ end
249
+ end
250
+ end
251
+
252
+ describe '#gen_intermediate' do
253
+ it 'generates nested list matching code' do
254
+ nested_code {|g| g.gen_intermediate list }.
255
+ should == (<<-CODE).strip
256
+ begin
257
+ a = []
258
+ lp = nil
259
+ while r = @scanner.scan(/w+/)
260
+ lp = @scanner.pos
261
+ a << r
262
+ break unless @scanner.skip(/[,;]/)
263
+ end
264
+ @scanner.pos = lp unless lp.nil?
265
+ a unless a.empty?
266
+ end
267
+ CODE
268
+ end
269
+ end
270
+
271
+ describe '#gen_intermediate_assert' do
272
+ it 'generates nested positive lookahead code' do
273
+ nested_code {|g| g.gen_assert list }.
274
+ should == '(@scanner.skip(/(?=w+)/) && true)'
275
+ end
276
+ end
277
+
278
+ describe '#gen_intermediate_disallow' do
279
+ it 'generates nested one-or-more negative lookahead code' do
280
+ nested_code {|g| g.gen_disallow list }.
281
+ should == '(@scanner.skip(/(?!w+)/) && true)'
282
+ end
283
+ end
284
+
285
+ describe '#gen_intermediate_skip' do
286
+ it 'generates nested list skipping code' do
287
+ nested_code {|g| g.gen_intermediate_skip list }.
288
+ should == (<<-CODE).strip
289
+ begin
290
+ r = false
291
+ lp = nil
292
+ while @scanner.skip(/w+/)
293
+ r = true
294
+ lp = @scanner.pos
295
+ break unless @scanner.skip(/[,;]/)
296
+ end
297
+ @scanner.pos = lp unless lp.nil?
298
+ r
299
+ end
300
+ CODE
301
+ end
302
+ end
303
+
304
+ end
@@ -0,0 +1,288 @@
1
+ require File.expand_path(File.dirname(__FILE__) + '/../../../spec_helper')
2
+
3
+ include Rattler::Parsers
4
+
5
+ describe Rattler::BackEnd::ParserGenerator::ListGenerator do
6
+
7
+ include ParserGeneratorSpecHelper
8
+
9
+ let(:list) { List[Match[/w+/], Match[/[,;]/]] }
10
+
11
+ describe '#gen_basic' do
12
+
13
+ let :list do
14
+ List[Choice[Match[/[[:alpha:]]/], Match[/[[:digit:]]/]], Match[/[,;]/]]
15
+ end
16
+
17
+ context 'when nested' do
18
+ it 'generates nested list matching code' do
19
+ nested_code {|g| g.gen_basic list }.
20
+ should == (<<-CODE).strip
21
+ begin
22
+ a = []
23
+ lp = nil
24
+ while r = begin
25
+ @scanner.scan(/[[:alpha:]]/) ||
26
+ @scanner.scan(/[[:digit:]]/)
27
+ end
28
+ lp = @scanner.pos
29
+ a << r
30
+ break unless @scanner.skip(/[,;]/)
31
+ end
32
+ @scanner.pos = lp unless lp.nil?
33
+ a
34
+ end
35
+ CODE
36
+ end
37
+ end
38
+
39
+ context 'when top-level' do
40
+ it 'generates top-level list matching code' do
41
+ top_level_code {|g| g.gen_basic list }.
42
+ should == (<<-CODE).strip
43
+ a = []
44
+ lp = nil
45
+ while r = begin
46
+ @scanner.scan(/[[:alpha:]]/) ||
47
+ @scanner.scan(/[[:digit:]]/)
48
+ end
49
+ lp = @scanner.pos
50
+ a << r
51
+ break unless @scanner.skip(/[,;]/)
52
+ end
53
+ @scanner.pos = lp unless lp.nil?
54
+ a
55
+ CODE
56
+ end
57
+ end
58
+
59
+ context 'with a non-capturing parser' do
60
+
61
+ let(:list) { List[Skip[Match[/w+/]], Match[/[,;]/]] }
62
+
63
+ context 'when nested' do
64
+ it 'generates nested list skipping code' do
65
+ nested_code {|g| g.gen_basic list }.
66
+ should == (<<-CODE).strip
67
+ begin
68
+ lp = nil
69
+ while @scanner.skip(/w+/)
70
+ lp = @scanner.pos
71
+ break unless @scanner.skip(/[,;]/)
72
+ end
73
+ @scanner.pos = lp unless lp.nil?
74
+ true
75
+ end
76
+ CODE
77
+ end
78
+ end
79
+
80
+ context 'when top-level' do
81
+ it 'generates nested list skipping code' do
82
+ top_level_code {|g| g.gen_basic list }.
83
+ should == (<<-CODE).strip
84
+ lp = nil
85
+ while @scanner.skip(/w+/)
86
+ lp = @scanner.pos
87
+ break unless @scanner.skip(/[,;]/)
88
+ end
89
+ @scanner.pos = lp unless lp.nil?
90
+ true
91
+ CODE
92
+ end
93
+ end
94
+ end
95
+ end
96
+
97
+ describe '#gen_assert' do
98
+
99
+ context 'when nested' do
100
+ it 'generates "true"' do
101
+ nested_code {|g| g.gen_assert list }.should == 'true'
102
+ end
103
+ end
104
+
105
+ context 'when top-level' do
106
+ it 'generates "true"' do
107
+ top_level_code {|g| g.gen_assert list }.should == 'true'
108
+ end
109
+ end
110
+ end
111
+
112
+ describe '#gen_disallow' do
113
+
114
+ context 'when nested' do
115
+ it 'generates "false"' do
116
+ nested_code {|g| g.gen_disallow list }.should == 'false'
117
+ end
118
+ end
119
+
120
+ context 'when top-level' do
121
+ it 'generates "false"' do
122
+ top_level_code {|g| g.gen_disallow list }.should == 'false'
123
+ end
124
+ end
125
+ end
126
+
127
+ describe '#gen_dispatch_action' do
128
+
129
+ context 'when nested' do
130
+ it 'generates nested list matching code with a dispatch action' do
131
+ nested_code {|g| g.gen_dispatch_action list, 'Word', 'parsed' }.
132
+ should == (<<-CODE).strip
133
+ begin
134
+ a = []
135
+ lp = nil
136
+ while r = @scanner.scan(/w+/)
137
+ lp = @scanner.pos
138
+ a << r
139
+ break unless @scanner.skip(/[,;]/)
140
+ end
141
+ @scanner.pos = lp unless lp.nil?
142
+ Word.parsed(select_captures(a))
143
+ end
144
+ CODE
145
+ end
146
+ end
147
+
148
+ context 'when top-level' do
149
+ it 'generates top level list matching code with a dispatch action' do
150
+ top_level_code {|g| g.gen_dispatch_action list, 'Word', 'parsed' }.
151
+ should == (<<-CODE).strip
152
+ a = []
153
+ lp = nil
154
+ while r = @scanner.scan(/w+/)
155
+ lp = @scanner.pos
156
+ a << r
157
+ break unless @scanner.skip(/[,;]/)
158
+ end
159
+ @scanner.pos = lp unless lp.nil?
160
+ Word.parsed(select_captures(a))
161
+ CODE
162
+ end
163
+ end
164
+ end
165
+
166
+ describe '#gen_direct_action' do
167
+
168
+ context 'when nested' do
169
+ it 'generates nested list matching code with a dispatch action' do
170
+ nested_code {|g| g.gen_direct_action list, ActionCode.new('|_| _.size') }.
171
+ should == (<<-CODE).strip
172
+ begin
173
+ a = []
174
+ lp = nil
175
+ while r = @scanner.scan(/w+/)
176
+ lp = @scanner.pos
177
+ a << r
178
+ break unless @scanner.skip(/[,;]/)
179
+ end
180
+ @scanner.pos = lp unless lp.nil?
181
+ (select_captures(a).size)
182
+ end
183
+ CODE
184
+ end
185
+ end
186
+
187
+ context 'when top-level' do
188
+ it 'generates top level list matching code with a dispatch action' do
189
+ top_level_code {|g| g.gen_direct_action list, ActionCode.new('|_| _.size') }.
190
+ should == (<<-CODE).strip
191
+ a = []
192
+ lp = nil
193
+ while r = @scanner.scan(/w+/)
194
+ lp = @scanner.pos
195
+ a << r
196
+ break unless @scanner.skip(/[,;]/)
197
+ end
198
+ @scanner.pos = lp unless lp.nil?
199
+ (select_captures(a).size)
200
+ CODE
201
+ end
202
+ end
203
+ end
204
+
205
+ describe '#gen_skip' do
206
+
207
+ context 'when nested' do
208
+ it 'generates nested list skipping code' do
209
+ nested_code {|g| g.gen_skip list }.
210
+ should == (<<-CODE).strip
211
+ begin
212
+ lp = nil
213
+ while @scanner.skip(/w+/)
214
+ lp = @scanner.pos
215
+ break unless @scanner.skip(/[,;]/)
216
+ end
217
+ @scanner.pos = lp unless lp.nil?
218
+ true
219
+ end
220
+ CODE
221
+ end
222
+ end
223
+
224
+ context 'when top-level' do
225
+ it 'generates top-level list skipping code' do
226
+ top_level_code {|g| g.gen_skip list }.
227
+ should == (<<-CODE).strip
228
+ lp = nil
229
+ while @scanner.skip(/w+/)
230
+ lp = @scanner.pos
231
+ break unless @scanner.skip(/[,;]/)
232
+ end
233
+ @scanner.pos = lp unless lp.nil?
234
+ true
235
+ CODE
236
+ end
237
+ end
238
+ end
239
+
240
+ describe '#gen_intermediate' do
241
+ it 'generates nested list matching code' do
242
+ nested_code {|g| g.gen_intermediate list }.
243
+ should == (<<-CODE).strip
244
+ begin
245
+ a = []
246
+ lp = nil
247
+ while r = @scanner.scan(/w+/)
248
+ lp = @scanner.pos
249
+ a << r
250
+ break unless @scanner.skip(/[,;]/)
251
+ end
252
+ @scanner.pos = lp unless lp.nil?
253
+ a
254
+ end
255
+ CODE
256
+ end
257
+ end
258
+
259
+ describe '#gen_intermediate_assert' do
260
+ it 'generates "true"' do
261
+ nested_code {|g| g.gen_intermediate_assert list }.should == 'true'
262
+ end
263
+ end
264
+
265
+ describe '#gen_intermediate_disallow' do
266
+ it 'generates "false"' do
267
+ nested_code {|g| g.gen_intermediate_disallow list }.should == 'false'
268
+ end
269
+ end
270
+
271
+ describe '#gen_intermediate_skip' do
272
+ it 'generates nested list skipping code' do
273
+ nested_code {|g| g.gen_intermediate_skip list }.
274
+ should == (<<-CODE).strip
275
+ begin
276
+ lp = nil
277
+ while @scanner.skip(/w+/)
278
+ lp = @scanner.pos
279
+ break unless @scanner.skip(/[,;]/)
280
+ end
281
+ @scanner.pos = lp unless lp.nil?
282
+ true
283
+ end
284
+ CODE
285
+ end
286
+ end
287
+
288
+ end