rattler 0.2.2 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. data/README.rdoc +83 -64
  2. data/features/grammar/comments.feature +24 -0
  3. data/features/grammar/list_matching.feature +41 -0
  4. data/features/grammar/symantic_action.feature +30 -12
  5. data/lib/rattler/back_end/parser_generator/assert_generator.rb +27 -27
  6. data/lib/rattler/back_end/parser_generator/choice_generator.rb +29 -29
  7. data/lib/rattler/back_end/parser_generator/direct_action_generator.rb +17 -17
  8. data/lib/rattler/back_end/parser_generator/disallow_generator.rb +27 -27
  9. data/lib/rattler/back_end/parser_generator/dispatch_action_generator.rb +17 -17
  10. data/lib/rattler/back_end/parser_generator/expr_generator.rb +129 -40
  11. data/lib/rattler/back_end/parser_generator/label_generator.rb +15 -15
  12. data/lib/rattler/back_end/parser_generator/list1_generator.rb +61 -0
  13. data/lib/rattler/back_end/parser_generator/list_generating.rb +71 -0
  14. data/lib/rattler/back_end/parser_generator/list_generator.rb +57 -0
  15. data/lib/rattler/back_end/parser_generator/one_or_more_generator.rb +14 -15
  16. data/lib/rattler/back_end/parser_generator/optional_generator.rb +24 -24
  17. data/lib/rattler/back_end/parser_generator/predicate_propogating.rb +9 -9
  18. data/lib/rattler/back_end/parser_generator/repeat_generating.rb +16 -16
  19. data/lib/rattler/back_end/parser_generator/sequence_generator.rb +40 -40
  20. data/lib/rattler/back_end/parser_generator/skip_generator.rb +18 -18
  21. data/lib/rattler/back_end/parser_generator/skip_propogating.rb +5 -5
  22. data/lib/rattler/back_end/parser_generator/sub_generating.rb +128 -0
  23. data/lib/rattler/back_end/parser_generator/token_generator.rb +15 -15
  24. data/lib/rattler/back_end/parser_generator/token_propogating.rb +1 -1
  25. data/lib/rattler/back_end/parser_generator/zero_or_more_generator.rb +12 -13
  26. data/lib/rattler/back_end/parser_generator.rb +10 -7
  27. data/lib/rattler/grammar/grammar_parser.rb +16 -21
  28. data/lib/rattler/grammar/metagrammar.rb +1039 -1035
  29. data/lib/rattler/grammar/rattler.rtlr +28 -28
  30. data/lib/rattler/parsers/action_code.rb +20 -9
  31. data/lib/rattler/parsers/fail.rb +7 -1
  32. data/lib/rattler/parsers/list.rb +57 -0
  33. data/lib/rattler/parsers/list1.rb +58 -0
  34. data/lib/rattler/parsers/parser_dsl.rb +60 -38
  35. data/lib/rattler/parsers.rb +5 -3
  36. data/lib/rattler/runtime/extended_packrat_parser.rb +88 -20
  37. data/lib/rattler/runtime/packrat_parser.rb +21 -14
  38. data/lib/rattler/runtime/parser.rb +74 -18
  39. data/lib/rattler/runtime/recursive_descent_parser.rb +15 -46
  40. data/spec/rattler/back_end/compiler_spec.rb +173 -107
  41. data/spec/rattler/back_end/parser_generator/list1_generator_spec.rb +304 -0
  42. data/spec/rattler/back_end/parser_generator/list_generator_spec.rb +288 -0
  43. data/spec/rattler/grammar/grammar_parser_spec.rb +65 -76
  44. data/spec/rattler/parsers/action_code_spec.rb +84 -34
  45. data/spec/rattler/parsers/direct_action_spec.rb +56 -34
  46. data/spec/rattler/parsers/fail_spec.rb +20 -0
  47. data/spec/rattler/parsers/list1_spec.rb +82 -0
  48. data/spec/rattler/parsers/list_spec.rb +82 -0
  49. data/spec/rattler/parsers/parser_dsl_spec.rb +48 -19
  50. data/spec/rattler/runtime/extended_packrat_parser_spec.rb +0 -1
  51. metadata +92 -173
  52. data/bin/rtlr.bat +0 -3
  53. data/lib/rattler/back_end/parser_generator/generator_helper.rb +0 -130
  54. data/lib/rattler/back_end/parser_generator/generators.rb +0 -86
  55. data/lib/rattler/back_end/parser_generator/nested_generators.rb +0 -15
  56. data/lib/rattler/back_end/parser_generator/top_level_generators.rb +0 -15
@@ -1,20 +1,21 @@
1
1
  require 'rattler/back_end/parser_generator'
2
2
 
3
3
  module Rattler::BackEnd::ParserGenerator
4
-
4
+
5
5
  # @private
6
6
  class OneOrMoreGenerator < ExprGenerator #:nodoc:
7
7
  include RepeatGenerating
8
+ include NestedSubGenerating
8
9
  include PredicatePropogating
9
-
10
+
10
11
  def gen_skip_top_level(one_or_more)
11
12
  (@g << "#{result_name} = false").newline
12
13
  @g << 'while '
13
- generate one_or_more.child, :gen_intermediate_skip
14
+ generate one_or_more.child, :intermediate_skip
14
15
  @g.block('') { @g << "#{result_name} = true" }.newline
15
16
  @g << result_name
16
17
  end
17
-
18
+
18
19
  protected
19
20
 
20
21
  def gen_result(captures)
@@ -22,34 +23,32 @@ module Rattler::BackEnd::ParserGenerator
22
23
  end
23
24
 
24
25
  end
25
-
26
+
26
27
  # @private
27
28
  class NestedOneOrMoreGenerator < OneOrMoreGenerator #:nodoc:
28
29
  include Nested
29
- include NestedGenerators
30
30
  end
31
-
31
+
32
32
  def OneOrMoreGenerator.nested(*args)
33
33
  NestedOneOrMoreGenerator.new(*args)
34
34
  end
35
-
35
+
36
36
  # @private
37
37
  class TopLevelOneOrMoreGenerator < OneOrMoreGenerator #:nodoc:
38
38
  include TopLevel
39
- include NestedGenerators
40
39
 
41
40
  def gen_assert(parser)
42
- generate parser.child, :gen_assert_top_level
41
+ generate parser.child, :assert_top_level
43
42
  end
44
-
43
+
45
44
  def gen_disallow(parser)
46
- generate parser.child, :gen_disallow_top_level
45
+ generate parser.child, :disallow_top_level
47
46
  end
48
-
47
+
49
48
  end
50
-
49
+
51
50
  def OneOrMoreGenerator.top_level(*args)
52
51
  TopLevelOneOrMoreGenerator.new(*args)
53
52
  end
54
-
53
+
55
54
  end
@@ -1,15 +1,15 @@
1
1
  require 'rattler/back_end/parser_generator'
2
2
 
3
3
  module Rattler::BackEnd::ParserGenerator
4
-
4
+
5
5
  # @private
6
6
  class OptionalGenerator < ExprGenerator #:nodoc:
7
- include NestedGenerators
8
-
7
+ include NestedSubGenerating
8
+
9
9
  def gen_basic_nested(optional)
10
10
  atomic_expr { gen_basic_top_level optional }
11
11
  end
12
-
12
+
13
13
  def gen_basic_top_level(optional)
14
14
  if optional.capturing?
15
15
  @g.surround("(#{result_name} = ", ')') { generate optional.child }
@@ -18,57 +18,57 @@ module Rattler::BackEnd::ParserGenerator
18
18
  gen_skip_top_level optional
19
19
  end
20
20
  end
21
-
21
+
22
22
  def gen_assert(optional)
23
23
  @g << 'true'
24
24
  end
25
-
25
+
26
26
  def gen_disallow(optional)
27
27
  @g << 'false'
28
28
  end
29
-
29
+
30
30
  def gen_dispatch_action_nested(optional, target, method_name)
31
31
  atomic_block { gen_dispatch_action_top_level optional, target, method_name }
32
32
  end
33
-
33
+
34
34
  def gen_dispatch_action_top_level(optional, target, method_name)
35
35
  @g << "#{result_name} = "
36
36
  generate optional.child
37
37
  @g.newline << dispatch_action_result(target, method_name,
38
38
  :array_expr => "#{result_name} ? [#{result_name}] : []")
39
39
  end
40
-
40
+
41
41
  def gen_direct_action_nested(optional, action)
42
42
  atomic_block { gen_direct_action_top_level optional, action }
43
43
  end
44
-
44
+
45
45
  def gen_direct_action_top_level(optional, action)
46
46
  @g << "#{result_name} = "
47
47
  generate optional.child
48
48
  @g.newline << direct_action_result(action,
49
49
  :bind_args => ["(#{result_name} ? [#{result_name}] : [])"])
50
50
  end
51
-
51
+
52
52
  def gen_token_nested(optional)
53
53
  atomic_block { gen_token_top_level optional }
54
54
  end
55
-
55
+
56
56
  def gen_token_top_level(optional)
57
- generate optional.child, :gen_token
57
+ generate optional.child, :token
58
58
  @g << " || ''"
59
59
  end
60
-
60
+
61
61
  def gen_skip_nested(optional)
62
62
  atomic_block { gen_skip_top_level optional }
63
63
  end
64
-
64
+
65
65
  def gen_skip_top_level(optional)
66
- generate optional.child, :gen_intermediate_skip
66
+ generate optional.child, :intermediate_skip
67
67
  @g.newline << 'true'
68
68
  end
69
-
69
+
70
70
  private
71
-
71
+
72
72
  def gen_capturing(optional)
73
73
  if optional.capturing?
74
74
  yield
@@ -76,25 +76,25 @@ module Rattler::BackEnd::ParserGenerator
76
76
  gen_skip_top_level optional
77
77
  end
78
78
  end
79
-
79
+
80
80
  end
81
-
81
+
82
82
  # @private
83
83
  class NestedOptionalGenerator < OptionalGenerator #:nodoc:
84
84
  include Nested
85
85
  end
86
-
86
+
87
87
  def OptionalGenerator.nested(*args)
88
88
  NestedOptionalGenerator.new(*args)
89
89
  end
90
-
90
+
91
91
  # @private
92
92
  class TopLevelOptionalGenerator < OptionalGenerator #:nodoc:
93
93
  include TopLevel
94
94
  end
95
-
95
+
96
96
  def OptionalGenerator.top_level(*args)
97
97
  TopLevelOptionalGenerator.new(*args)
98
98
  end
99
-
99
+
100
100
  end
@@ -3,22 +3,22 @@ require 'rattler/back_end/parser_generator'
3
3
  module Rattler::BackEnd::ParserGenerator
4
4
  # @private
5
5
  module PredicatePropogating #:nodoc:
6
-
6
+
7
7
  def gen_assert(parser)
8
- generate parser.child, :gen_assert
8
+ generate parser.child, :assert
9
9
  end
10
-
10
+
11
11
  def gen_disallow(parser)
12
- generate parser.child, :gen_disallow
12
+ generate parser.child, :disallow
13
13
  end
14
-
14
+
15
15
  def gen_intermediate_assert(parser)
16
- generate parser.child, :gen_intermediate_assert
16
+ generate parser.child, :intermediate_assert
17
17
  end
18
-
18
+
19
19
  def gen_intermediate_disallow(parser)
20
- generate parser.child, :gen_intermediate_disallow
20
+ generate parser.child, :intermediate_disallow
21
21
  end
22
-
22
+
23
23
  end
24
24
  end
@@ -7,63 +7,63 @@ module Rattler::BackEnd::ParserGenerator
7
7
  def gen_basic_nested(repeat)
8
8
  atomic_block { gen_basic_top_level repeat }
9
9
  end
10
-
10
+
11
11
  def gen_basic_top_level(repeat)
12
12
  if repeat.capturing?
13
- gen_loop repeat, :gen_basic_nested
13
+ gen_loop repeat, :basic_nested
14
14
  gen_result accumulator_name
15
15
  else
16
16
  gen_skip_top_level repeat
17
17
  end
18
18
  end
19
-
19
+
20
20
  def gen_dispatch_action_nested(repeat, target, method_name)
21
21
  atomic_block do
22
22
  gen_dispatch_action_top_level repeat, target, method_name
23
23
  end
24
24
  end
25
-
25
+
26
26
  def gen_dispatch_action_top_level(repeat, target, method_name)
27
27
  gen_loop repeat
28
28
  gen_result dispatch_action_result(target, method_name,
29
29
  :array_expr => "select_captures(#{accumulator_name})")
30
30
  end
31
-
31
+
32
32
  def gen_direct_action_nested(repeat, action)
33
33
  atomic_block { gen_direct_action_top_level repeat, action }
34
34
  end
35
-
35
+
36
36
  def gen_direct_action_top_level(repeat, action)
37
37
  gen_loop repeat
38
38
  gen_result direct_action_result(action,
39
39
  :bind_args => ["select_captures(#{accumulator_name})"])
40
40
  end
41
-
41
+
42
42
  def gen_token_nested(repeat)
43
43
  atomic_block { gen_token_top_level repeat }
44
44
  end
45
-
45
+
46
46
  def gen_token_top_level(repeat)
47
- gen_loop repeat, :gen_token
47
+ gen_loop repeat, :token
48
48
  gen_result "#{accumulator_name}.join"
49
49
  end
50
-
50
+
51
51
  def gen_skip_nested(repeat)
52
52
  atomic_block { gen_skip_top_level repeat }
53
53
  end
54
-
54
+
55
55
  private
56
-
57
- def gen_loop(repeat, gen_method = :gen_basic)
56
+
57
+ def gen_loop(repeat, child_as = :basic)
58
58
  (@g << "#{accumulator_name} = []").newline
59
59
  @g << "while #{result_name} = "
60
- generate repeat.child, gen_method
60
+ generate repeat.child, child_as
61
61
  @g.block('') { @g << "#{accumulator_name} << #{result_name}" }.newline
62
62
  end
63
-
63
+
64
64
  def accumulator_name
65
65
  "a#{repeat_level}"
66
66
  end
67
-
67
+
68
68
  end
69
69
  end
@@ -1,22 +1,22 @@
1
1
  require 'rattler/back_end/parser_generator'
2
2
 
3
3
  module Rattler::BackEnd::ParserGenerator
4
-
4
+
5
5
  # @private
6
6
  class SequenceGenerator < ExprGenerator #:nodoc:
7
- include NestedGenerators
8
-
7
+ include NestedSubGenerating
8
+
9
9
  include Rattler::Parsers
10
-
10
+
11
11
  def initialize(*args)
12
12
  super
13
13
  @capture_names = []
14
14
  end
15
-
15
+
16
16
  def gen_basic_nested(sequence)
17
17
  atomic_block { gen_basic_top_level sequence }
18
18
  end
19
-
19
+
20
20
  def gen_basic_top_level(sequence)
21
21
  with_backtracking do
22
22
  if sequence.capture_count == 1 and sequence.children.last.capturing?
@@ -29,16 +29,16 @@ module Rattler::BackEnd::ParserGenerator
29
29
  end
30
30
  end
31
31
  end
32
-
32
+
33
33
  def gen_assert_nested(sequence)
34
34
  atomic_block { gen_assert_top_level sequence }
35
35
  end
36
-
36
+
37
37
  def gen_assert_top_level(sequence)
38
38
  lookahead do
39
39
  @g.block("#{result_name} = begin") do
40
40
  sequence.each do |_|
41
- @g.suffix(' &&') { generate _, :gen_intermediate_skip }.newline
41
+ @g.suffix(' &&') { generate _, :intermediate_skip }.newline
42
42
  end
43
43
  @g << "true"
44
44
  end
@@ -46,75 +46,75 @@ module Rattler::BackEnd::ParserGenerator
46
46
  end
47
47
  @g << result_name
48
48
  end
49
-
49
+
50
50
  def gen_disallow_nested(sequence)
51
51
  atomic_block { gen_disallow_top_level sequence }
52
52
  end
53
-
53
+
54
54
  def gen_disallow_top_level(sequence)
55
55
  lookahead do
56
56
  @g.block("#{result_name} = !begin") do
57
57
  @g.intersperse_nl(sequence, ' &&') do |_|
58
- generate _, :gen_intermediate_skip
58
+ generate _, :intermediate_skip
59
59
  end
60
60
  end
61
61
  @g.newline
62
62
  end
63
63
  @g << result_name
64
64
  end
65
-
65
+
66
66
  def gen_dispatch_action_nested(sequence, target, method_name)
67
67
  atomic_block do
68
68
  gen_dispatch_action_top_level sequence, target, method_name
69
69
  end
70
70
  end
71
-
71
+
72
72
  def gen_dispatch_action_top_level(sequence, target, method_name)
73
73
  gen_action_code(sequence) do |labeled|
74
74
  dispatch_action_result(target, method_name,
75
75
  :array_expr => result_array_expr(sequence), :labeled => labeled)
76
76
  end
77
77
  end
78
-
78
+
79
79
  def gen_direct_action_nested(sequence, action)
80
80
  atomic_block { gen_direct_action_top_level sequence, action }
81
81
  end
82
-
82
+
83
83
  def gen_direct_action_top_level(sequence, action)
84
84
  gen_action_code(sequence) do |labeled|
85
85
  direct_action_result(action,
86
86
  :bind_args => @capture_names, :labeled => labeled)
87
87
  end
88
88
  end
89
-
89
+
90
90
  def gen_token_nested(sequence)
91
91
  atomic_block { gen_token_top_level sequence }
92
92
  end
93
-
93
+
94
94
  def gen_token_top_level(sequence)
95
95
  with_backtracking do
96
96
  sequence.each do |_|
97
- @g.suffix(' &&') { generate _, :gen_intermediate_skip }.newline
97
+ @g.suffix(' &&') { generate _, :intermediate_skip }.newline
98
98
  end
99
99
  @g << "@scanner.string[#{saved_pos_name}...(@scanner.pos)]"
100
100
  end
101
101
  end
102
-
102
+
103
103
  def gen_skip_nested(sequence)
104
104
  atomic_block { gen_skip_top_level sequence }
105
105
  end
106
-
106
+
107
107
  def gen_skip_top_level(sequence)
108
108
  with_backtracking do
109
109
  sequence.each do |_|
110
- @g.suffix(' &&') { generate _, :gen_intermediate_skip }.newline
110
+ @g.suffix(' &&') { generate _, :intermediate_skip }.newline
111
111
  end
112
112
  @g << "true"
113
113
  end
114
114
  end
115
-
115
+
116
116
  private
117
-
117
+
118
118
  def gen_capturing(child)
119
119
  if child.capturing?
120
120
  if block_given?
@@ -123,10 +123,10 @@ module Rattler::BackEnd::ParserGenerator
123
123
  @g.surround("(#{capture_name} = ", ')') { generate child }
124
124
  end
125
125
  else
126
- generate child, :gen_intermediate
126
+ generate child, :intermediate
127
127
  end
128
128
  end
129
-
129
+
130
130
  def with_backtracking
131
131
  (@g << "#{saved_pos_name} = @scanner.pos").newline
132
132
  @g.block 'begin' do
@@ -136,7 +136,7 @@ module Rattler::BackEnd::ParserGenerator
136
136
  (@g << "@scanner.pos = #{saved_pos_name}").newline << 'false'
137
137
  end
138
138
  end
139
-
139
+
140
140
  def result_expr(sequence)
141
141
  case @capture_names.size
142
142
  when 0 then 'true'
@@ -144,7 +144,7 @@ module Rattler::BackEnd::ParserGenerator
144
144
  else result_array_expr(sequence)
145
145
  end
146
146
  end
147
-
147
+
148
148
  def result_array_expr(sequence)
149
149
  if sequence.any? {|_| Apply === _ }
150
150
  "select_captures(#{capture_names_expr})"
@@ -152,7 +152,7 @@ module Rattler::BackEnd::ParserGenerator
152
152
  capture_names_expr
153
153
  end
154
154
  end
155
-
155
+
156
156
  def gen_action_code(sequence)
157
157
  with_backtracking do
158
158
  labeled = {}
@@ -163,44 +163,44 @@ module Rattler::BackEnd::ParserGenerator
163
163
  @g << yield(labeled)
164
164
  end
165
165
  end
166
-
166
+
167
167
  def capture_names_expr
168
168
  '[' + @capture_names.join(', ') + ']'
169
169
  end
170
-
170
+
171
171
  def capture_name
172
172
  @capture_names << "r#{sequence_level}_#{@capture_names.size}"
173
173
  @capture_names.last
174
174
  end
175
-
175
+
176
176
  def last_capture_name
177
177
  @capture_names.last
178
178
  end
179
-
179
+
180
180
  attr_reader :capture_names
181
-
181
+
182
182
  def saved_pos_name
183
183
  "p#{sequence_level}"
184
184
  end
185
-
185
+
186
186
  end
187
-
187
+
188
188
  # @private
189
189
  class NestedSequenceGenerator < SequenceGenerator #:nodoc:
190
190
  include Nested
191
191
  end
192
-
192
+
193
193
  def SequenceGenerator.nested(*args)
194
194
  NestedSequenceGenerator.new(*args)
195
195
  end
196
-
196
+
197
197
  # @private
198
198
  class TopLevelSequenceGenerator < SequenceGenerator #:nodoc:
199
199
  include TopLevel
200
200
  end
201
-
201
+
202
202
  def SequenceGenerator.top_level(*args)
203
203
  TopLevelSequenceGenerator.new(*args)
204
204
  end
205
-
205
+
206
206
  end
@@ -1,65 +1,65 @@
1
1
  require 'rattler/back_end/parser_generator'
2
2
 
3
3
  module Rattler::BackEnd::ParserGenerator
4
-
4
+
5
5
  # @private
6
6
  class SkipGenerator < ExprGenerator #:nodoc:
7
7
  include PredicatePropogating
8
8
  include TokenPropogating
9
9
  include SkipPropogating
10
-
10
+
11
11
  def gen_basic(skip)
12
- generate skip.child, :gen_skip
12
+ generate skip.child, :skip
13
13
  end
14
-
14
+
15
15
  def gen_dispatch_action_nested(skip, target, method_name)
16
16
  atomic_block { gen_dispatch_action_top_level skip, target, method_name }
17
17
  end
18
-
18
+
19
19
  def gen_dispatch_action_top_level(skip, target, method_name)
20
20
  gen_intermediate_skip skip
21
21
  (@g << ' &&').newline
22
22
  @g << dispatch_action_result(target, method_name, :array_expr => '[]')
23
23
  end
24
-
24
+
25
25
  def gen_direct_action_nested(skip, action)
26
26
  atomic_block { gen_direct_action_top_level skip, action }
27
27
  end
28
-
28
+
29
29
  def gen_direct_action_top_level(skip, action)
30
30
  gen_intermediate_skip skip
31
31
  (@g << ' &&').newline
32
32
  @g << direct_action_result(action, :bind_args => [])
33
33
  end
34
-
34
+
35
35
  def gen_intermediate(skip)
36
- generate skip.child, :gen_intermediate_skip
36
+ generate skip.child, :intermediate_skip
37
37
  end
38
-
38
+
39
39
  def gen_intermediate_skip(skip)
40
40
  gen_intermediate skip
41
41
  end
42
-
42
+
43
43
  end
44
-
44
+
45
45
  # @private
46
46
  class NestedSkipGenerator < SkipGenerator #:nodoc:
47
47
  include Nested
48
- include NestedGenerators
48
+ include NestedSubGenerating
49
49
  end
50
-
50
+
51
51
  def SkipGenerator.nested(*args)
52
52
  NestedSkipGenerator.new(*args)
53
53
  end
54
-
54
+
55
55
  # @private
56
56
  class TopLevelSkipGenerator < SkipGenerator #:nodoc:
57
57
  include TopLevel
58
- include TopLevelGenerators
58
+ include TopLevelSubGenerating
59
59
  end
60
-
60
+
61
61
  def SkipGenerator.top_level(*args)
62
62
  TopLevelSkipGenerator.new(*args)
63
63
  end
64
-
64
+
65
65
  end
@@ -3,14 +3,14 @@ require 'rattler/back_end/parser_generator'
3
3
  module Rattler::BackEnd::ParserGenerator
4
4
  # @private
5
5
  module SkipPropogating #:nodoc:
6
-
6
+
7
7
  def gen_skip(parser)
8
- generate parser.child, :gen_skip
8
+ generate parser.child, :skip
9
9
  end
10
-
10
+
11
11
  def gen_intermediate_skip(parser)
12
- generate parser.child, :gen_intermediate_skip
12
+ generate parser.child, :intermediate_skip
13
13
  end
14
-
14
+
15
15
  end
16
16
  end