rattler 0.2.2 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. data/README.rdoc +83 -64
  2. data/features/grammar/comments.feature +24 -0
  3. data/features/grammar/list_matching.feature +41 -0
  4. data/features/grammar/symantic_action.feature +30 -12
  5. data/lib/rattler/back_end/parser_generator/assert_generator.rb +27 -27
  6. data/lib/rattler/back_end/parser_generator/choice_generator.rb +29 -29
  7. data/lib/rattler/back_end/parser_generator/direct_action_generator.rb +17 -17
  8. data/lib/rattler/back_end/parser_generator/disallow_generator.rb +27 -27
  9. data/lib/rattler/back_end/parser_generator/dispatch_action_generator.rb +17 -17
  10. data/lib/rattler/back_end/parser_generator/expr_generator.rb +129 -40
  11. data/lib/rattler/back_end/parser_generator/label_generator.rb +15 -15
  12. data/lib/rattler/back_end/parser_generator/list1_generator.rb +61 -0
  13. data/lib/rattler/back_end/parser_generator/list_generating.rb +71 -0
  14. data/lib/rattler/back_end/parser_generator/list_generator.rb +57 -0
  15. data/lib/rattler/back_end/parser_generator/one_or_more_generator.rb +14 -15
  16. data/lib/rattler/back_end/parser_generator/optional_generator.rb +24 -24
  17. data/lib/rattler/back_end/parser_generator/predicate_propogating.rb +9 -9
  18. data/lib/rattler/back_end/parser_generator/repeat_generating.rb +16 -16
  19. data/lib/rattler/back_end/parser_generator/sequence_generator.rb +40 -40
  20. data/lib/rattler/back_end/parser_generator/skip_generator.rb +18 -18
  21. data/lib/rattler/back_end/parser_generator/skip_propogating.rb +5 -5
  22. data/lib/rattler/back_end/parser_generator/sub_generating.rb +128 -0
  23. data/lib/rattler/back_end/parser_generator/token_generator.rb +15 -15
  24. data/lib/rattler/back_end/parser_generator/token_propogating.rb +1 -1
  25. data/lib/rattler/back_end/parser_generator/zero_or_more_generator.rb +12 -13
  26. data/lib/rattler/back_end/parser_generator.rb +10 -7
  27. data/lib/rattler/grammar/grammar_parser.rb +16 -21
  28. data/lib/rattler/grammar/metagrammar.rb +1039 -1035
  29. data/lib/rattler/grammar/rattler.rtlr +28 -28
  30. data/lib/rattler/parsers/action_code.rb +20 -9
  31. data/lib/rattler/parsers/fail.rb +7 -1
  32. data/lib/rattler/parsers/list.rb +57 -0
  33. data/lib/rattler/parsers/list1.rb +58 -0
  34. data/lib/rattler/parsers/parser_dsl.rb +60 -38
  35. data/lib/rattler/parsers.rb +5 -3
  36. data/lib/rattler/runtime/extended_packrat_parser.rb +88 -20
  37. data/lib/rattler/runtime/packrat_parser.rb +21 -14
  38. data/lib/rattler/runtime/parser.rb +74 -18
  39. data/lib/rattler/runtime/recursive_descent_parser.rb +15 -46
  40. data/spec/rattler/back_end/compiler_spec.rb +173 -107
  41. data/spec/rattler/back_end/parser_generator/list1_generator_spec.rb +304 -0
  42. data/spec/rattler/back_end/parser_generator/list_generator_spec.rb +288 -0
  43. data/spec/rattler/grammar/grammar_parser_spec.rb +65 -76
  44. data/spec/rattler/parsers/action_code_spec.rb +84 -34
  45. data/spec/rattler/parsers/direct_action_spec.rb +56 -34
  46. data/spec/rattler/parsers/fail_spec.rb +20 -0
  47. data/spec/rattler/parsers/list1_spec.rb +82 -0
  48. data/spec/rattler/parsers/list_spec.rb +82 -0
  49. data/spec/rattler/parsers/parser_dsl_spec.rb +48 -19
  50. data/spec/rattler/runtime/extended_packrat_parser_spec.rb +0 -1
  51. metadata +92 -173
  52. data/bin/rtlr.bat +0 -3
  53. data/lib/rattler/back_end/parser_generator/generator_helper.rb +0 -130
  54. data/lib/rattler/back_end/parser_generator/generators.rb +0 -86
  55. data/lib/rattler/back_end/parser_generator/nested_generators.rb +0 -15
  56. data/lib/rattler/back_end/parser_generator/top_level_generators.rb +0 -15
@@ -0,0 +1,128 @@
1
+ require 'rattler/back_end/parser_generator'
2
+
3
+ module Rattler::BackEnd::ParserGenerator
4
+
5
+ # @private
6
+ module SubGenerating #:nodoc:
7
+
8
+ include Rattler::Parsers
9
+
10
+ protected
11
+
12
+ def generate(parser, as=:basic, *args)
13
+ case parser
14
+ when Eof then gen_eof
15
+ else generator(parser).send(:"gen_#{as}", parser, *args)
16
+ end
17
+ end
18
+
19
+
20
+ def gen_eof
21
+ @g << '@scanner.eos?'
22
+ end
23
+
24
+ private
25
+
26
+ def generator(parser)
27
+ case parser
28
+
29
+ when Match
30
+ cache_generator MatchGenerator
31
+
32
+ when Choice
33
+ cache_generator ChoiceGenerator, :new_choice_level => true
34
+
35
+ when Sequence
36
+ new_generator SequenceGenerator, :new_sequence_level => true
37
+
38
+ when Optional
39
+ cache_generator OptionalGenerator
40
+
41
+ when ZeroOrMore
42
+ cache_generator ZeroOrMoreGenerator, :new_repeat_level => true
43
+
44
+ when OneOrMore
45
+ cache_generator OneOrMoreGenerator, :new_repeat_level => true
46
+
47
+ when List
48
+ cache_generator ListGenerator, :new_repeat_level => true
49
+
50
+ when List1
51
+ cache_generator List1Generator, :new_repeat_level => true
52
+
53
+ when Apply
54
+ cache_generator ApplyGenerator
55
+
56
+ when Assert
57
+ cache_generator AssertGenerator
58
+
59
+ when Disallow
60
+ cache_generator DisallowGenerator
61
+
62
+ when DispatchAction
63
+ cache_generator DispatchActionGenerator
64
+
65
+ when DirectAction
66
+ cache_generator DirectActionGenerator
67
+
68
+ when Token
69
+ cache_generator TokenGenerator
70
+
71
+ when Skip
72
+ cache_generator SkipGenerator
73
+
74
+ when Label
75
+ cache_generator LabelGenerator
76
+
77
+ when Fail
78
+ cache_generator FailGenerator
79
+
80
+ end
81
+ end
82
+
83
+ def cache_generator(factory, *args)
84
+ generator_cache.fetch factory do
85
+ generator_cache[factory] = new_generator factory, *args
86
+ end
87
+ end
88
+
89
+ def new_generator(factory, opts = {})
90
+ factory.send factory_method, @g,
91
+ new_level(choice_level, opts[:new_choice_level]),
92
+ new_level(sequence_level, opts[:new_sequence_level]),
93
+ new_level(repeat_level, opts[:new_repeat_level])
94
+ end
95
+
96
+ def new_level(old_level, inc=true)
97
+ if inc
98
+ old_level ? (old_level + 1) : 0
99
+ else
100
+ old_level
101
+ end
102
+ end
103
+
104
+ def generator_cache
105
+ @generator_cache ||= {}
106
+ end
107
+
108
+ end
109
+
110
+ module NestedSubGenerating
111
+ include SubGenerating
112
+
113
+ protected
114
+ def factory_method
115
+ :nested
116
+ end
117
+ end
118
+
119
+ module TopLevelSubGenerating
120
+ include SubGenerating
121
+
122
+ protected
123
+ def factory_method
124
+ :top_level
125
+ end
126
+ end
127
+
128
+ end
@@ -1,55 +1,55 @@
1
1
  require 'rattler/back_end/parser_generator'
2
2
 
3
3
  module Rattler::BackEnd::ParserGenerator
4
-
4
+
5
5
  # @private
6
6
  class TokenGenerator < ExprGenerator #:nodoc:
7
7
  include PredicatePropogating
8
8
  include TokenPropogating
9
9
  include SkipPropogating
10
-
10
+
11
11
  def gen_basic(token)
12
- generate token.child, :gen_token
12
+ generate token.child, :token
13
13
  end
14
-
14
+
15
15
  def gen_dispatch_action_nested(token, target, method_name)
16
16
  atomic_block { gen_dispatch_action_top_level token, target, method_name }
17
17
  end
18
-
18
+
19
19
  def gen_dispatch_action_top_level(token, target, method_name)
20
20
  @g.surround("(#{result_name} = ", ')') { gen_basic token }
21
21
  (@g << ' &&').newline << dispatch_action_result(target, method_name)
22
22
  end
23
-
23
+
24
24
  def gen_direct_action_nested(token, action)
25
25
  atomic_block { gen_direct_action_top_level token, action }
26
26
  end
27
-
27
+
28
28
  def gen_direct_action_top_level(token, action)
29
29
  @g.surround("(#{result_name} = ", ')') { gen_basic token }
30
30
  (@g << ' &&').newline << direct_action_result(action)
31
31
  end
32
-
32
+
33
33
  end
34
-
34
+
35
35
  # @private
36
36
  class NestedTokenGenerator < TokenGenerator #:nodoc:
37
37
  include Nested
38
- include NestedGenerators
38
+ include NestedSubGenerating
39
39
  end
40
-
40
+
41
41
  def TokenGenerator.nested(*args)
42
42
  NestedTokenGenerator.new(*args)
43
43
  end
44
-
44
+
45
45
  # @private
46
46
  class TopLevelTokenGenerator < TokenGenerator #:nodoc:
47
47
  include TopLevel
48
- include TopLevelGenerators
48
+ include TopLevelSubGenerating
49
49
  end
50
-
50
+
51
51
  def TokenGenerator.top_level(*args)
52
52
  TopLevelTokenGenerator.new(*args)
53
53
  end
54
-
54
+
55
55
  end
@@ -4,7 +4,7 @@ module Rattler::BackEnd::ParserGenerator
4
4
  # @private
5
5
  module TokenPropogating #:nodoc:
6
6
  def gen_token(parser)
7
- generate parser.child, :gen_token
7
+ generate parser.child, :token
8
8
  end
9
9
  end
10
10
  end
@@ -1,26 +1,27 @@
1
1
  require 'rattler/back_end/parser_generator'
2
2
 
3
3
  module Rattler::BackEnd::ParserGenerator
4
-
4
+
5
5
  # @private
6
6
  class ZeroOrMoreGenerator < ExprGenerator #:nodoc:
7
7
  include RepeatGenerating
8
-
8
+ include NestedSubGenerating
9
+
9
10
  def gen_assert(optional)
10
11
  @g << 'true'
11
12
  end
12
-
13
+
13
14
  def gen_disallow(optional)
14
15
  @g << 'false'
15
16
  end
16
-
17
+
17
18
  def gen_skip_top_level(repeat)
18
19
  @g << 'while '
19
- generate repeat.child, :gen_intermediate_skip
20
+ generate repeat.child, :intermediate_skip
20
21
  (@g << '; end').newline
21
22
  @g << 'true'
22
23
  end
23
-
24
+
24
25
  protected
25
26
 
26
27
  def gen_result(captures)
@@ -28,25 +29,23 @@ module Rattler::BackEnd::ParserGenerator
28
29
  end
29
30
 
30
31
  end
31
-
32
+
32
33
  # @private
33
34
  class NestedZeroOrMoreGenerator < ZeroOrMoreGenerator #:nodoc:
34
35
  include Nested
35
- include NestedGenerators
36
36
  end
37
-
37
+
38
38
  def ZeroOrMoreGenerator.nested(*args)
39
39
  NestedZeroOrMoreGenerator.new(*args)
40
40
  end
41
-
41
+
42
42
  # @private
43
43
  class TopLevelZeroOrMoreGenerator < ZeroOrMoreGenerator #:nodoc:
44
44
  include TopLevel
45
- include NestedGenerators
46
45
  end
47
-
46
+
48
47
  def ZeroOrMoreGenerator.top_level(*args)
49
48
  TopLevelZeroOrMoreGenerator.new(*args)
50
49
  end
51
-
50
+
52
51
  end
@@ -13,14 +13,14 @@ module Rattler::BackEnd
13
13
  # @author Jason Arhart
14
14
  #
15
15
  module ParserGenerator
16
-
16
+
17
17
  autoload :RuleGenerator, 'rattler/back_end/parser_generator/rule_generator'
18
18
  autoload :ExprGenerator, 'rattler/back_end/parser_generator/expr_generator'
19
19
  autoload :GeneratorHelper, 'rattler/back_end/parser_generator/generator_helper'
20
20
  autoload :TopLevelGenerator, 'rattler/back_end/parser_generator/expr_generator'
21
- autoload :Generators, 'rattler/back_end/parser_generator/generators'
22
- autoload :NestedGenerators, 'rattler/back_end/parser_generator/nested_generators'
23
- autoload :TopLevelGenerators, 'rattler/back_end/parser_generator/top_level_generators'
21
+ autoload :SubGenerating, 'rattler/back_end/parser_generator/sub_generating'
22
+ autoload :NestedSubGenerating, 'rattler/back_end/parser_generator/sub_generating'
23
+ autoload :TopLevelSubGenerating, 'rattler/back_end/parser_generator/sub_generating'
24
24
  autoload :Nested, 'rattler/back_end/parser_generator/nested'
25
25
  autoload :TopLevel, 'rattler/back_end/parser_generator/top_level'
26
26
  autoload :MatchGenerator, 'rattler/back_end/parser_generator/match_generator'
@@ -29,6 +29,8 @@ module Rattler::BackEnd
29
29
  autoload :OptionalGenerator, 'rattler/back_end/parser_generator/optional_generator'
30
30
  autoload :ZeroOrMoreGenerator, 'rattler/back_end/parser_generator/zero_or_more_generator'
31
31
  autoload :OneOrMoreGenerator, 'rattler/back_end/parser_generator/one_or_more_generator'
32
+ autoload :ListGenerator, 'rattler/back_end/parser_generator/list_generator'
33
+ autoload :List1Generator, 'rattler/back_end/parser_generator/list1_generator'
32
34
  autoload :ApplyGenerator, 'rattler/back_end/parser_generator/apply_generator'
33
35
  autoload :AssertGenerator, 'rattler/back_end/parser_generator/assert_generator'
34
36
  autoload :DisallowGenerator, 'rattler/back_end/parser_generator/disallow_generator'
@@ -39,11 +41,12 @@ module Rattler::BackEnd
39
41
  autoload :LabelGenerator, 'rattler/back_end/parser_generator/label_generator'
40
42
  autoload :FailGenerator, 'rattler/back_end/parser_generator/fail_generator'
41
43
  autoload :RepeatGenerating, 'rattler/back_end/parser_generator/repeat_generating'
44
+ autoload :ListGenerating, 'rattler/back_end/parser_generator/list_generating'
42
45
  autoload :PredicatePropogating, 'rattler/back_end/parser_generator/predicate_propogating'
43
46
  autoload :TokenPropogating, 'rattler/back_end/parser_generator/token_propogating'
44
47
  autoload :SkipPropogating, 'rattler/back_end/parser_generator/skip_propogating'
45
48
  autoload :GEN_METHOD_NAMES, 'rattler/back_end/parser_generator/gen_method_names'
46
-
49
+
47
50
  # Generate parsing code for a parser model using a ruby generator +g+.
48
51
  #
49
52
  # @overload generate(g, grammar)
@@ -70,7 +73,7 @@ module Rattler::BackEnd
70
73
  RuleGenerator.new(g).generate(parser)
71
74
  nil
72
75
  end
73
-
76
+
74
77
  # Generate parsing code for +parser+ using a new {RubyGenerator} with the
75
78
  # given options and return the generated code.
76
79
  #
@@ -93,6 +96,6 @@ module Rattler::BackEnd
93
96
  def self.code_for(parser, options={})
94
97
  ::Rattler::BackEnd::RubyGenerator.code(options) {|g| generate(g, parser) }
95
98
  end
96
-
99
+
97
100
  end
98
101
  end
@@ -9,44 +9,39 @@ require 'rattler/grammar'
9
9
  module Rattler::Grammar
10
10
  # @private
11
11
  class GrammarParser < Rattler::Runtime::ExtendedPackratParser #:nodoc:
12
-
12
+
13
13
  include Metagrammar
14
14
  include Rattler::Parsers
15
-
15
+
16
16
  def initialize(*args)
17
17
  super
18
18
  @ws = nil
19
19
  @wc = Match[/[[:alnum:]_]/]
20
20
  @directive_stack = []
21
-
22
- @ws_stack = []
23
- @ws = nil
24
- @wc_stack = []
25
- @wc = Match[/[[:alnum:]_]/]
26
21
  end
27
-
22
+
28
23
  private
29
-
24
+
30
25
  def start_ws(e)
31
26
  @directive_stack.push(:type => :ws, :value => @ws)
32
27
  set_ws(e)
33
28
  end
34
-
29
+
35
30
  def set_ws(e)
36
31
  @ws = e
37
32
  true
38
33
  end
39
-
34
+
40
35
  def start_wc(e)
41
36
  @directive_stack.push(:type => :wc, :value => @wc)
42
37
  set_wc(e)
43
38
  end
44
-
39
+
45
40
  def set_wc(e)
46
41
  @wc = e
47
42
  true
48
43
  end
49
-
44
+
50
45
  def end_block
51
46
  if d = @directive_stack.pop
52
47
  case d[:type]
@@ -56,34 +51,34 @@ module Rattler::Grammar
56
51
  true
57
52
  end
58
53
  end
59
-
54
+
60
55
  def heading(requires, modules, includes)
61
56
  requires.merge(modules.first || {}).merge(includes)
62
57
  end
63
-
58
+
64
59
  def parser_decl(name, base)
65
60
  {:parser_name => name, :base_name => base.first}
66
61
  end
67
-
62
+
68
63
  def rule(name, parser)
69
64
  Rule[name, (@ws ? parser.with_ws(@ws) : parser)]
70
65
  end
71
-
66
+
72
67
  def literal(e)
73
68
  Match[Regexp.compile(Regexp.escape(eval(e, TOPLEVEL_BINDING)))]
74
69
  end
75
-
70
+
76
71
  def word_literal(e)
77
72
  Token[Sequence[
78
73
  Match[Regexp.compile(Regexp.escape(eval("%q#{e}", TOPLEVEL_BINDING)))],
79
74
  Disallow[@wc]
80
75
  ]]
81
76
  end
82
-
77
+
83
78
  def char_class(e)
84
79
  Match[Regexp.compile(e)]
85
80
  end
86
-
81
+
87
82
  def posix_class(name)
88
83
  if name == 'WORD'
89
84
  Match[/[[:alnum:]_]/]
@@ -91,6 +86,6 @@ module Rattler::Grammar
91
86
  char_class("[[:#{name.downcase}:]]")
92
87
  end
93
88
  end
94
-
89
+
95
90
  end
96
91
  end