rattler 0.2.2 → 0.3.0

data/README.rdoc

@@ -14,13 +14,13 @@ A language syntax is specified in a grammar using the Rattler syntax. Parser
 classes and modules can be generated statically using the "rtlr" command or
 dynamically from strings.
 
-{RDoc}[http://rubydoc.info/gems/rattler/0.2.1/frames]
+{RDoc}[http://rubydoc.info/gems/rattler/0.2.2/frames]
 
 == FEATURES:
 
 * Uses readable PEG-like grammars
 * A packrat parser to cache results for backtracking
-* A parser that supports directly left-recursive grammars
+* An extended packrat parser that supports left-recursive grammars
 * Whitespace can be specified in one place and skipped automatically
 * Automatic and custom parse error reporting
 * Optimizing parser generator
@@ -30,91 +30,109 @@ dynamically from strings.
 
 == PROBLEMS/LIMITATIONS:
 
-* Indirect left-recursion is not yet supported
 * Optimizations are very limited at this point
 * Only strings can be parsed, so files have to be read completely before parsing
-* There are undoubtedly many bugs
+* There are many holes in the tests so there are undoubtedly many bugs
 
 == EXAMPLES:
 
 === Example 1: Statically generated parser class
 
-==== In file calculator.rtlr:
+==== json.rtlr:
   
-  parser Calculator < Rattler::Runtime::WDMParser
-  
-  %whitespace SPACE*
-  
-  start   <-  expr EOF
-  
-  expr    <-  expr ~'+' term            {|a, b| a + b }
-            | expr ~'-' term            {|a, b| a - b }
-            | term
-  
-  term    <-  term ~'*' primary         {|a, b| a * b }
-            | term ~'/' primary         {|a, b| a / b }
-            | primary
-  
-  primary <-  ~'(' expr ~')'
-            | @(DIGIT+ ('.' DIGIT+)?)   {|_| _.to_f }
-  
-==== In a shell:
+  # JSON parser based on the grammar at http://www.json.org
+
+  parser JsonParser < Rattler::Runtime::PackratParser
+
+  %whitespace (SPACE+ | '/*' (! '*/' .)* '*/' | '//' [^\n]*)*
+
+  object    <-  ~'{' members ~'}'                                   { object _ }
+
+  members   <-  pair *^ ','
+
+  pair      <-  string ~':' value
+
+  array     <-  ~'[' elements ~']'
+
+  elements  <-  value *^ ','
+
+  value     <-  string                                              { string _ }
+              | number !DIGIT                                       { _.to_f }
+              | object
+              | array
+              | `true`                                              { :true }
+              | `false`                                             { :false }
+              | `null`                                              { :null }
+              | fail "value expected"
+
+  string    <-  @('"' ('\\' . | [^"])* '"')
+
+  number    <-  @('-'? ('0' | [1-9] DIGIT*) ('.' DIGIT+)? ([eE] [+-]? DIGIT+)?)
+
+==== json_helper.rb:
+
+  module JsonHelper
+
+    def object(members)
+      Hash[*members.flatten(1)]
+    end
+
+    def string(expr)
+      eval expr, TOPLEVEL_BINDING
+    end
+
+  end
+
+==== $ rtlr json.rtlr
 
-  $ rtlr calculator.rtlr
-  calculator.rtlr -> calculator.rb
+  json.rtlr -> json_parser.rb
 
-==== In file calculator.rb:
+==== parse_json.rb:
 
   require 'rubygems'
   require 'rattler'
-  require 'calculator'
+  require 'json_parser'
   
   begin
-    puts Calculator.parse!(expr)
+    p JsonParser.parse!(open(ARGV[0]) {|io| io.read })
   rescue Rattler::Runtime::SyntaxError => e
     puts e
   end
 
 === Example 2: Statically generated grammar module
 
-==== calculator.rtlr:
-
-  grammar CalculatorGrammar
-  
-  %whitespace SPACE*
-  
-  start   <-  expr EOF
+==== json.rtlr:
   
-  expr    <-  expr ~'+' term            {|a, b| a + b }
-            | expr ~'-' term            {|a, b| a - b }
-            | term
-  
-  term    <-  term ~'*' primary         {|a, b| a * b }
-            | term ~'/' primary         {|a, b| a / b }
-            | primary
-  
-  primary <-  ~'(' expr ~')'
-            | @(DIGIT+ ('.' DIGIT+)?)   {|_| _.to_f }
+  # JSON parser based on the grammar at http://www.json.org
+
+  grammar JsonGrammar
+
+  %whitespace (SPACE+ | '/*' (! '*/' .)* '*/' | '//' [^\n]*)*
+
+  ...
 
-==== In a shell:
+==== $ rtlr json.rtlr
 
-  $ rtlr calculator.rtlr
-  calculator.rtlr -> calculator_grammar.rb
+  json.rtlr -> json_grammar.rb
 
-==== calculator.rb:
+==== json_parser.rb:
 
   require 'rubygems'
   require 'rattler'
-  require 'calculator_grammar'
+  require 'json_grammar'
  
-  class Calculator < Rattler::Runtime::WDMParser
-    include CalculatorGrammar
-  end
- 
-  begin
-    puts Calculator.parse!(expr)
-  rescue Rattler::Runtime::SyntaxError => e
-    puts e
+  class JsonParser < Rattler::Runtime::PackratParser
+
+    include JsonGrammar
+
+    def object(members)
+      Hash[*members.flatten(1)]
+    end
+
+    def string(expr)
+      eval expr, TOPLEVEL_BINDING
+    end
+
   end
 
 === Example 3: Dynamically generated parser class
@@ -127,17 +145,18 @@ dynamically from strings.
     
     start   <-  expr EOF
     
-    expr    <-  expr ~'+' term            {|a, b| a + b }
-              | expr ~'-' term            {|a, b| a - b }
+    expr    <-  expr ~'+' term              {|a, b| a + b }
+              | expr ~'-' term              {|a, b| a - b }
               | term
     
-    term    <-  term ~'*' primary         {|a, b| a * b }
-              | term ~'/' primary         {|a, b| a / b }
+    term    <-  term ~'*' primary           {|a, b| a * b }
+              | term ~'/' primary           {|a, b| a / b }
               | primary
     
     primary <-  ~'(' expr ~')'
-              | @(DIGIT+ ('.' DIGIT+)?)   {|_| _.to_f }
-  }, :type => :wdm
+              | @('-'? DIGIT+ '.' DIGIT+)   { _.to_f }
+              | @('-'? DIGIT+)              { _.to_i }
+  }, :type => :extended_packrat
  
   begin
     puts Calculator.parse!(expr)

data/features/grammar/comments.feature

@@ -0,0 +1,24 @@
+Feature: Whitespace
+  
+  Comments start with "#" and continue to the end of the line.
+  
+  In order to clarify my grammars
+  As a language designer
+  I want to use comments
+  
+  Scenario: Entire line as a comment
+    Given a grammar with:
+      """
+      # match one or more alphanumeric or underscore characters
+      word <- @WORD+
+      """
+    When I parse "foo"
+    Then the parse result should be "foo"
+  
+  Scenario: Comment at the end of a line
+    Given a grammar with:
+      """
+      word <- @WORD+  # WORD matches any alphanumeric or underscore character
+      """
+    When I parse "foo"
+    Then the parse result should be "foo"

data/features/grammar/list_matching.feature

@@ -0,0 +1,41 @@
+Feature: List Matching
+
+  An term expression followed by "*^" or "+^" and a separator expression means
+  to match a list of terms with separators between them. "*^" matches a list of
+  zero or more terms, "+^" matches a list of one or more term.
+
+  In order to clearly and easily match list expressions
+  As a language designer
+  I want to use a list-matching expression
+
+  Scenario Outline: Zero or more terms
+    Given a grammar with:
+      """
+      words <-  @WORD+ *^ ','
+      """
+    When I parse <input>
+    Then the parse result should be <result>
+      And the parse position should be <pos>
+  
+  Examples:
+    | input         | result                | pos |
+    | "foo"         | ["foo"]               | 3   |
+    | "foo,bar,baz" | ["foo", "bar", "baz"] | 11  |
+    | "foo,bar,"    | ["foo", "bar"]        | 7   |
+    | "   "         | []                    | 0   |
+
+  Scenario Outline: One or more terms
+    Given a grammar with:
+      """
+      words <-  @WORD+ +^ ','
+      """
+    When I parse <input>
+    Then the parse result should be <result>
+      And the parse position should be <pos>
+  
+  Examples:
+    | input         | result                | pos |
+    | "foo"         | ["foo"]               | 3   |
+    | "foo,bar,baz" | ["foo", "bar", "baz"] | 11  |
+    | "foo,bar,"    | ["foo", "bar"]        | 7   |
+    | "   "         | FAIL                  | 0   |

data/features/grammar/symantic_action.feature

@@ -6,7 +6,7 @@ Feature: Symantic Actions
   specified like Ruby block parameters and are bound to the parse results from
   the match. If there are fewer parameters than parse results the extra results
   are simply ignored. Labeled parse results can be refered to as identifiers in
-  the action.
+  the action. The special identifier "_" refers to the entire parse results.
   
   In order to add simple symantics to parse results
   As a language designer
@@ -15,19 +15,11 @@ Feature: Symantic Actions
   Scenario: Single token
     Given a grammar with:
       """
-      integer <- /\d+/ {|_| _.to_i }
+      integer <- /\d+/ {|s| s.to_i }
       """
     When I parse "42"
     Then the parse result should be 42
   
-  Scenario: Shortcut form
-    Given a grammar with:
-      """
-      integer <- /\d+/ <.to_i>
-      """
-    When I parse "23"
-    Then the parse result should be 23
-  
   Scenario: Sequence
     Given a grammar with:
       """
@@ -36,7 +28,7 @@ Feature: Symantic Actions
       """
     When I parse "3 16"
     Then the parse result should be 48
-    
+  
   Scenario: Sequence with non-capturing expressions
     Given a grammar with:
       """
@@ -51,4 +43,30 @@ Feature: Symantic Actions
       sum <- "(" left:/\d+/ "+" right:/\d+/ ")" { left.to_i + right.to_i }
       """
     When I parse "(17+29)"
-    Then the parse result should be 46
+    Then the parse result should be 46
+  
+  Scenario: Single token using "_"
+    Given a grammar with:
+      """
+      integer <- /\d+/ { _.to_i }
+      """
+    When I parse "23"
+    Then the parse result should be 23
+  
+  Scenario: Sequence using "_"
+    Given a grammar with:
+      """
+      %whitespace SPACE*
+      ints <- /\d+/ /\d+/ { _.reverse }
+      """
+    When I parse "3 16"
+    Then the parse result should be ["16", "3"]
+  
+  Scenario: Sequence using "_" as a parameter name
+    Given a grammar with:
+      """
+      %whitespace SPACE*
+      ints <- /\d+/ /\d+/ {|_| _.to_i }
+      """
+    When I parse "3 16"
+    Then the parse result should be 3

data/lib/rattler/back_end/parser_generator/assert_generator.rb

@@ -1,94 +1,94 @@
 require 'rattler/back_end/parser_generator'
 
 module Rattler::BackEnd::ParserGenerator
-  
+
   # @private
   class AssertGenerator < ExprGenerator #:nodoc:
-    
+
     def gen_basic(assert)
-      generate assert.child, :gen_assert
+      generate assert.child, :assert
     end
-    
+
     def gen_assert(assert)
       gen_basic assert
     end
-    
+
     def gen_disallow(assert)
       @g << 'false'
     end
-    
+
     def gen_skip_nested(assert)
       gen_basic_nested assert
     end
-    
+
     def gen_skip_top_level(assert)
       gen_basic_top_level assert
     end
-    
+
     def gen_dispatch_action_nested(assert, target, method_name)
       atomic_block { gen_dispatch_action_top_level assert, target, method_name }
     end
-    
+
     def gen_dispatch_action_top_level(assert, target, method_name)
       gen_action assert,
         dispatch_action_result(target, method_name, :array_expr => '[]')
     end
-    
+
     def gen_direct_action_nested(assert, code)
       atomic_block { gen_direct_action_top_level assert, code }
     end
-    
+
     def gen_direct_action_top_level(assert, code)
       gen_action assert, direct_action_result(code)
     end
-    
+
     def gen_token_nested(assert)
       atomic_block { gen_token_top_level assert }
     end
-    
+
     def gen_token_top_level(assert)
       gen_action assert, "''"
     end
-    
+
     def gen_intermediate(assert)
-      generate assert.child, :gen_intermediate_assert
+      generate assert.child, :intermediate_assert
     end
-    
+
     def gen_intermediate_assert(assert)
       gen_intermediate assert
     end
-    
+
     def gen_intermediate_skip(assert)
       gen_intermediate assert
     end
-    
+
     private
-    
+
     def gen_action(assert, result_code)
       gen_intermediate assert
       (@g << ' &&').newline << result_code
     end
-    
+
   end
-  
+
   # @private
   class NestedAssertGenerator < AssertGenerator #:nodoc:
     include Nested
-    include NestedGenerators
+    include NestedSubGenerating
   end
-  
+
   def AssertGenerator.nested(*args)
     NestedAssertGenerator.new(*args)
   end
-  
+
   # @private
   class TopLevelAssertGenerator < AssertGenerator #:nodoc:
     include TopLevel
-    include TopLevelGenerators
+    include TopLevelSubGenerating
   end
-  
+
   def AssertGenerator.top_level(*args)
     TopLevelAssertGenerator.new(*args)
   end
-  
+
 end

data/lib/rattler/back_end/parser_generator/choice_generator.rb

@@ -1,88 +1,88 @@
 require 'rattler/back_end/parser_generator'
 
 module Rattler::BackEnd::ParserGenerator
-  
+
   # @private
   class ChoiceGenerator < ExprGenerator #:nodoc:
-    include NestedGenerators
-    
+    include NestedSubGenerating
+
     def gen_basic_nested(choice)
       atomic_block { gen_basic_top_level choice }
     end
-    
+
     def gen_basic_top_level(choice)
       @g.intersperse_nl(choice, ' ||') {|_| generate _ }
     end
-    
+
     def gen_assert_nested(choice)
       atomic_expr { gen_assert_top_level choice }
     end
-    
+
     def gen_assert_top_level(choice)
       gen_intermediate_assert choice
       @g << ' && true'
     end
-    
+
     def gen_disallow(choice)
       @g << '!'
       gen_intermediate_assert choice
     end
-    
+
     def gen_dispatch_action_nested(choice, target, method_name)
       atomic_block { gen_dispatch_action_top_level choice, target, method_name }
     end
-    
+
     def gen_dispatch_action_top_level(choice, target, method_name)
       gen_action_code choice do |labeled|
         dispatch_action_result(target, method_name, :labeled => labeled)
       end
     end
-    
+
     def gen_direct_action_nested(choice, action)
       atomic_block { gen_direct_action_top_level choice, action }
     end
-    
+
     def gen_direct_action_top_level(choice, action)
       gen_action_code choice do |labeled|
         direct_action_result(action, :labeled => labeled)
       end
     end
-    
+
     def gen_token_nested(choice)
       atomic_block { gen_token_top_level choice }
     end
-    
+
     def gen_token_top_level(choice)
-      @g.intersperse_nl(choice, ' ||') {|_| generate _, :gen_token }
+      @g.intersperse_nl(choice, ' ||') {|_| generate _, :token }
     end
-    
+
     def gen_skip_nested(choice)
       @g.surround('(', ')') { gen_skip_top_level choice }
     end
-    
+
     def gen_skip_top_level(choice)
       gen_intermediate_skip choice
       @g << ' && true'
     end
-    
+
     def gen_intermediate_assert(choice)
       atomic_block do
         @g.intersperse_nl(choice, ' ||') do |_|
-          generate _, :gen_intermediate_assert
+          generate _, :intermediate_assert
         end
       end
     end
-    
+
     def gen_intermediate_skip(choice)
       atomic_block do
         @g.intersperse_nl(choice, ' ||') do |_|
-          generate _, :gen_intermediate_skip
+          generate _, :intermediate_skip
         end
       end
     end
-    
+
     private
-    
+
     def gen_action_code(choice)
       labeled = choice.any? {|_| _.labeled? } ? labeled_name : nil
       @g.block("(#{result_name} = begin", 'end)') do
@@ -101,29 +101,29 @@ module Rattler::BackEnd::ParserGenerator
       end << ' && '
       @g << yield(labeled)
     end
-    
+
     def labeled_name
       "l#{choice_level}"
     end
-    
+
   end
-  
+
   # @private
   class NestedChoiceGenerator < ChoiceGenerator #:nodoc:
     include Nested
   end
-  
+
   def ChoiceGenerator.nested(*args)
     NestedChoiceGenerator.new(*args)
   end
-  
+
   # @private
   class TopLevelChoiceGenerator < ChoiceGenerator #:nodoc:
     include TopLevel
   end
-  
+
   def ChoiceGenerator.top_level(*args)
     TopLevelChoiceGenerator.new(*args)
   end
-  
+
 end

data/lib/rattler/back_end/parser_generator/direct_action_generator.rb

@@ -1,59 +1,59 @@
 require 'rattler/back_end/parser_generator'
 
 module Rattler::BackEnd::ParserGenerator
-  
+
   # @private
   class DirectActionGenerator < ExprGenerator #:nodoc:
     include PredicatePropogating
     include TokenPropogating
     include SkipPropogating
-    
+
     def gen_basic_nested(action)
-      generate action.child, :gen_direct_action_nested, action.bindable_code
+      generate action.child, :direct_action_nested, action.bindable_code
     end
-    
+
     def gen_basic_top_level(action)
-      generate action.child, :gen_direct_action_top_level, action.bindable_code
+      generate action.child, :direct_action_top_level, action.bindable_code
     end
-    
+
     def gen_dispatch_action_nested(inner, target, method_name)
       atomic_block { gen_dispatch_action_top_level inner, target, method_name }
     end
-    
+
     def gen_dispatch_action_top_level(inner, target, method_name)
       @g.surround("(#{result_name} = ", ')') { gen_basic_nested inner }
       (@g << ' &&').newline << dispatch_action_result(target, method_name)
     end
-    
+
     def gen_direct_action_nested(inner, code)
       atomic_block { gen_direct_action_top_level inner, code }
     end
-    
+
     def gen_direct_action_top_level(inner, code)
       @g.surround("(#{result_name} = ", ')') { gen_basic_nested inner }
       (@g << ' &&').newline << direct_action_result(code)
     end
-    
+
   end
-  
+
   # @private
   class NestedDirectActionGenerator < DirectActionGenerator #:nodoc:
     include Nested
-    include NestedGenerators
+    include NestedSubGenerating
   end
-  
+
   def DirectActionGenerator.nested(*args)
     NestedDirectActionGenerator.new(*args)
   end
-  
+
   # @private
   class TopLevelDirectActionGenerator < DirectActionGenerator #:nodoc:
     include TopLevel
-    include TopLevelGenerators
+    include TopLevelSubGenerating
   end
-  
+
   def DirectActionGenerator.top_level(*args)
     TopLevelDirectActionGenerator.new(*args)
   end
-  
+
 end