rattler 0.3.0 → 0.4.0

data/README.rdoc

@@ -14,25 +14,25 @@ A language syntax is specified in a grammar using the Rattler syntax. Parser
 classes and modules can be generated statically using the "rtlr" command or
 dynamically from strings.
 
-{RDoc}[http://rubydoc.info/gems/rattler/0.2.2/frames]
+{RDoc}[http://rubydoc.info/gems/rattler/0.4.0/frames]
 
 == FEATURES:
 
 * Uses readable PEG-like grammars
-* A packrat parser to cache results for backtracking
-* An extended packrat parser that supports left-recursive grammars
+* Supports directly and indirectly left-recursive grammars
 * Whitespace can be specified in one place and skipped automatically
 * Automatic and custom parse error reporting
-* Optimizing parser generator
-* Generated parsers are relatively readable
+* Optimizing parser generator generates efficient pure-ruby parsers
 * Compatible with Ruby 1.8.7 and 1.9.2, tested with MRI and JRuby on Linux and
   Windows
 
 == PROBLEMS/LIMITATIONS:
 
-* Optimizations are very limited at this point
 * Only strings can be parsed, so files have to be read completely before parsing
 * There are many holes in the tests so there are undoubtedly many bugs
+* Optimzations are not complete
+* Support for composable grammars is not yet implemented
+* Symantics of automatic whitespace skipping could be refined
 
 == EXAMPLES:
 
@@ -42,32 +42,52 @@ dynamically from strings.
   
   # JSON parser based on the grammar at http://www.json.org
 
+  require File.expand_path('json_helper', File.dirname(__FILE__))
+
   parser JsonParser < Rattler::Runtime::PackratParser
 
-  %whitespace (SPACE+ | '/*' (! '*/' .)* '*/' | '//' [^\n]*)*
+  include JsonHelper
+
+  %whitespace (SPACE+ / comment)* {
+
+    object    <-  ~'{' members ~'}'                                   { object _ }
+
+    members   <-  pair *, ','
+
+    pair      <-  string ~':' value
+
+    array     <-  ~'[' elements ~']'
+
+    elements  <-  value *, ','
+
+    value     <-  string
+                / number
+                / object
+                / array
+                / `true`                                              { :true }
+                / `false`                                             { :false }
+                / `null`                                              { :null }
+                / fail "value expected"
 
-  object    <-  ~'{' members ~'}'                                   { object _ }
+    string    <-  @('"' char* '"')                                    { string _ }
 
-  members   <-  pair *^ ','
+    number    <-  @(int frac? exp?)                                   { number _ }
+  }
 
-  pair      <-  string ~':' value
+  %inline {
 
-  array     <-  ~'[' elements ~']'
+    char      <-  !('"' / '\\' / CNTRL) .
+                / '\\' (["\\/bfnrt] / 'u' XDIGIT XDIGIT XDIGIT XDIGIT)
 
-  elements  <-  value *^ ','
+    int       <-  '-'? ('0' !DIGIT / [1-9] DIGIT*)
 
-  value     <-  string                                              { string _ }
-              | number !DIGIT                                       { _.to_f }
-              | object
-              | array
-              | `true`                                              { :true }
-              | `false`                                             { :false }
-              | `null`                                              { :null }
-              | fail "value expected"
+    frac      <-  '.' DIGIT+
 
-  string    <-  @('"' ('\\' . | [^"])* '"')
+    exp       <-  [eE] [+-]? DIGIT+
 
-  number    <-  @('-'? ('0' | [1-9] DIGIT*) ('.' DIGIT+)? ([eE] [+-]? DIGIT+)?)
+    comment   <-  '/*' (! '*/' .)* '*/'
+                / '//' [^\n]*
+  }
 
 ==== json_helper.rb:
 
@@ -78,6 +98,10 @@ dynamically from strings.
     end
 
     def string(expr)
+      eval "%q#{expr}", TOPLEVEL_BINDING
+    end
+
+    def number(expr)
       eval expr, TOPLEVEL_BINDING
     end
 
@@ -107,7 +131,7 @@ dynamically from strings.
 
   grammar JsonGrammar
 
-  %whitespace (SPACE+ | '/*' (! '*/' .)* '*/' | '//' [^\n]*)*
+  %whitespace (SPACE+ / comment)* {
 
   ...
 
@@ -129,11 +153,7 @@ dynamically from strings.
       Hash[*members.flatten(1)]
     end
 
-    def string(expr)
-      eval expr, TOPLEVEL_BINDING
-    end
-
-  end
+    ...
 
 === Example 3: Dynamically generated parser class
 
@@ -145,18 +165,18 @@ dynamically from strings.
     
     start   <-  expr EOF
     
-    expr    <-  expr ~'+' term              {|a, b| a + b }
-              | expr ~'-' term              {|a, b| a - b }
-              | term
+    expr    <-  expr ~'+' term                  {|a, b| a + b }
+              / expr ~'-' term                  {|a, b| a - b }
+              / term
     
-    term    <-  term ~'*' primary           {|a, b| a * b }
-              | term ~'/' primary           {|a, b| a / b }
-              | primary
+    term    <-  term ~'*' primary               {|a, b| a * b }
+              / term ~'/' primary               {|a, b| a / b }
+              / primary
     
     primary <-  ~'(' expr ~')'
-              | @('-'? DIGIT+ '.' DIGIT+)   { _.to_f }
-              | @('-'? DIGIT+)              { _.to_i }
-  }, :type => :extended_packrat
+              / @('-'? DIGIT+ ('.' DIGIT+)?)    { _.to_f }
+  },
+  :type => :extended_packrat
  
   begin
     puts Calculator.parse!(expr)

data/features/command_line/dest_option.feature

@@ -1,32 +1,19 @@
 Feature: --dest option
 
-  Use the --dest (or -d) option to specify a different (base) destination
-  directory.
+  Use the --dest (or -d) option to specify an explicit destination directory.
+  By default Rattler uses the module path of the grammar to build a destination
+  directory by converting camel-case to underscore.
 
-  Scenario: Grammar as top-level module
-    Given a file named "binary.rtlr" with:
-      """
-      grammar BinaryGrammar
-      expr <- [01]*
-      """
-      And a directory named "lib"
-    When I run "rtlr --dest lib binary.rtlr"
-    Then the output should contain "binary.rtlr -> lib/binary_grammar.rb"
-      And the file "lib/binary_grammar.rb" should contain:
-        """
-        module BinaryGrammar
-        """
-
-  Scenario: Grammar as top-level module
+  Scenario: Overriding the destination directory
     Given a file named "binary.rtlr" with:
       """
       grammar Examples::BinaryGrammar
       expr <- [01]*
       """
-      And a directory named "lib/examples"
-    When I run "rtlr --dest lib binary.rtlr"
-    Then the output should contain "binary.rtlr -> lib/examples/binary_grammar.rb"
-      And the file "lib/examples/binary_grammar.rb" should contain:
+      And a directory named "lib/my_examples"
+    When I run "rtlr --dest lib/my_examples binary.rtlr"
+    Then the output should contain "binary.rtlr -> lib/my_examples/binary_grammar.rb"
+      And the file "lib/my_examples/binary_grammar.rb" should contain:
         """
         module Examples
           # @private

data/features/command_line/lib_option.feature

@@ -0,0 +1,37 @@
+Feature: --lib option
+
+  Use the --lib (or -l) option to specify a different destination library
+  directory. If not specified, the default is the current directory. Rattler
+  uses the module path of the grammar to build a destination directory based
+  on the library directory by converting camel-case to underscore. To specify
+  an explicit destination directory use the --dest (or -d) option.
+
+  Scenario: Grammar as top-level module
+    Given a file named "binary.rtlr" with:
+      """
+      grammar BinaryGrammar
+      expr <- [01]*
+      """
+      And a directory named "lib"
+    When I run "rtlr --lib lib binary.rtlr"
+    Then the output should contain "binary.rtlr -> lib/binary_grammar.rb"
+      And the file "lib/binary_grammar.rb" should contain:
+        """
+        module BinaryGrammar
+        """
+
+  Scenario: Grammar as a nested module
+    Given a file named "binary.rtlr" with:
+      """
+      grammar Examples::BinaryGrammar
+      expr <- [01]*
+      """
+      And a directory named "lib/examples"
+    When I run "rtlr --lib lib binary.rtlr"
+    Then the output should contain "binary.rtlr -> lib/examples/binary_grammar.rb"
+      And the file "lib/examples/binary_grammar.rb" should contain:
+        """
+        module Examples
+          # @private
+          module BinaryGrammar
+        """

data/features/command_line/parser_generator.feature

@@ -9,9 +9,12 @@ Feature: Parser Generator
       """
       Usage: rtlr FILENAME [options]
 
-          -d, --dest DIRECTORY             Specify the destination directory
-          -o, --output FILENAME            Specify a different output filename
+          -l, --lib DIRECTORY              Specify the destination lib directory
+          -d, --dest DIRECTORY             Specify an explicit destination directory
+          -o, --output FILENAME            Specify a different output filename ("-" = STDOUT)
           -f, --force                      Force overwrite if the output file exists
+          -s, --standalone                 Optimize for use as a standalone parser
+          -n, --no-optimize                Disable optimization
 
           -h, --help                       Show this message
       """
@@ -22,7 +25,7 @@ Feature: Parser Generator
       grammar BinaryGrammar
       expr <- [01]*
       """
-    When I run "rtlr binary.rtlr"
+    When I run "rtlr binary.rtlr --standalone"
     Then the output should contain "binary.rtlr -> binary_grammar.rb"
       And the file "binary_grammar.rb" should contain:
         """
@@ -53,7 +56,7 @@ Feature: Parser Generator
       parser BinaryParser < Rattler::Runtime::PackratParser
       expr <- [01]*
       """
-    When I run "rtlr binary.rtlr"
+    When I run "rtlr binary.rtlr --standalone"
     Then the output should contain "binary.rtlr -> binary_parser.rb"
       And the file "binary_parser.rb" should contain:
         """

data/features/grammar/back_reference.feature

@@ -0,0 +1,37 @@
+Feature: Back References
+
+  A back reference is a "$" followed by a label defined earlier in a sequence.
+  The reference refers to the previous parse result and it means to match the
+  same exact input again.
+
+  In order to reuse a previous parse result to match the same text again
+  As a language designer
+  I want to use back references in my grammar
+
+  Scenario Outline: Parsing
+    Given a grammar with:
+      """
+      expr <- a:ALPHA DIGIT+ $a
+      """
+    When I parse <input>
+    Then the parse result should be <result>
+  
+  Examples:
+    | input  | result                 |
+    | "a12a" | ["a", ["1", "2"], "a"] |
+    | "b12b" | ["b", ["1", "2"], "b"] |
+    | "a12b" | FAIL                   |
+
+  Scenario Outline: Token
+    Given a grammar with:
+      """
+      string <- @('%' q:PUNCT (! $q .)* $q)
+      """
+    When I parse <input>
+    Then the parse result should be <result>
+  
+  Examples:
+    | input      | result   |
+    | "%/a b/ c" | "%/a b/" |
+    | "%!a b! c" | "%!a b!" |
+    | "%/a b! c" | FAIL     |

data/features/grammar/fail.feature

@@ -25,7 +25,7 @@ Feature: Fail Expressions
     Given a grammar with:
       """
       expr  <-  fail_rule "something really bad happened"
-              | .*
+              / .*
       """
     When I parse "anything"
     Then the parse should fail
@@ -34,7 +34,7 @@ Feature: Fail Expressions
   Scenario: Fail-parse
     Given a grammar with:
       """
-      a <- b | .*
+      a <- b / .*
       b <- fail_parse "something catastrophic happened"
       """
     When I parse "anything"
@@ -44,7 +44,7 @@ Feature: Fail Expressions
   Scenario: Fail-expression at the end of an ordered choice
     Given a grammar with:
       """
-      expr  <- a | b | fail "something bad happened"
+      expr  <- a / b / fail "something bad happened"
       a     <- "a"
       b     <- "b"
       """

data/features/grammar/labels.feature

@@ -8,7 +8,7 @@ Feature: Labels
   Scenario: Normal symantic action
     Given a grammar with:
       """
-      fraction <- numer:/\d+/ "/" denom:/\d+/ { numer.to_i / denom.to_i }
+      fraction <- numer:@DIGIT+ "/" denom:@DIGIT+ { numer.to_i / denom.to_i }
       """
     When I parse "6/2"
     Then the parse result should be 3
@@ -16,11 +16,19 @@ Feature: Labels
   Scenario: Node action
     Given a grammar with:
       """
-      fraction <- numer:/\d+/ "/" denom:/\d+/ <Fraction>
+      fraction <- numer:@DIGIT+ "/" denom:@DIGIT+ <Fraction>
       """
       And a class definition:
         """
         class Fraction < Rattler::Runtime::ParseNode; end
         """
     When I parse "6/2"
-    Then the parse result should be Fraction[["6", "/", "2"], {:labeled => {:numer => "6", :denom => "2"}}]
+    Then the parse result should be Fraction[["6", "/", "2"], {:labeled => {:numer => "6", :denom => "2"}}]
+
+  Scenario: Nested scope
+    Given a grammar with:
+      """
+      a <- word:@ALPHA+ (@DIGIT+ {|num| "#{num} #{word}" })
+      """
+    When I parse "abc123"
+    Then the parse result should be ["abc", "123 abc"]

data/features/grammar/list_matching.feature

@@ -1,8 +1,8 @@
 Feature: List Matching
 
-  An term expression followed by "*^" or "+^" and a separator expression means
-  to match a list of terms with separators between them. "*^" matches a list of
-  zero or more terms, "+^" matches a list of one or more term.
+  A term expression followed by "*," or "+," and a separator expression means
+  to match a list of terms with separators between them. "*," matches a list of
+  zero or more terms, "+," matches a list of one or more term.
 
   In order to clearly and easily match list expressions
   As a language designer
@@ -11,7 +11,7 @@ Feature: List Matching
   Scenario Outline: Zero or more terms
     Given a grammar with:
       """
-      words <-  @WORD+ *^ ','
+      words <-  @WORD+ *, ','
       """
     When I parse <input>
     Then the parse result should be <result>
@@ -27,7 +27,7 @@ Feature: List Matching
   Scenario Outline: One or more terms
     Given a grammar with:
       """
-      words <-  @WORD+ +^ ','
+      words <-  @WORD+ +, ','
       """
     When I parse <input>
     Then the parse result should be <result>
@@ -39,3 +39,12 @@ Feature: List Matching
     | "foo,bar,baz" | ["foo", "bar", "baz"] | 11  |
     | "foo,bar,"    | ["foo", "bar"]        | 7   |
     | "   "         | FAIL                  | 0   |
+
+  Scenario: Using whitespace
+    Given a grammar with:
+      """
+      %whitespace SPACE*
+      words <-  @WORD+ *, ','
+      """
+    When I parse "foo, bar ,baz"
+    Then the parse result should be ["foo", "bar", "baz"]

data/features/grammar/literal.feature

@@ -1,13 +1,13 @@
 Feature: Literal Expressions
   
-  A string literal is like a normal Ruby string literal and it means to match
-  if that exact text is next in the input.
+  A string literal has the same syntax as a Ruby string literal and it means to
+  match if that exact text is next in the input.
   
   In order to define symbols and keywords
   As a language designer
   I want to use string literals in my grammar
   
-  Scenario Outline: Parsing
+  Scenario Outline: Normal String
     Given a grammar with:
       """
       a <- "while"
@@ -19,4 +19,30 @@ Feature: Literal Expressions
     | input    | result  |
     | "while"  | "while" |
     | " while" | FAIL    |
-    | "whip"   | FAIL    |
+    | "whip"   | FAIL    |
+  
+  Scenario Outline: General delimited String with brackets
+    Given a grammar with:
+      """
+      a <- %{while}
+      """
+    When I parse <input>
+    Then the parse result should be <result>
+  
+  Examples:
+    | input    | result  |
+    | "while"  | "while" |
+    | "whip"   | FAIL    |
+  
+  Scenario Outline: General delimited String with arbitrary delimiters
+    Given a grammar with:
+      """
+      a <- %!while!
+      """
+    When I parse <input>
+    Then the parse result should be <result>
+  
+  Examples:
+    | input    | result  |
+    | "while"  | "while" |
+    | "whip"   | FAIL    |