walrus 0.1

data/lib/walrus/grammar/parslet_combining.rb

@@ -0,0 +1,154 @@
+# Copyright 2007 Wincent Colaiuta
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# in the accompanying file, "LICENSE.txt", for more details.
+#
+# $Id$
+
+require 'walrus'
+
+module Walrus
+  class Grammar
+    
+    # The ParsletCombining module, together with the ParsletCombination class and its subclasses, provides simple container classes for encapsulating relationships among Parslets. By storing this information outside of the Parslet objects themselves their design is kept clean and they can become immutable objects which are much more easily copied and shared among multiple rules in a Grammar.
+    module ParsletCombining
+      
+      # Convenience method.
+      def memoizing_parse(string, options = {})
+        self.to_parseable.memoizing_parse(string, options)
+      end
+      
+      # Convenience method.
+      def parse(string, options = {})
+        self.to_parseable.parse(string, options)
+      end
+      
+      # Defines a sequence of Parslets (or ParsletCombinations).
+      # Returns a ParsletSequence instance.
+      def sequence(first, second, *others)
+        Walrus::Grammar::ParsletSequence.new(first.to_parseable, second.to_parseable, *others)
+      end
+      
+      # Shorthand for ParsletCombining.sequence(first, second).
+      def &(next_parslet)
+        self.sequence(self, next_parslet)
+      end
+      
+      # Defines a sequence of Parslets similar to the sequence method but with the difference that the contents of array results from the component parslets will be merged into a single array rather than being added as arrays. To illustrate:
+      #
+      #   'foo' & 'bar'.one_or_more   # returns results like ['foo', ['bar', 'bar', 'bar']]
+      #   'foo' >> 'bar'.one_or_more  # returns results like ['foo', 'bar', 'bar', 'bar']
+      #
+      def merge(first, second, *others)
+        Walrus::Grammar::ParsletMerge.new(first.to_parseable, second.to_parseable, *others)
+      end
+      
+      # Shorthand for ParsletCombining.sequence(first, second)
+      def >>(next_parslet)
+        self.merge(self, next_parslet)
+      end
+      
+      # Defines a choice of Parslets (or ParsletCombinations).
+      # Returns a ParsletChoice instance.
+      def choice(left, right, *others)
+        Walrus::Grammar::ParsletChoice.new(left.to_parseable, right.to_parseable, *others)
+      end
+      
+      # Shorthand for ParsletCombining.choice(left, right)
+      def |(alternative_parslet)
+        self.choice(self, alternative_parslet)
+      end
+      
+      # Defines a repetition of the supplied Parslet (or ParsletCombination).
+      # Returns a ParsletRepetition instance.
+      def repetition(parslet, min, max)
+        Walrus::Grammar::ParsletRepetition.new(parslet.to_parseable, min, max)
+      end
+      
+      # Shorthand for ParsletCombining.repetition.
+      def repeat(min = nil, max = nil)
+        self.repetition(self, min, max)
+      end
+      
+      def repetition_with_default(parslet, min, max, default)
+        Walrus::Grammar::ParsletRepetitionDefault.new(parslet.to_parseable, min, max, default)
+      end
+      
+      def repeat_with_default(min = nil, max = nil, default = nil)
+        self.repetition_with_default(self, min, max, default)
+      end
+      
+      # Shorthand for ParsletCombining.repetition(0, 1).
+      # This method optionally takes a single parameter specifying what object should be returned as a placeholder when there are no matches; this is useful for packing into ASTs where it may be better to parse an empty Array rather than nil. The specified object is cloned and returned in the event that there are no matches. As a convenience, the specified object is automatically extended using the LocationTracking module (this is a convenience so that you can specify empty Arrays, "[]", rather than explicitly passing an "ArrayResult.new")
+      def optional(default_return_value = NoParameterMarker.instance)
+        if default_return_value == NoParameterMarker.instance
+          self.repeat(0, 1) # default behaviour
+        else
+          self.repeat_with_default(0, 1, default_return_value)
+        end
+      end
+      
+      # Alternative to optional.
+      def zero_or_one
+        self.optional
+      end
+      
+      # possible synonym "star"
+      def zero_or_more(default_return_value = NoParameterMarker.instance)
+        if default_return_value == NoParameterMarker.instance
+          self.repeat(0) # default behaviour
+        else
+          self.repeat_with_default(0, nil, default_return_value)
+        end
+      end
+      
+      # possible synonym "plus"
+      def one_or_more
+        self.repeat(1)
+      end
+      
+      # Parsing Expression Grammar support.
+      # Succeeds if parslet succeeds but consumes no input (throws an :AndPredicateSuccess symbol).
+      def and_predicate(parslet)
+        Walrus::Grammar::AndPredicate.new(parslet.to_parseable)
+      end
+      
+      # Shorthand for and_predicate
+      # Strictly speaking, this shorthand breaks with established Ruby practice that "?" at the end of a method name should indicate a method that returns true or false.
+      def and?
+        self.and_predicate(self)
+      end
+      
+      # Parsing Expression Grammar support.
+      # Succeeds if parslet fails (throws a :NotPredicateSuccess symbol).
+      # Fails if parslet succeeds (raise a ParseError).
+      # Consumes no output.
+      # This method will almost invariably be used in conjuntion with the & operator, like this:
+      #       rule :foo, :p1 & :p2.not_predicate
+      #       rule :foo, :p1 & :p2.not!
+      def not_predicate(parslet)
+        Walrus::Grammar::NotPredicate.new(parslet.to_parseable)
+      end
+      
+      # Shorthand for not_predicate.
+      # Strictly speaking, this shorthand breaks with established Ruby practice that "!" at the end of a method name should indicate a destructive behaviour on (mutation of) the receiver.
+      def not!
+        self.not_predicate(self)
+      end
+      
+      # Succeeds if parsing succeeds, consuming the output, but doesn't actually return anything.
+      # This is for elements which are required but which shouldn't appear in the final AST.
+      def omission(parslet)
+        Walrus::Grammar::ParsletOmission.new(parslet.to_parseable)
+      end
+      
+      # Shorthand for ParsletCombining.omission
+      def skip
+        self.omission(self)
+      end
+      
+    end # module ParsletCombining
+    
+  end # class Grammar
+end # module Walrus

data/lib/walrus/grammar/parslet_merge.rb

@@ -0,0 +1,88 @@
+# Copyright 2007 Wincent Colaiuta
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# in the accompanying file, "LICENSE.txt", for more details.
+#
+# $Id$
+
+require 'walrus'
+
+module Walrus
+  class Grammar
+    
+    class ParsletMerge < ParsletSequence
+      
+      def parse(string, options = {})
+        raise ArgumentError if string.nil?
+        state                         = ParserState.new(string, options)
+        last_caught                   = nil # keep track of the last kind of throw to be caught
+        @components.each do |parseable|
+          catch :ProcessNextComponent do
+            catch :NotPredicateSuccess do
+              catch :AndPredicateSuccess do
+                catch :ZeroWidthParseSuccess do
+                  begin
+                    parsed = parseable.memoizing_parse(state.remainder, state.options)
+                    if parsed.respond_to? :each
+                      parsed.each { |element| state.parsed(element) }
+                    else
+                      state.parsed(parsed)
+                    end
+                  rescue SkippedSubstringException => e
+                    state.skipped(e)
+                  # rescue ParseError => e # failed, will try to skip; save original error in case skipping fails                                        
+                  #   if options.has_key?(:skipping_override) : skipping_parslet = options[:skipping_override]
+                  #   elsif options.has_key?(:skipping)       : skipping_parslet = options[:skipping]
+                  #   else                                      skipping_parslet = nil
+                  #   end
+                  #   raise e if skipping_parslet.nil?        # no skipper defined, raise original error
+                  #   begin
+                  #     parsed = skipping_parslet.memoizing_parse(state.remainder, state.options) # guard against self references (possible infinite recursion) here?
+                  #     state.skipped(parsed)
+                  #     redo              # skipping succeeded, try to redo
+                  #   rescue ParseError
+                  #     raise e           # skipping didn't help either, raise original error
+                  #   end
+                  end
+                  last_caught = nil
+                  throw :ProcessNextComponent # can't use "next" here because it will only break out of innermost "do"
+                end
+                last_caught = :ZeroWidthParseSuccess
+                throw :ProcessNextComponent
+              end
+              last_caught = :AndPredicateSuccess
+              throw :ProcessNextComponent
+            end
+            last_caught = :NotPredicateSuccess
+          end
+        end
+        
+        if state.results.respond_to? :empty? and state.results.empty? and
+         throw last_caught
+        else
+          state.results
+        end
+        
+      end
+      
+      def eql?(other)
+        return false if not other.instance_of? ParsletMerge
+        other_components = other.components
+        return false if @components.length != other_components.length
+        for i in 0..(@components.length - 1)
+          return false unless @components[i].eql? other_components[i]
+        end
+        true
+      end
+      
+    private
+      
+      def hash_offset
+        53
+      end
+      
+    end # class ParsletMerge
+  end # class Grammar
+end # module Walrus
+

data/lib/walrus/grammar/parslet_omission.rb

@@ -0,0 +1,57 @@
+# Copyright 2007 Wincent Colaiuta
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# in the accompanying file, "LICENSE.txt", for more details.
+#
+# $Id$
+
+require 'walrus'
+
+module Walrus
+  class Grammar
+    
+    class ParsletOmission < ParsletCombination
+      
+      attr_reader :hash
+      
+      # Raises an ArgumentError if parseable is nil.
+      def initialize(parseable)
+        raise ArgumentError if parseable.nil?
+        @parseable = parseable
+        @hash = @parseable.hash + 46 # fixed offset to avoid unwanted collisions with similar classes
+      end
+      
+      def parse(string, options = {})
+        raise ArgumentError if string.nil?
+        substring = StringResult.new
+        substring.start = [options[:line_start], options[:column_start]]
+        substring.end   = [options[:line_start], options[:column_start]]
+        
+        # possibly should catch these here as well
+        #catch :NotPredicateSuccess do
+        #catch :AndPredicateSuccess do
+        # one of the fundamental problems is that if a parslet throws such a symbol any info about already skipped material is lost (because the symbols contain nothing)
+        # this may be one reason to change these to exceptions...
+        catch :ZeroWidthParseSuccess do
+          substring = @parseable.memoizing_parse(string, options)
+        end
+        
+        # not enough to just return a ZeroWidthParseSuccess here; that could cause higher levels to stop parsing and in any case there'd be no clean way to embed the scanned substring in the symbol
+        raise SkippedSubstringException.new(substring, 
+                                            :line_start => options[:line_start],  :column_start => options[:column_start],
+                                            :line_end   => substring.line_end,    :column_end   => substring.column_end)
+      end
+      
+      def eql?(other)
+        other.instance_of? ParsletOmission and other.parseable.eql? @parseable
+      end
+      
+    protected
+      
+      # For determining equality.
+      attr_reader :parseable
+      
+    end # class ParsletOmission
+  end # class Grammar
+end # module Walrus

data/lib/walrus/grammar/parslet_repetition.rb

@@ -0,0 +1,97 @@
+# Copyright 2007 Wincent Colaiuta
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# in the accompanying file, "LICENSE.txt", for more details.
+#
+# $Id$
+
+require 'walrus'
+
+module Walrus
+  class Grammar
+    
+    class ParsletRepetition < ParsletCombination
+      
+      attr_reader :hash
+      
+      # Raises an ArgumentError if parseable or min is nil.
+      def initialize(parseable, min, max = nil)
+        raise ArgumentError if parseable.nil?
+        raise ArgumentError if min.nil?
+        @parseable = parseable
+        self.min = min
+        self.max = max
+      end
+      
+      def parse(string, options = {})
+        raise ArgumentError if string.nil?
+        state                         = ParserState.new(string, options)
+        catch :ZeroWidthParseSuccess do             # a zero-width match is grounds for immediate abort
+          while @max.nil? or state.length < @max    # try forever if max is nil; otherwise keep trying while match count < max
+            begin
+              parsed = @parseable.memoizing_parse(state.remainder, state.options)
+              state.parsed(parsed)
+            rescue SkippedSubstringException => e
+              state.skipped(e)
+            rescue ParseError => e # failed, will try to skip; save original error in case skipping fails
+              if options.has_key?(:skipping_override) : skipping_parslet = options[:skipping_override]
+              elsif options.has_key?(:skipping)       : skipping_parslet = options[:skipping]
+              else                                      skipping_parslet = nil
+              end
+              break if skipping_parslet.nil?
+              begin
+                parsed = skipping_parslet.memoizing_parse(state.remainder, state.options) # guard against self references (possible infinite recursion) here?
+                state.skipped(parsed)
+                redo                                  # skipping succeeded, try to redo
+              rescue ParseError
+                break                                 # skipping didn't help either, give up
+              end
+            end
+          end          
+        end
+        
+        # now assess whether our tries met the requirements
+        if state.length == 0 and @min == 0 # success (special case)
+          throw :ZeroWidthParseSuccess
+        elsif state.length < @min          # matches < min (failure)
+          raise ParseError.new('required %d matches but obtained %d while parsing "%s"' % [@min, state.length, string],
+                               :line_end    => state.options[:line_end],    :column_end   => state.options[:column_end])
+        else                              # success (general case)
+          state.results                   # returns multiple matches as an array, single matches as a single object
+        end
+        
+      end
+      
+      def eql?(other)
+        other.instance_of? ParsletRepetition and @min == other.min and @max == other.max and @parseable.eql? other.parseable
+      end
+      
+    protected
+      
+      # For determining equality.
+      attr_reader :parseable, :min, :max
+      
+    private
+      
+      def hash_offset
+        87
+      end
+      
+      def update_hash
+        @hash = @min.hash + @max.hash + @parseable.hash + hash_offset # fixed offset to minimize risk of collisions
+      end
+      
+      def min=(min)
+        @min = (min.clone rescue min)
+        update_hash
+      end
+      
+      def max=(max)
+        @max = (max.clone rescue max)
+        update_hash
+      end
+      
+    end # class ParsletRepetition
+  end # class Grammar
+end # module Walrus

data/lib/walrus/grammar/parslet_repetition_default.rb

@@ -0,0 +1,58 @@
+# Copyright 2007 Wincent Colaiuta
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# in the accompanying file, "LICENSE.txt", for more details.
+#
+# $Id$
+
+require 'walrus'
+
+module Walrus
+  class Grammar
+    
+    # ParsletRepetitionDefault is a subclass that modifies the behaviour of its parent, ParsletRepetition, in a very small way. Namely, if the outcome of parsing is a ZeroWidthParse success then it is caught and the default value (defined at initialization time) is returned instead.
+    class ParsletRepetitionDefault < ParsletRepetition
+      
+      # Possible re-factoring to consider for the future: roll the functionality of this class in to ParsletRepetition itself.
+      # Benefit of keeping it separate is that the ParsletRepetition itself is kept simple.
+      def initialize(parseable, min, max = nil, default = nil)
+        super(parseable, min, max)
+        self.default  = default
+      end
+      
+      def parse(string, options = {})
+        catch :ZeroWidthParseSuccess do 
+          return super(string, options) 
+        end
+        @default.clone rescue @default
+      end
+      
+      def eql?(other)
+        other.instance_of? ParsletRepetitionDefault and @min == other.min and @max == other.max and @parseable.eql? other.parseable and @default == other.default
+      end
+      
+    protected
+      
+      # For determining equality.
+      attr_reader :default
+      
+    private
+      
+      def hash_offset
+        69
+      end
+      
+      def update_hash
+        @hash = super + @default.hash # let super calculate its share of the hash first
+      end
+      
+      def default=(default)
+        @default = (default.clone rescue default)
+        @default.extend(LocationTracking)
+        update_hash
+      end
+      
+    end # class ParsletRepetitionDefault
+  end # class Grammar
+end # module Walrus

data/lib/walrus/grammar/parslet_sequence.rb

@@ -0,0 +1,202 @@
+# Copyright 2007 Wincent Colaiuta
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# in the accompanying file, "LICENSE.txt", for more details.
+#
+# $Id$
+
+require 'walrus'
+
+module Walrus
+  class Grammar
+    
+    class ParsletSequence < ParsletCombination
+      
+      attr_reader :hash
+      
+      # first and second may not be nil.
+      def initialize(first, second, *others)
+        raise ArgumentError if first.nil?
+        raise ArgumentError if second.nil?
+        @components = [first, second] + others
+        update_hash
+      end
+      
+      # Override so that sequences are appended to an existing sequence:
+      # Consider the following example:
+      #     A & B
+      # This constitutes a single sequence:
+      #     (A & B)
+      # If we then make this a three-element sequence:
+      #     A & B & C
+      # We are effectively creating an nested sequence containing the original sequence and an additional element:
+      #     ((A & B) & C)
+      # Although such a nested sequence is correctly parsed it produces unwanted nesting in the results because instead of returning a one-dimensional an array of results:
+      #     [a, b, c]
+      # It returns a nested array:
+      #     [[a, b], c]
+      # The solution to this unwanted nesting is to allowing appending to an existing sequence by using the private "append" method.
+      # This ensures that:
+      #     A & B & C
+      # Translates to a single sequence:
+      #     (A & B & C)
+      # And a single, uni-dimensional results array:
+      #     [a, b, c]
+      #
+      def &(next_parslet)
+        append(next_parslet)
+      end
+      
+      SKIP_FIRST  = true
+      NO_SKIP     = false
+      
+      def parse(string, options = {})
+        parse_common(NO_SKIP, string, options)
+      end
+      
+      def parse_remainder(string, options = {})
+        parse_common(SKIP_FIRST, string, options)
+      end
+      
+      def parse_common(skip_first, string, options = {})
+        raise ArgumentError if string.nil?
+        state                         = ParserState.new(string, options)
+        last_caught                   = nil   # keep track of the last kind of throw to be caught
+        left_recursion                = false # keep track of whether left recursion was detected
+        
+        @components.each_with_index do |parseable, index|
+          
+          if index == 0 # for first component only
+            if skip_first
+              next
+            end
+            begin
+              check_left_recursion(parseable, options)
+            rescue LeftRecursionException => e
+              left_recursion  = true
+              continuation    = nil
+              value           = callcc { |c| continuation = c }         
+              if value == continuation                    # first time that we're here
+                e.continuation = continuation             # pack continuation into exception
+                raise e                                   # and propagate
+              else
+                grammar   = state.options[:grammar]
+                rule_name = state.options[:rule_name]
+                state.parsed(grammar.wrap(value, rule_name))
+                next
+              end
+            end
+          end
+          
+          catch :ProcessNextComponent do
+            catch :NotPredicateSuccess do
+              catch :AndPredicateSuccess do
+                catch :ZeroWidthParseSuccess do
+                  begin
+                    parsed = parseable.memoizing_parse(state.remainder, state.options)
+                    state.parsed(parsed)
+                  rescue SkippedSubstringException => e
+                    state.skipped(e)
+                  rescue ParseError => e # failed, will try to skip; save original error in case skipping fails                                        
+                    if options.has_key?(:skipping_override) : skipping_parslet = options[:skipping_override]
+                    elsif options.has_key?(:skipping)       : skipping_parslet = options[:skipping]
+                    else                                      skipping_parslet = nil
+                    end
+                    raise e if skipping_parslet.nil?        # no skipper defined, raise original error
+                    begin
+                      parsed = skipping_parslet.memoizing_parse(state.remainder, state.options) # guard against self references (possible infinite recursion) here?
+                      state.skipped(parsed)
+                      redo              # skipping succeeded, try to redo
+                    rescue ParseError
+                      raise e           # skipping didn't help either, raise original error
+                    end
+                  end
+                  last_caught = nil
+                  throw :ProcessNextComponent   # can't use "next" here because it would only break out of innermost "do" rather than continuing the iteration
+                end
+                last_caught = :ZeroWidthParseSuccess
+                throw :ProcessNextComponent
+              end
+              last_caught = :AndPredicateSuccess
+              throw :ProcessNextComponent
+            end
+            last_caught = :NotPredicateSuccess
+          end
+        end
+        
+        if left_recursion
+          results = recurse(state)
+        else
+          results = state.results
+        end
+        
+        if skip_first
+          return results
+        end
+        
+        if results.respond_to? :empty? and results.empty? and last_caught
+          throw last_caught
+        else
+          results
+        end
+        
+      end
+      
+      # Left-recursion helper
+      def recurse(state)
+        return state.results if state.remainder == '' # further recursion is not possible
+        new_state = ParserState.new(state.remainder, state.options)
+        last_successful_result = nil
+        while state.remainder != ''
+          begin
+            new_results = parse_remainder(new_state.remainder, new_state.options)
+            new_state.parsed(new_results)
+            last_successful_result = ArrayResult[last_successful_result || state.results, new_results]
+          rescue ParseError
+            break
+          end
+        end
+        last_successful_result || state.results
+      end
+      
+      def eql?(other)
+        return false if not other.instance_of? ParsletSequence
+        other_components = other.components
+        return false if @components.length != other_components.length
+        for i in 0..(@components.length - 1)
+          return false unless @components[i].eql? other_components[i]
+        end
+        true
+      end
+      
+    protected
+      
+      # For determining equality.
+      attr_reader :components
+      
+    private
+      
+      def hash_offset
+        40
+      end
+      
+      def update_hash
+        @hash = hash_offset # fixed offset to avoid unwanted collisions with similar classes
+        @components.each { |parseable| @hash += parseable.hash }
+      end
+      
+      # Appends another Parslet, ParsletCombination or Predicate to the receiver and returns the receiver.
+      # Raises if next_parslet is nil.
+      # Cannot use << as a method name because Ruby cannot parse it without the self, and self is not allowed as en explicit receiver for private messages.
+      def append(next_parslet)
+        raise ArgumentError if next_parslet.nil?
+        @components << next_parslet.to_parseable
+        update_hash
+        self
+      end
+      
+    end # class ParsletSequence
+  end # class Grammar
+end # module Walrus
+