walrus 0.1

data/lib/walrus/grammar/memoizing_cache.rb

@@ -0,0 +1,94 @@
+# Copyright 2007 Wincent Colaiuta
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# in the accompanying file, "LICENSE.txt", for more details.
+#
+# $Id$
+
+require 'walrus'
+
+module Walrus
+  class Grammar
+    
+    # The MemoizingCache class memoizes the outcomes of parse operations. The functionality is implemented as a separate class so as to minimize the amount of "contamination" of other classes by memoizing code, and to allow memoizing to be cleanly turned on or off at will. If a MemoizingCache is passed to a Parslet, ParsletCombination or Predicate as a value for the :memoizer key in the options hash passed to a parse method, the class implementing that method will call the parse method on the cache rather than proceeding normally. The cache will either propagate the previously memoized result, or will defer back to the original class to obtain the result. A circular dependency is avoided by setting the :skip_memoizer flag in the options dictionary. If no MemoizingCache is passed then normal program flow takes place.
+    class MemoizingCache
+      
+      # Singleton class that serves as a default value for unset keys in a Hash.
+      class NoValueForKey
+        
+        require 'singleton'
+        include Singleton
+        
+      end
+      
+      def initialize
+        # The results of parse operations are stored (memoized) in a cache, keyed on a unique identifier comprising the Parslet, ParsletCombination or Predicate used in the parse operation, the location of the operation (the line_start and column_start), and the skipping override (if any). The values may be:
+        #
+        #   - ParseErrors raised during parsing
+        #   - SkippedSubstringExceptions raised during parsing
+        #   - :ZeroWidthParseSuccess symbols thrown during parsing
+        #   - :AndPredicateSuccess symbols thrown during parsing
+        #   - :NotPredicateSuccess symbols thrown during parsing
+        #   - String instances returned as parse results
+        #   - MatchDataWrapper instance returned as parse results
+        #   - Array instances containing ordered collections of parse results
+        #   - Node subclass instances containing AST productions
+        #
+        @cache = Hash.new(NoValueForKey.instance)
+      end
+      
+      # The receiver checks whether there is already a stored result corresponding to that a unique identifier that specifies the "coordinates" of a parsing operation (location, parseable, skipping override). If found propogates the result directly to the caller rather than performing the parse method all over again. Here "propagation" means re-raising parse errors, re-throwing symbols, and returning object references. If not found, performs the parsing operation and stores the result in the cache before propagating it.
+      def parse(string, options = {})
+        raise ArgumentError if string.nil?
+        
+        # construct a unique identifier
+        identifier = [options[:parseable], options[:line_start], options[:column_start]]
+        identifier << options[:origin] if options.has_key? :origin
+        identifier << options[:skipping_override] if options.has_key? :skipping_override
+        
+        if (result = @cache[identifier]) != NoValueForKey.instance
+          if result.kind_of? Symbol       : throw result
+          elsif result.kind_of? Exception : raise result
+          else                              return result
+          end
+        else    # first time for this parseable/location/skipping_override (etc) combination, capture result and propagate
+          catch :NotPredicateSuccess do
+            catch :AndPredicateSuccess do
+              catch :ZeroWidthParseSuccess do
+                begin
+                  options[:ignore_memoizer] = true
+                  
+                  # short-circuit left recursion here rather than infinite looping
+                  if options[:parseable].kind_of? SymbolParslet
+                    check_left_recursion(options[:parseable], options)
+                    @last_seen_symbol_parslet             = options[:parseable]
+                    @last_seen_symbol_parslet_location    = [options[:line_start], options[:column_start]]
+                  end
+                  
+                  return @cache[identifier] = options[:parseable].memoizing_parse(string, options)  # store and return
+                rescue Exception => e
+                  raise @cache[identifier] = e                                                      # store and re-raise
+                end
+              end
+              throw @cache[identifier] = :ZeroWidthParseSuccess                                     # store and re-throw
+            end
+            throw @cache[identifier] = :AndPredicateSuccess                                         # store and re-throw
+          end
+          throw @cache[identifier] = :NotPredicateSuccess                                           # store and re-throw
+        end
+      end
+      
+      def check_left_recursion(parseable, options = {})
+        if parseable.kind_of? SymbolParslet and
+           @last_seen_symbol_parslet == parseable and
+           @last_seen_symbol_parslet_location == [options[:line_start], options[:column_start]]
+          raise LeftRecursionException
+        end
+      end
+      
+    end # class MemoizingCache
+    
+  end # class Grammar
+end # module Walrus
+

data/lib/walrus/grammar/node.rb

@@ -0,0 +1,60 @@
+# Copyright 2007 Wincent Colaiuta
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# in the accompanying file, "LICENSE.txt", for more details.
+#
+# $Id$
+
+require 'walrus'
+
+module Walrus
+  class Grammar
+    
+    # Make subclasses of this for us in Abstract Syntax Trees (ASTs).
+    class Node
+      
+      include Walrus::Grammar::LocationTracking
+      
+      def to_s
+        @string_value
+      end
+      
+      # Dynamically creates a Node descendant.
+      # subclass_name should be a Symbol or String containing the name of the subclass to be created.
+      # namespace should be the module in which the new subclass should be created; it defaults to Walrus::Grammar.
+      # results are optional symbols expected to be parsed when initializing an instance of the subclass. If no optional symbols are provided then a default initializer is created that expects a single parameter and stores a reference to it in an instance variable called "lexeme".
+      def self.subclass(subclass_name, namespace = Walrus::Grammar, *results)
+        raise ArgumentError if subclass_name.nil?
+        
+        # create new anonymous class with Node as superclass, assigning it to a constant effectively names the class
+        new_class = namespace.const_set(subclass_name.to_s, Class.new(self))
+        
+        # set up accessors
+        for result in results
+          new_class.class_eval { attr_reader result }
+        end
+        
+        # set up initializer
+        if results.length == 0 # default case, store sole parameter in "lexeme"
+          new_class.class_eval { attr_reader :lexeme }
+          initialize_body = "def initialize(lexeme)\n"
+          initialize_body << "@string_value = lexeme.to_s\n"
+          initialize_body << "@lexeme = lexeme\n"
+        else
+          initialize_body = "def initialize(#{results.collect { |symbol| symbol.to_s}.join(', ')})\n"
+          initialize_body << "@string_value = \"\"\n"
+          for result in results
+            initialize_body << "@#{result.to_s} = #{result.to_s}\n"
+            initialize_body << "@string_value << #{result.to_s}.to_s\n"
+          end
+        end
+        initialize_body << "end\n"
+        new_class.class_eval initialize_body        
+        new_class
+      end
+      
+    end # class Node
+    
+  end # class Grammar
+end # module Walrus

data/lib/walrus/grammar/not_predicate.rb

@@ -0,0 +1,40 @@
+# Copyright 2007 Wincent Colaiuta
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# in the accompanying file, "LICENSE.txt", for more details.
+#
+# $Id$
+
+require 'walrus'
+
+module Walrus
+  class Grammar
+    
+    class NotPredicate < Predicate
+      
+      def parse(string, options = {})
+        raise ArgumentError if string.nil?
+        catch :ZeroWidthParseSuccess do
+          begin
+            @parseable.memoizing_parse(string, options)
+          rescue ParseError # failed to pass (which is just what we wanted)
+            throw :NotPredicateSuccess
+          end
+        end
+        
+        # getting this far means that parsing succeeded (not what we wanted)
+        raise ParseError.new('predicate not satisfied ("%s" not allowed) while parsing "%s"' % [@parseable.to_s, string],
+                             :line_end => options[:line_start], :column_end => options[:column_start])
+      end
+      
+    private
+      
+      def hash_offset
+        11
+      end
+      
+    end
+    
+  end # class Grammar
+end # module Walrus

data/lib/walrus/grammar/parse_error.rb

@@ -0,0 +1,39 @@
+# Copyright 2007 Wincent Colaiuta
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# in the accompanying file, "LICENSE.txt", for more details.
+#
+# $Id$
+
+require 'walrus'
+
+module Walrus
+  class Grammar
+    
+    class ParseError < Exception
+      
+      include Walrus::Grammar::LocationTracking
+      
+      # Takes an optional hash (for packing extra info into exception).
+      # position in string (irrespective of line number, column number)
+      # line number, column number
+      # filename
+      def initialize(message, info = {})
+        super message
+        self.line_start     = info[:line_start]
+        self.column_start   = info[:column_start]
+        self.line_end       = info[:line_end]
+        self.column_end     = info[:column_end]
+      end
+      
+      def inspect
+        # TODO also return filename if available
+        '#<%s: %s @line_end=%d, @column_end=%d>' % [ self.class.to_s, self.to_s, self.line_end, self.column_end ]
+      end
+      
+    end # class ParseError
+    
+  end # class Grammar
+end # module Walrus
+

data/lib/walrus/grammar/parser_state.rb

@@ -0,0 +1,181 @@
+# Copyright 2007 Wincent Colaiuta
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# in the accompanying file, "LICENSE.txt", for more details.
+#
+# $Id: /mirrors/Walrus/trunk/walrus/lib/walrus/grammar/parser_state.rb 6704 2007-04-09T18:30:00.421185Z wincent  $
+
+require 'walrus'
+
+module Walrus
+  class Grammar
+    
+    # Simple class for maintaining state during a parse operation.
+    class ParserState
+      
+      attr_reader :options
+      
+      # Returns the remainder (the unparsed portion) of the string. Will return an empty string if already at the end of the string.
+      attr_reader :remainder
+      
+      # Raises an ArgumentError if string is nil.
+      def initialize(string, options = {})
+        raise ArgumentError if string.nil?
+        self.base_string        = string
+        @results                = ArrayResult.new                     # for accumulating results
+        @remainder              = @base_string.clone
+        @scanned                = ''
+        @options                = options.clone
+        
+        # start wherever we last finished (doesn't seem to behave different to the alternative)
+        @options[:line_start]   = (@options[:line_end] or @options[:line_start] or 0)
+        @options[:column_start] = (@options[:column_end] or @options[:column_start] or 0)
+#        @options[:line_start]   = 0 if @options[:line_start].nil?
+#        @options[:column_start] = 0 if @options[:column_start].nil?
+
+        @options[:line_end]     = @options[:line_start]               # before parsing begins, end point is equal to start point
+        @options[:column_end]   = @options[:column_start]
+        @original_line_start    = @options[:line_start]
+        @original_column_start  = @options[:column_start]
+      end
+      
+      # The parsed method is used to inform the receiver of a successful parsing event.
+      # Note that substring need not actually be a String but it must respond to the following messages:
+      #   - "line_end" and "column_end" so that the end position of the receiver can be updated
+      # As a convenience returns the remainder.
+      # Raises an ArgumentError if substring is nil.
+      def parsed(substring)
+        raise ArgumentError if substring.nil?
+        update_and_return_remainder_for_string(substring, true)
+      end
+      
+      # The skipped method is used to inform the receiver of a successful parsing event where the parsed substring should be consumed but not included in the accumulated results.
+      # The substring should respond to "line_end" and "column_end".
+      # In all other respects this method behaves exactly like the parsed method.
+      def skipped(substring)
+        raise ArgumentError if substring.nil?
+        update_and_return_remainder_for_string(substring)
+      end
+      
+      # The skipped method is used to inform the receiver of a successful parsing event where the parsed substring should be consumed but not included in the accumulated results and furthermore the parse event should not effect the overall bounds of the parse result. In reality this means that the method is only ever called upon the successful use of a automatic intertoken "skipping" parslet. By definition this method should only be called for intertoken skipping otherwise incorrect results will be produced.
+      def auto_skipped(substring)
+        raise ArgumentError if substring.nil?
+        a, b, c, d = @options[:line_start], @options[:column_start], @options[:line_end], @options[:column_end] # save
+        remainder = update_and_return_remainder_for_string(substring)
+        @options[:line_start], @options[:column_start], @options[:line_end], @options[:column_end] = a, b, c, d # restore
+        remainder
+      end
+      
+      # Returns the results accumulated so far.
+      # Returns an empty array if no results have been accumulated.
+      # Returns a single object if only one result has been accumulated.
+      # Returns an array of objects if multiple results have been accumulated.
+      def results
+        
+        updated_start       = [@original_line_start, @original_column_start]
+        updated_end         = [@options[:line_end], @options[:column_end]]
+        updated_source_text = @scanned.clone
+        
+        if @results.length == 1
+          
+          # he we ask the single result to exhibit container-like properties
+          # use the "outer" variants so as to not overwrite any data internal to the result itself
+          # this can happen where a lone result is surrounded only by skipped elements
+          # the result has to convey data about its own limits, plus those of the context just around it
+          results = @results[0]
+          results.outer_start       = updated_start if results.start != updated_start
+          results.outer_end         = updated_end if results.end != updated_end
+          results.outer_source_text = updated_source_text if results.source_text != updated_source_text
+          
+          # the above trick fixes some of the location tracking issues but opens up another can of worms
+          # uncomment this line to see
+          #return results
+          
+          # need some way of handling unwrapped results (raw results, not AST nodes) as well
+          results.start             = updated_start
+          results.end               = updated_end
+          results.source_text       = updated_source_text
+          
+        else
+          results = @results
+          results.start             = updated_start
+          results.end               = updated_end
+          results.source_text       = updated_source_text
+        end        
+        
+        results
+      end
+      
+      # Returns the number of results accumulated so far.
+      def length
+        @results.length
+      end
+      
+      # TODO: possibly implement "undo/rollback" and "reset" methods
+      # if I implement "undo" will probbaly do it as a stack
+      # will have the option of implementing "redo" as well but I'd only do that if I could think of a use for it
+      
+    private
+      
+      def update_and_return_remainder_for_string(input, store = false)
+        previous_line_end       = @options[:line_end]                                           # remember old end point
+        previous_column_end     = @options[:column_end]                                         # remember old end point
+        
+        # special case handling for literal String objects
+        if input.instance_of? String
+          input = StringResult.new(input)
+          input.start = [previous_line_end, previous_column_end]
+          if (line_count  = input.scan(/\r\n|\r|\n/).length) != 0       # count number of newlines in receiver
+            column_end    = input.jlength - input.jrindex(/\r|\n/) - 1  # calculate characters on last line
+          else                                                          # no newlines in match
+            column_end    = input.jlength + previous_column_end
+          end
+          input.end   = [previous_line_end + line_count, column_end]
+        end
+        
+        @results << input if (store)
+        
+        if input.line_end > previous_line_end       # end line has advanced
+          @options[:line_end]   = input.line_end
+          @options[:column_end] = 0
+        end
+        
+        if input.column_end > @options[:column_end] # end column has advanced
+          @options[:column_end] = input.column_end
+        end
+        
+        
+        @options[:line_start]   = @options[:line_end]                                           # new start point is old end point
+        @options[:column_start] = @options[:column_end]                                         # new start point is old end point
+        
+        # calculate remainder
+        line_delta              = @options[:line_end] - previous_line_end
+        if line_delta > 0                                               # have consumed newline(s)
+          line_delta.times do                                           # remove them from remainder
+            newline_location    = @remainder.jindex /\r\n|\r|\n/        # find the location of the next newline
+            newline_location    += $~[0].length                         # add the actual characters used to indicate the newline
+            @scanned            << @remainder[0...newline_location]     # record scanned text
+            @remainder          = @remainder[newline_location..-1]      # strip everything up to and including the newline
+          end
+          @scanned              << @remainder[0...@options[:column_end]]
+          @remainder            = @remainder[@options[:column_end]..-1] # delete up to the current column
+        else                                                            # no newlines consumed
+          column_delta          = @options[:column_end] - previous_column_end
+          if column_delta > 0                                           # there was movement within currentline
+            @scanned            << @remainder[0...column_delta]
+            @remainder          = @remainder[column_delta..-1]          # delete up to the current column
+          end
+        end
+        @remainder
+      end
+      
+      def base_string=(string)
+        @base_string = (string.clone rescue string)
+      end
+      
+    end # class ParserState
+    
+  end # class Grammar
+end # module Walrus
+

data/lib/walrus/grammar/parslet.rb

@@ -0,0 +1,28 @@
+# Copyright 2007 Wincent Colaiuta
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# in the accompanying file, "LICENSE.txt", for more details.
+#
+# $Id$
+
+require 'walrus'
+
+module Walrus
+  class Grammar
+    class Parslet
+      
+      include Walrus::Grammar::ParsletCombining
+      include Walrus::Grammar::Memoizing
+      
+      def to_parseable
+        self
+      end
+      
+      def parse(string, options = {})
+        raise NotImplementedError # subclass responsibility
+      end
+      
+    end # class Parslet
+  end # class Grammar
+end # module Walrus

data/lib/walrus/grammar/parslet_choice.rb

@@ -0,0 +1,120 @@
+# Copyright 2007 Wincent Colaiuta
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# in the accompanying file, "LICENSE.txt", for more details.
+#
+# $Id$
+
+require 'walrus'
+
+module Walrus
+  class Grammar
+    
+    class ParsletChoice < ParsletCombination
+      
+      attr_reader :hash
+      
+      # Either parameter may be a Parslet or a ParsletCombination.
+      # Neither parmeter may be nil.
+      def initialize(left, right, *others)
+        raise ArgumentError if left.nil?
+        raise ArgumentError if right.nil?
+        @alternatives = [left, right] + others
+        update_hash
+      end
+      
+      # Override so that alternatives are appended to an existing sequence:
+      # Consider the following example:
+      #     A | B
+      # This constitutes a single choice:
+      #     (A | B)
+      # If we then make this a three-element sequence:
+      #     A | B | C
+      # We are effectively creating an nested sequence containing the original sequence and an additional element:
+      #     ((A | B) | C)
+      # Although such a nested sequence is correctly parsed it is not as architecturally clean as a single sequence without nesting:
+      #     (A | B | C)
+      # This method allows us to use the architecturally cleaner format.
+      #
+      def |(next_parslet)
+        append(next_parslet)
+      end
+      
+      # First tries to parse the left option, falling back and trying the right option and then the any subsequent options in the others instance variable on failure. If no options successfully complete parsing then an ParseError is raised. Any zero-width parse successes thrown by alternative parsers will flow on to a higher level.
+      def parse(string, options = {})
+        raise ArgumentError if string.nil?
+        error           = nil # for error reporting purposes will track which parseable gets farthest to the right before failing
+        left_recursion  = nil # will also track any left recursion that we detect
+        @alternatives.each do |parseable|
+          begin
+            result = parseable.memoizing_parse(string, options) # successful parse
+            if left_recursion and left_recursion.continuation   # and we have a continuation
+              continuation = left_recursion.continuation        # continuations are once-only, one-way tickets
+              left_recursion = nil                              # set this to nil so as not to call it again without meaning to
+              continuation.call(result)                         # so jump back to where we were before
+            end
+            return result
+          rescue LeftRecursionException => e
+            left_recursion = e
+            
+            # TODO:
+            # it's not enough to just catch this kind of exception and remember the last one
+            # may need to accumulate these in an array
+            # consider the example rule:
+            #   :a, :a & :b | :a & :c | :a & :d | :b
+            # the first option will raise a LeftRecursionException
+            # the next option will raise for the same reason
+            # the third likewise
+            # finally we get to the fourth option, the first which might succeed
+            # at that point we should have three continuations
+            # we should try the first, falling back to the second and third if necessary
+            # on successfully retrying, need to start all over again and try all the options again, just in case further recursion is possible
+            # so it is quite complicated
+            # the question is, is it more complicated than the other ways of getting right-associativity into Walrus-generated parsers?
+            
+          rescue ParseError => e
+            if error.nil?   :   error = e
+            else                error = e unless error.rightmost?(e)
+            end
+          end
+        end
+        raise ParseError.new('no valid alternatives while parsing "%s" (%s)' % [string, error.to_s],
+                             :line_end => error.line_end, :column_end => error.column_end)                  # should generally report the rightmost error
+      end
+      
+      def eql?(other)
+        return false if not other.instance_of? ParsletChoice
+        other_alternatives = other.alternatives
+        return false if @alternatives.length != other_alternatives.length
+        for i in 0..(@alternatives.length - 1)
+          return false unless @alternatives[i].eql? other_alternatives[i]
+        end
+        true
+      end
+      
+    protected
+      
+      # For determining equality.
+      attr_reader :alternatives
+      
+    private
+      
+      def update_hash
+        @hash = 30 # fixed offset to avoid unwanted collisions with similar classes
+        @alternatives.each { |parseable| @hash += parseable.hash }
+      end
+      
+      # Appends another Parslet (or ParsletCombination) to the receiver and returns the receiver.
+      # Raises if parslet is nil.
+      # Cannot use << as a method name because Ruby cannot parse it without the self, and self is not allowed as en explicit receiver for private messages.
+      def append(next_parslet)
+        raise ArgumentError if next_parslet.nil?
+        @alternatives << next_parslet.to_parseable
+        update_hash
+        self
+      end
+      
+    end # class ParsletChoice
+  end # class Grammar
+end # module Walrus

data/lib/walrus/grammar/parslet_combination.rb

@@ -0,0 +1,26 @@
+# Copyright 2007 Wincent Colaiuta
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# in the accompanying file, "LICENSE.txt", for more details.
+#
+# $Id$
+
+require 'walrus'
+
+module Walrus
+  class Grammar
+    
+    class ParsletCombination
+      
+      include Walrus::Grammar::ParsletCombining
+      include Walrus::Grammar::Memoizing
+      
+      def to_parseable
+        self
+      end
+      
+    end # module ParsletCombination
+    
+  end # class Grammar
+end # module Walrus