walrus 0.1

data/lib/walrus/grammar/predicate.rb

@@ -0,0 +1,57 @@
+# Copyright 2007 Wincent Colaiuta
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# in the accompanying file, "LICENSE.txt", for more details.
+#
+# $Id$
+
+require 'walrus'
+
+module Walrus
+  class Grammar
+    
+    # Predicates parse input without consuming it.
+    # On success they throw a subclass-specific symbol (see the AndPredicate and NotPredicate classes).
+    # On failure they raise a ParseError.
+    class Predicate
+      
+      include Walrus::Grammar::ParsletCombining
+      include Walrus::Grammar::Memoizing
+      
+      attr_reader :hash
+      
+      # Raises if parseable is nil.
+      def initialize(parseable)
+        raise ArgumentError if parseable.nil?
+        @parseable = parseable
+        @hash = @parseable.hash + hash_offset # fixed offset to avoid collisions with @parseable objects
+      end
+      
+      def to_parseable
+        self
+      end
+      
+      def parse(string, options = {})
+        raise NotImplementedError # subclass responsibility
+      end
+      
+      def eql?(other)
+        other.instance_of? self.class and other.parseable.eql? @parseable
+      end
+      
+    protected
+      
+      # for equality comparisons
+      attr_reader :parseable
+      
+    private
+      
+      def hash_offset
+        10
+      end
+      
+    end
+    
+  end # class Grammar
+end # module Walrus

data/lib/walrus/grammar/proc_parslet.rb

@@ -0,0 +1,52 @@
+# Copyright 2007 Wincent Colaiuta
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# in the accompanying file, "LICENSE.txt", for more details.
+#
+# $Id$
+
+require 'walrus'
+
+module Walrus
+  class Grammar
+    
+    class ProcParslet < Parslet
+      
+      attr_reader :hash
+      
+      def initialize(proc)
+        raise ArgumentError if proc.nil?
+        self.expected_proc = proc
+      end
+      
+      def parse(string, options = {})
+        raise ArgumentError if string.nil?
+        @expected_proc.call string, options
+      end
+      
+      def eql?(other)
+        other.instance_of? ProcParslet and other.expected_proc == @expected_proc
+      end
+      
+    protected
+      
+      # For equality comparisons.
+      attr_reader :expected_proc
+      
+    private
+      
+      def expected_proc=(proc)
+        @expected_proc = ( proc.clone rescue proc )
+        update_hash
+      end
+      
+      def update_hash
+        @hash = @expected_proc.hash + 105 # fixed offset to avoid collisions with @parseable objects
+      end
+      
+    end # class ProcParslet
+    
+  end # class Grammar
+end # module Walrus
+

data/lib/walrus/grammar/regexp_parslet.rb

@@ -0,0 +1,73 @@
+# Copyright 2007 Wincent Colaiuta
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# in the accompanying file, "LICENSE.txt", for more details.
+#
+# $Id$
+
+require 'walrus'
+
+module Walrus
+  class Grammar
+    
+    class RegexpParslet < Parslet
+      
+      attr_reader :hash
+      
+      def initialize(regexp)
+        raise ArgumentError if regexp.nil?
+        self.expected_regexp = /\A#{regexp}/ # for efficiency, anchor all regexps to the start of the string
+      end
+      
+      def parse(string, options = {})
+        raise ArgumentError if string.nil?
+        if (string =~ @expected_regexp)
+          wrapper = MatchDataWrapper.new($~)
+          match   = $~[0]
+          
+          if (line_count = match.scan(/\r\n|\r|\n/).length) != 0        # count number of newlines in match
+            column_end    = match.jlength - match.jrindex(/\r|\n/) - 1  # calculate characters on last line
+          else                                                          # no newlines in match
+            column_end    = match.jlength + (options[:column_start] || 0)
+          end
+          
+          wrapper.start       = [options[:line_start], options[:column_start]]
+          wrapper.end         = [wrapper.line_start + line_count, column_end]
+          wrapper.source_text = match.to_s.clone
+          wrapper
+        else
+          raise ParseError.new('non-matching characters "%s" while parsing regular expression "%s"' % [string, @expected_regexp.inspect],
+                               :line_end    => (options[:line_start] || 0), :column_end    => (options[:column_start] || 0))
+        end
+      end
+      
+      def eql?(other)
+        other.instance_of? RegexpParslet and other.expected_regexp == @expected_regexp
+      end
+      
+      def inspect
+        '#<%s:0x%x @expected_regexp=%s>' % [self.class.to_s, self.object_id, @expected_regexp.inspect]
+      end
+      
+    protected
+      
+      # For equality comparisons.
+      attr_reader :expected_regexp
+      
+    private
+      
+      def expected_regexp=(regexp)
+        @expected_regexp = ( regexp.clone rescue regexp )
+        update_hash
+      end
+      
+      def update_hash
+        @hash = @expected_regexp.hash + 15 # fixed offset to avoid collisions with @parseable objects
+      end
+      
+    end # class RegexpParslet
+    
+  end # class Grammar
+end # module Walrus
+

data/lib/walrus/grammar/skipped_substring_exception.rb

@@ -0,0 +1,36 @@
+# Copyright 2007 Wincent Colaiuta
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# in the accompanying file, "LICENSE.txt", for more details.
+#
+# $Id$
+
+require 'walrus'
+
+module Walrus
+  class Grammar
+    
+    # I don't really like using Exceptions for non-error situations, but it seems that using throw/catch here would not be adequate (not possible to embed information in the thrown symbol).
+    class SkippedSubstringException < Exception
+      
+      include Walrus::Grammar::LocationTracking
+      
+      def initialize(substring, info = {})
+        super substring
+        
+        # TODO: this code is just like the code in ParseError. could save repeating it by setting up inheritance
+        # but would need to pay careful attention to the ordering of my rescue blocks
+        # and also change many instances of "kind_of" in my specs to "instance_of "
+        # alternatively, could look at using a mix-in
+        self.line_start     = info[:line_start]
+        self.column_start   = info[:column_start]
+        self.line_end       = info[:line_end]
+        self.column_end     = info[:column_end]
+      end
+      
+    end # class SkippedSubstringException
+    
+  end # class Grammar
+end # module Walrus
+

data/lib/walrus/grammar/string_enumerator.rb

@@ -0,0 +1,45 @@
+# Copyright 2007 Wincent Colaiuta
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# in the accompanying file, "LICENSE.txt", for more details.
+#
+# $Id$
+
+require 'strscan'
+require 'walrus'
+
+module Walrus
+  class Grammar
+    
+    # Unicode-aware (UTF-8) string enumerator.
+    # For Unicode support $KCODE must be set to 'U' (UTF-8).
+    class StringEnumerator
+      
+      # Returns the char most recently scanned before the last "next" call, or nil if nothing previously scanned.
+      attr_reader :last
+      
+      def initialize(string)
+        raise ArgumentError if string.nil?
+        @scanner  = StringScanner.new(string)
+        @current  = nil
+        @last     = nil
+      end
+      
+      # This method will only work as expected if $KCODE is set to 'U' (UTF-8).
+      def next
+        @last     = @current
+        @current  = @scanner.scan(/./m) # must use multiline mode or "." won't match newlines
+      end
+      
+      # Take a peek at the next character without actually consuming it. Returns nil if there is no next character.
+      # TODO: consider deleting this method as it's not currently used.
+      def peek
+        if char = self.next : @scanner.unscan; end
+        char
+      end
+      
+    end # class StringEnumerator
+    
+  end # class Grammar
+end # module Walrus

data/lib/walrus/grammar/string_parslet.rb

@@ -0,0 +1,75 @@
+# Copyright 2007 Wincent Colaiuta
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# in the accompanying file, "LICENSE.txt", for more details.
+#
+# $Id$
+
+require 'walrus'
+
+module Walrus
+  class Grammar
+    
+    class StringParslet < Parslet
+      
+      attr_reader :hash
+      
+      def initialize(string)
+        raise ArgumentError if string.nil?
+        self.expected_string = string
+      end
+      
+      def parse(string, options = {})
+        raise ArgumentError if string.nil?
+        chars         = StringEnumerator.new(string)
+        parsed        = StringResult.new
+        parsed.start  = [options[:line_start], options[:column_start]]
+        parsed.end    = parsed.start
+        expected_string.each_char do |expected_char|
+          actual_char = chars.next
+          if actual_char.nil?
+            raise ParseError.new('unexpected end-of-string (expected "%s") while parsing "%s"' % [ expected_char, expected_string ],
+                                 :line_end    => parsed.line_end,   :column_end   => parsed.column_end)
+          elsif actual_char != expected_char
+            raise ParseError.new('unexpected character "%s" (expected "%s") while parsing "%s"' % [ actual_char, expected_char, expected_string],
+                                 :line_end    => parsed.line_end,   :column_end   => parsed.column_end)
+          else
+            if actual_char == "\r" or (actual_char == "\n" and chars.last != "\r")  # catches Mac, Windows and UNIX end-of-line markers
+              parsed.column_end = 0
+              parsed.line_end   = parsed.line_end + 1
+            elsif actual_char != "\n"                     # \n is ignored if it is preceded by an \r (already counted above)
+              parsed.column_end = parsed.column_end + 1   # everything else gets counted
+            end
+            parsed << actual_char
+          end
+        end
+        parsed.source_text = parsed.to_s.clone
+        parsed
+      end
+      
+      def eql?(other)
+        other.instance_of? StringParslet and other.expected_string == @expected_string
+      end
+      
+    protected
+      
+      # For equality comparisons.
+      attr_reader :expected_string
+      
+    private
+      
+      def expected_string=(string)
+        @expected_string = ( string.clone rescue string )
+        update_hash
+      end
+      
+      def update_hash
+        @hash = @expected_string.hash + 20 # fixed offset to avoid collisions with @parseable objects
+      end
+      
+    end # class StringParslet
+    
+  end # class Grammar
+end # module Walrus
+

data/lib/walrus/grammar/string_result.rb

@@ -0,0 +1,24 @@
+# Copyright 2007 Wincent Colaiuta
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# in the accompanying file, "LICENSE.txt", for more details.
+#
+# $Id$
+
+require 'walrus'
+
+module Walrus
+  class Grammar
+    class StringResult < String
+      
+      include Walrus::Grammar::LocationTracking
+      
+      def initialize(string = "")
+        self.source_text = string
+        super
+      end
+      
+    end # class StringResult
+  end # class Grammar
+end # module Walrus

data/lib/walrus/grammar/symbol_parslet.rb

@@ -0,0 +1,63 @@
+# Copyright 2007 Wincent Colaiuta
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# in the accompanying file, "LICENSE.txt", for more details.
+#
+# $Id$
+
+require 'walrus'
+
+module Walrus
+  class Grammar
+    
+    # A SymbolParslet allows for evaluation of a parslet to be deferred until runtime (or parse time, to be more precise).
+    class SymbolParslet < Parslet
+      
+      attr_reader :hash
+      
+      def initialize(symbol)
+        raise ArgumentError if symbol.nil?
+        @symbol = symbol
+        @hash = @symbol.hash + 20 # fixed offset to avoid collisions with @parseable objects
+      end
+      
+      # SymbolParslets don't actually know what Grammar they are associated with at the time of their definition. They expect the Grammar to be passed in with the options hash under the ":grammar" key.
+      # Raises if string is nil, or if the options hash does not include a :grammar key.
+      def parse(string, options = {})
+        raise ArgumentError if string.nil?
+        raise ArgumentError unless options.has_key?(:grammar)
+        grammar = options[:grammar]
+        augmented_options = options.clone
+        augmented_options[:rule_name] = @symbol
+        augmented_options[:skipping_override] = grammar.skipping_overrides[@symbol] if grammar.skipping_overrides.has_key?(@symbol)
+        result = grammar.rules[@symbol].memoizing_parse(string, augmented_options)
+        grammar.wrap(result, @symbol)
+      end
+      
+      # We override the to_s method as it can make parsing error messages more readable. Instead of messages like this:
+      #   predicate not satisfied (expected "#<Walrus::Grammar::SymbolParslet:0x10cd504>") while parsing "hello world"
+      # We can print messages like this:
+      #   predicate not satisfied (expected "rule: end_of_input") while parsing "hello world"
+      def to_s
+        'rule: ' + @symbol.to_s
+      end
+      
+      def ==(other)
+        eql?(other)
+      end
+      
+      def eql?(other)
+        other.instance_of? SymbolParslet and other.symbol == @symbol
+      end
+      
+    protected
+      
+      # For equality comparisons.
+      attr_reader :symbol
+      
+    end # class SymbolParslet
+    
+  end # class Grammar
+end # module Walrus
+

data/lib/walrus/grammar.rb

@@ -0,0 +1,170 @@
+# Copyright 2007 Wincent Colaiuta
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# in the accompanying file, "LICENSE.txt", for more details.
+#
+# $Id: /mirrors/Walrus/trunk/walrus/lib/walrus/grammar.rb 6704 2007-04-09T18:30:00.421185Z wincent  $
+
+require 'walrus'
+
+module Walrus
+  class Grammar
+    
+    attr_accessor :memoizing
+    attr_reader   :rules
+    attr_reader   :skipping_overrides
+    
+    # Creates a Grammar subclass named according to subclass_name and instantiates an instance of the new class, returning it after evaluating the optional block in the context of the newly created instance. The advantage of working inside a new subclass is that any constants defined in the new grammar will be in a separate namespace.
+    # The subclass_name parameter should be a String.
+    def self.subclass(subclass_name, &block)
+      raise ArgumentError if subclass_name.nil?
+      raise ArgumentError if Walrus::const_defined?(subclass_name)
+      Walrus::const_set(subclass_name, Class.new(Grammar))
+      subclass = Walrus::module_eval(subclass_name)
+      begin
+        subclass.new(&block)
+      rescue ContinuationWrapperException => c # a Symbol in a production rule wants to know what namespace its being used in
+        c.continuation.call(subclass)
+      end
+    end
+    
+    def initialize(&block)
+      @rules              = Hash.new { |hash, key| raise StandardError.new('no rule for key "%s"' % key.to_s) }
+      @productions        = Hash.new { |hash, key| raise StandardError.new('no production for key "%s"' % key.to_s) }
+      @skipping_overrides = Hash.new { |hash, key| raise StandardError.new('no skipping override for key "%s"' % key.to_s) }
+      @memoizing          = true
+      self.instance_eval(&block) if block_given?
+    end
+    
+    # TODO: consider making grammars copiable (could be used in threaded context then)
+    #def initialize_copy(from); end
+    #def clone; end
+    #def dupe; end
+    
+    # Starts with starting_symbol.
+    def parse(string, options = {})
+      raise ArgumentError if string.nil?
+      raise StandardError if @starting_symbol.nil?
+      options[:grammar]       = self
+      options[:rule_name]     = @starting_symbol
+      options[:skipping]      = @skipping
+      options[:line_start]    = 0 # "richer" information (more human-friendly) than that provided in "location"
+      options[:column_start]  = 0 # "richer" information (more human-friendly) than that provided in "location"
+      options[:memoizer]      = MemoizingCache.new if @memoizing
+      @starting_symbol.to_parseable.memoizing_parse(string, options)
+    end
+    
+    # Defines a rule and stores it 
+    # Expects an object that responds to the parse message, such as a Parslet or ParsletCombination.
+    # As this is intended to work with Parsing Expression Grammars, each rule may only be defined once. Defining a rule more than once will raise an ArgumentError.
+    def rule(symbol, parseable)
+      raise ArgumentError if symbol.nil?
+      raise ArgumentError if parseable.nil?
+      raise ArgumentError if @rules.has_key?(symbol)
+      @rules[symbol] = parseable
+    end
+    
+    # Dynamically creates a Node subclass inside the namespace of the current grammar. If parent_class is nil, Node is assumed.
+    # new_class_name must not be nil.
+    def node(new_class_name, parent_class = nil, *attributes)
+      raise ArgumentError if new_class_name.nil?
+      new_class_name = new_class_name.to_s.to_class_name # camel-case
+      if parent_class.nil?
+        Node.subclass(new_class_name, self.class, *attributes)
+      else
+        # convert parent_class to string, then camel case, then back to Symbol, then lookup the constant
+        self.class.const_get(parent_class.to_s.to_class_name.to_s).subclass(new_class_name, self.class, *attributes)
+      end
+    end
+    
+    # Specifies the Node subclass that will be used to encapsulate results for the rule identified by the symbol, rule_name.
+    # class_symbol, if present, will be converted to camel-case and explicitly names the class to be used. If class_symbol is not specified then a camel-cased version of the rule_name itself is used.
+    # rule_name must not be nil.
+    #
+    # Example; specifying that the results of rule "string_literal" should be encapsulated in a "StringLiteral" instance:
+    #
+    #   production :string_literal
+    #
+    # Example; specifying that the results of the rule "numeric_literal" should be encapsulated into a "RawToken" instance:
+    #
+    #   production :numeric_literal, :raw_token
+    #
+    # Example; using the "build" method to dynamically define an "AssigmentExpression" class with superclass "Expression" and assign the created class as the AST production class for the rule "assignment_expression":
+    #
+    #   production :assignment_expression.build(:expression, :target, :value)
+    #
+    def production(rule_name, class_symbol = nil)
+      raise ArgumentError if rule_name.nil?
+      raise ArgumentError if @productions.has_key?(rule_name)
+      raise ArgumentError unless @rules.has_key?(rule_name)
+      class_symbol = rule_name if class_symbol.nil?
+      @productions[rule_name] = class_symbol
+    end
+    
+    def wrap(result, rule_name)
+      if @productions.has_key?(rule_name.to_sym)    # figure out arity of "initialize" method and wrap results in AST node
+        node_class  = self.class.const_get(@productions[rule_name.to_sym].to_s.to_class_name)
+        param_count = node_class.instance_method(:initialize).arity
+        raise if param_count < 1
+        
+        # dynamically build up a message send
+        if param_count == 1
+          params = 'result'
+        else
+          params = 'result[0]'
+          for i in 1..(param_count - 1)
+            params << ", result[#{i.to_s}]"
+          end
+        end
+        
+        node                = node_class.class_eval('new(%s)' % params)
+        node.start          = (result.outer_start or result.start)              # propagate the start information
+        node.end            = (result.outer_end or result.end)                  # and the end information
+        node.source_text    = (result.outer_source_text or result.source_text)  # and the original source text
+        node
+      else
+        result.start        = result.outer_start if result.outer_start
+        result.end          = result.outer_end if result.outer_end
+        result.source_text  = result.source_text if result.outer_source_text
+        result
+      end
+    end
+    
+    # Sets the starting symbol.
+    # symbol must refer to a rule.
+    def starting_symbol(symbol)
+      @starting_symbol = symbol
+    end
+    
+    # Sets the default parslet that is used for skipping inter-token whitespace, and can be used to override the default on a rule-by-rule basis.
+    # This allows for simpler grammars which do not need to explicitly put optional whitespace parslets (or any other kind of parslet) between elements.
+    #
+    # There are two modes of operation for this method. In the first mode (when only one parameter is passed) the rule_or_parslet parameter is used to define the default parslet for inter-token skipping. rule_or_parslet must refer to a rule which itself is a Parslet or ParsletCombination and which is responsible for skipping. Note that the ability to pass an arbitrary parslet means that the notion of what consitutes the "whitespace" that should be skipped is completely flexible. Raises if a default skipping parslet has already been set.
+    #
+    # In the second mode of operation (when two parameters are passed) the rule_or_parslet parameter is interpreted to be the rule to which an override should be applied, where the parslet parameter specifies the parslet to be used in this case. If nil is explicitly passed then this overrides the default parslet; no parslet will be used for the purposes of inter-token skipping. Raises if an override has already been set for the named rule.
+    #
+    # The inter-token parslet is passed inside the "options" hash when invoking the "parse" methods. Any parser which fails will retry after giving this inter-token parslet a chance to consume and discard intervening whitespace.
+    # The initial, conservative implementation only performs this fallback skipping for ParsletSequence and ParsletRepetition combinations.
+    #
+    # Raises if rule_or_parslet is nil.
+    def skipping(rule_or_parslet, parslet = NoParameterMarker.instance)
+      raise ArgumentError if rule_or_parslet.nil?
+      if parslet == NoParameterMarker.instance  # first mode of operation: set default parslet
+        raise if @skipping                      # should not set a default skipping parslet twice
+        @skipping = rule_or_parslet
+      else                                      # second mode of operation: override default case
+        raise ArgumentError if @skipping_overrides.has_key?(rule_or_parslet)
+        raise ArgumentError unless @rules.has_key?(rule_or_parslet)
+        @skipping_overrides[rule_or_parslet] = parslet
+      end
+    end
+    
+    # TODO: pretty print method?
+    
+  end # class Grammar
+end # module Walrus
+
+
+
+

data/lib/walrus/no_parameter_marker.rb

@@ -0,0 +1,19 @@
+# Copyright 2007 Wincent Colaiuta
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# in the accompanying file, "LICENSE.txt", for more details.
+#
+# $Id$
+
+require 'walrus'
+
+module Walrus
+  
+  class NoParameterMarker
+    require 'singleton'
+    include Singleton
+  end
+  
+end # module Walrus
+