sexp_processor 3.0.0

data/History.txt

@@ -0,0 +1,15 @@
+=== 3.0.0 / 2008-10-22
+
+* 2 major enhancements:
+
+  * Released as its own project, splitting from ParseTree
+  * Added Environment to SexpProcessor and built it in. YAY!
+
+* 6 minor enhancements:
+
+  * Allowed CompositeSexpProcessor to be more ducktypey.
+  * Refactored Sexp#method_missing into find_node and find_nodes.
+  * Removed Sexp#for and other PT specific code.
+  * SexpProcessor#process now runs rewriters before everything else.
+  * SexpProcessor#rewrite context only for subs, EMPTY for top level rewrites.
+  * SexpProcessor#rewrite will stop iterating if the result isn't another Sexp.

data/Manifest.txt

@@ -0,0 +1,11 @@
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+lib/composite_sexp_processor.rb
+lib/sexp.rb
+lib/sexp_processor.rb
+test/test_composite_sexp_processor.rb
+test/test_environment.rb
+test/test_sexp.rb
+test/test_sexp_processor.rb

data/README.txt

@@ -0,0 +1,48 @@
+= SexpProcessor
+
+* FIX (url)
+
+== DESCRIPTION:
+
+FIX (describe your package)
+
+== FEATURES/PROBLEMS:
+
+* FIX (list of features or problems)
+
+== SYNOPSIS:
+
+  FIX (code sample of usage)
+
+== REQUIREMENTS:
+
+* FIX (list of requirements)
+
+== INSTALL:
+
+* FIX (sudo gem install, anything else)
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2008 FIX
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,15 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+
+Hoe.add_include_dirs "lib"
+
+require './lib/sexp_processor.rb'
+
+Hoe.new('sexp_processor', SexpProcessor::VERSION) do |sexp|
+  sexp.rubyforge_name = 'parsetree'
+  sexp.developer('Ryan Davis', 'ryand-ruby@zenspider.com')
+end
+
+# vim: syntax=Ruby

data/lib/composite_sexp_processor.rb

@@ -0,0 +1,49 @@
+require 'sexp_processor'
+
+##
+# Implements the Composite pattern on SexpProcessor. Need we say more?
+#
+# Yeah... probably. Implements a SexpProcessor of SexpProcessors so
+# you can easily chain multiple to each other. At some stage we plan
+# on having all of them run +process+ and but only ever output
+# something when +generate+ is called, allowing for deferred final
+# processing.
+
+class CompositeSexpProcessor < SexpProcessor
+
+  ##
+  # The list o' processors to run.
+
+  attr_reader :processors
+
+  def initialize # :nodoc:
+    super
+    @processors = []
+  end
+
+  ##
+  # Add a +processor+ to the list of processors to run.
+
+  def <<(processor)
+    raise ArgumentError, "Can only add sexp processors" unless
+      SexpProcessor === processor || processor.respond_to?(:process)
+    @processors << processor
+  end
+
+  ##
+  # Run +exp+ through all of the processors, returning the final
+  # result.
+
+  def process(exp)
+    @processors.each do |processor|
+      exp = processor.process(exp)
+    end
+    exp
+  end
+
+  def on_error_in(node_type, &block)
+    @processors.each do |processor|
+      processor.on_error_in(node_type, &block)
+    end
+  end
+end

data/lib/sexp.rb

@@ -0,0 +1,270 @@
+
+$TESTING ||= false # unless defined $TESTING
+
+##
+# Sexps are the basic storage mechanism of SexpProcessor.  Sexps have
+# a +type+ (to be renamed +node_type+) which is the first element of
+# the Sexp. The type is used by SexpProcessor to determine whom to
+# dispatch the Sexp to for processing.
+
+class Sexp < Array # ZenTest FULL
+
+  @@array_types = [ :array, :args, ]
+
+  ##
+  # Create a new Sexp containing +args+.
+
+  def initialize(*args)
+    super(args)
+  end
+
+  ##
+  # Creates a new Sexp from Array +a+.
+
+  def self.from_array(a)
+    ary = Array === a ? a : [a]
+
+    result = self.new
+
+    ary.each do |x|
+      case x
+      when Sexp
+        result << x
+      when Array
+        result << self.from_array(x)
+      else
+        result << x
+      end
+    end
+
+    result
+  end
+
+  def ==(obj) # :nodoc:
+    if obj.class == self.class then
+      super
+    else
+      false
+    end
+  end
+
+  ##
+  # Returns true if this Sexp's pattern matches +sexp+.
+
+  def ===(sexp)
+    return nil unless Sexp === sexp
+    pattern = self # this is just for my brain
+
+    return true if pattern == sexp
+
+    sexp.each do |subset|
+      return true if pattern === subset
+    end
+
+    return nil
+  end
+
+  ##
+  # Returns true if this Sexp matches +pattern+.  (Opposite of #===.)
+
+  def =~(pattern)
+    return pattern === self
+  end
+
+  ##
+  # Returns true if the node_type is +array+ or +args+.
+  #
+  # REFACTOR: to TypedSexp - we only care when we have units.
+
+  def array_type?
+    type = self.first
+    @@array_types.include? type
+  end
+
+  def compact # :nodoc:
+    self.delete_if { |o| o.nil? }
+  end
+
+  ##
+  # Enumeratates the sexp yielding to +b+ when the node_type == +t+.
+
+  def each_of_type(t, &b)
+    each do | elem |
+      if Sexp === elem then
+        elem.each_of_type(t, &b)
+        b.call(elem) if elem.first == t
+      end
+    end
+  end
+
+  ##
+  # Replaces all elements whose node_type is +from+ with +to+. Used
+  # only for the most trivial of rewrites.
+
+  def find_and_replace_all(from, to)
+    each_with_index do | elem, index |
+      if Sexp === elem then
+        elem.find_and_replace_all(from, to)
+      else
+        self[index] = to if elem == from
+      end
+    end
+  end
+
+  ##
+  # Replaces all Sexps matching +pattern+ with Sexp +repl+.
+
+  def gsub(pattern, repl)
+    return repl if pattern == self
+
+    new = self.map do |subset|
+      case subset
+      when Sexp then
+        subset.gsub(pattern, repl)
+      else
+        subset
+      end
+    end
+
+    return Sexp.from_array(new)
+  end
+
+  def inspect # :nodoc:
+    sexp_str = self.map {|x|x.inspect}.join(', ')
+    return "s(#{sexp_str})"
+  end
+
+  def find_node name, delete = false
+    matches = find_nodes name
+
+    case matches.size
+    when 0 then
+      nil
+    when 1 then
+      match = matches.first
+      delete match if delete
+      match
+    else
+      raise NoMethodError, "multiple nodes for #{name} were found in #{inspect}"
+    end
+  end
+
+  def find_nodes name
+    find_all { | sexp | Sexp === sexp and sexp.first == name }
+  end
+
+  ##
+  # Returns the node named +node+, deleting it if +delete+ is true.
+
+  def method_missing meth, delete = false
+    find_node meth, delete
+  end
+
+  def pretty_print(q) # :nodoc:
+    q.group(1, 's(', ')') do
+      q.seplist(self) {|v| q.pp v }
+    end
+  end
+
+  ##
+  # Returns the Sexp without the node_type.
+
+  def sexp_body
+    self[1..-1]
+  end
+
+  ##
+  # If run with debug, Sexp will raise if you shift on an empty
+  # Sexp. Helps with debugging.
+
+  def shift
+    raise "I'm empty" if self.empty?
+    super
+  end if $DEBUG or $TESTING
+
+  ##
+  # Returns the bare bones structure of the sexp.
+  # s(:a, :b, s(:c, :d), :e) => s(:a, s(:c))
+
+  def structure
+    result = self.class.new
+    if Array === self.first then
+      result = self.first.structure
+    else
+      result << self.first
+      self.grep(Array).each do |subexp|
+        result << subexp.structure
+      end
+    end
+    result
+  end
+
+  ##
+  # Replaces the Sexp matching +pattern+ with +repl+.
+
+  def sub(pattern, repl)
+    return repl.dup if pattern == self
+
+    done = false
+
+    new = self.map do |subset|
+      if done then
+        subset
+      else
+        case subset
+        when Sexp then
+          if pattern == subset then
+            done = true
+            repl.dup
+          elsif pattern === subset then
+            done = true
+            subset.sub pattern, repl
+          else
+            subset
+          end
+        else
+          subset
+        end
+      end
+    end
+
+    return Sexp.from_array(new)
+  end
+
+  def to_a # :nodoc:
+    self.map { |o| Sexp === o ? o.to_a : o }
+  end
+
+  def to_s # :nodoc:
+    inspect
+  end
+
+end
+
+class SexpMatchSpecial < Sexp; end
+
+class SexpAny < SexpMatchSpecial
+  def ==(o)
+    Sexp === o
+  end
+
+  def ===(o)
+    return Sexp === o
+  end
+
+  def inspect
+    "ANY"
+  end
+end
+
+module SexpMatchSpecials
+  def ANY(); return SexpAny.new; end
+end
+
+##
+# This is just a stupid shortcut to make indentation much cleaner.
+
+def s(*args)
+  Sexp.new(*args)
+end
+

data/lib/sexp_processor.rb

@@ -0,0 +1,407 @@
+
+$TESTING = false unless defined? $TESTING
+
+require 'sexp'
+
+##
+# SexpProcessor provides a uniform interface to process Sexps.
+#
+# In order to create your own SexpProcessor subclass you'll need
+# to call super in the initialize method, then set any of the
+# Sexp flags you want to be different from the defaults.
+#
+# SexpProcessor uses a Sexp's type to determine which process method
+# to call in the subclass.  For Sexp <code>s(:lit, 1)</code>
+# SexpProcessor will call #process_lit, if it is defined.
+#
+# You can also specify a default method to call for any Sexp types
+# without a process_<type> method or use the default processor provided to
+# skip over them.
+#
+# Here is a simple example:
+#
+#   class MyProcessor < SexpProcessor
+#     def initialize
+#       super
+#       self.strict = false
+#     end
+#
+#     def process_lit(exp)
+#       val = exp.shift
+#       return val
+#     end
+#   end
+
+class SexpProcessor
+
+  VERSION = '3.0.0'
+
+  ##
+  # Automatically shifts off the Sexp type before handing the
+  # Sexp to process_<type>
+
+  attr_accessor :auto_shift_type
+
+  ##
+  # Return a stack of contexts. Most recent node is first.
+
+  attr_reader :context
+
+  ##
+  # A Hash of Sexp types and Regexp.
+  #
+  # Print a debug message if the Sexp type matches the Hash key
+  # and the Sexp's #inspect output matches the Regexp.
+
+  attr_accessor :debug
+
+  ##
+  # A default method to call if a process_<type> method is not found
+  # for the Sexp type.
+
+  attr_accessor :default_method
+
+  ##
+  # Expected result class
+
+  attr_accessor :expected
+
+  ##
+  # Raise an exception if the Sexp is not empty after processing
+
+  attr_accessor :require_empty
+
+  ##
+  # Raise an exception if no process_<type> method is found for a Sexp.
+
+  attr_accessor :strict
+
+  ##
+  # An array that specifies node types that are unsupported by this
+  # processor. SexpProcessor will raise UnsupportedNodeError if you try
+  # to process one of those node types.
+
+  attr_accessor :unsupported
+
+  ##
+  # Emit a warning when the method in #default_method is called.
+
+  attr_accessor :warn_on_default
+
+  ##
+  # A scoped environment to make you happy.
+
+  attr_reader :env
+
+  ##
+  # Creates a new SexpProcessor.  Use super to invoke this
+  # initializer from SexpProcessor subclasses, then use the
+  # attributes above to customize the functionality of the
+  # SexpProcessor
+
+  def initialize
+    @default_method      = nil
+    @warn_on_default     = true
+    @auto_shift_type     = false
+    @strict              = false
+    @unsupported         = [:alloca, :cfunc, :cref, :ifunc, :last, :memo,
+                            :newline, :opt_n, :method]
+    @unsupported_checked = false
+    @debug               = {}
+    @expected            = Sexp
+    @require_empty       = true
+    @exceptions          = {}
+
+    # we do this on an instance basis so we can subclass it for
+    # different processors.
+    @processors = {}
+    @rewriters  = {}
+    @context    = []
+
+    public_methods.each do |name|
+      case name
+      when /^process_(.*)/ then
+        @processors[$1.intern] = name.intern
+      when /^rewrite_(.*)/ then
+        @rewriters[$1.intern]  = name.intern
+      end
+    end
+  end
+
+  def assert_empty(meth, exp, exp_orig)
+    unless exp.empty? then
+      msg = "exp not empty after #{self.class}.#{meth} on #{exp.inspect}"
+      msg += " from #{exp_orig.inspect}" if $DEBUG
+      raise NotEmptyError, msg
+    end
+  end
+
+  def rewrite(exp)
+    type = exp.first
+
+    self.context.unshift type
+
+    exp.map! { |sub| Array === sub ? rewrite(sub) : sub }
+
+    self.context.shift
+
+    begin
+      meth = @rewriters[type]
+      exp  = self.send(meth, exp) if meth
+      break unless Sexp === exp
+      old_type, type = type, exp.first
+    end until old_type == type
+
+    exp
+  end
+
+  ##
+  # Default Sexp processor.  Invokes process_<type> methods matching
+  # the Sexp type given.  Performs additional checks as specified by
+  # the initializer.
+
+  def process(exp)
+    return nil if exp.nil?
+    exp = self.rewrite(exp) if self.context.empty?
+
+    unless @unsupported_checked then
+      m = public_methods.grep(/^process_/) { |o| o.to_s.sub(/^process_/, '').intern }
+      supported = m - (m - @unsupported)
+
+      raise UnsupportedNodeError, "#{supported.inspect} shouldn't be in @unsupported" unless supported.empty?
+
+      @unsupported_checked = true
+    end
+
+    result = self.expected.new
+
+    type = exp.first
+    raise "type should be a Symbol, not: #{exp.first.inspect}" unless
+      Symbol === type
+
+    self.context.unshift type
+
+    if @debug.has_key? type then
+      str = exp.inspect
+      puts "// DEBUG: #{str}" if str =~ @debug[type]
+    end
+
+    exp_orig = nil
+    exp_orig = exp.deep_clone if $DEBUG or
+      @debug.has_key? type or @exceptions.has_key?(type)
+
+    raise UnsupportedNodeError, "'#{type}' is not a supported node type" if
+      @unsupported.include? type
+
+    if @debug.has_key? type then
+      str = exp.inspect
+      puts "// DEBUG (rewritten): #{str}" if str =~ @debug[type]
+    end
+
+    # now do a pass with the real processor (or generic)
+    meth = @processors[type] || @default_method
+    if meth then
+
+      if @warn_on_default and meth == @default_method then
+        warn "WARNING: Using default method #{meth} for #{type}"
+      end
+
+      exp.shift if @auto_shift_type and meth != @default_method
+
+      result = error_handler(type, exp_orig) do
+        self.send(meth, exp)
+      end
+
+      raise SexpTypeError, "Result must be a #{@expected}, was #{result.class}:#{result.inspect}" unless @expected === result
+
+      self.assert_empty(meth, exp, exp_orig) if @require_empty
+    else
+      unless @strict then
+        until exp.empty? do
+          sub_exp = exp.shift
+          sub_result = nil
+          if Array === sub_exp then
+            sub_result = error_handler(type, exp_orig) do
+              process(sub_exp)
+            end
+            raise "Result is a bad type" unless Array === sub_exp
+            raise "Result does not have a type in front: #{sub_exp.inspect}" unless Symbol === sub_exp.first unless sub_exp.empty?
+          else
+            sub_result = sub_exp
+          end
+          result << sub_result
+        end
+
+        # NOTE: this is costly, but we are in the generic processor
+        # so we shouldn't hit it too much with RubyToC stuff at least.
+        #if Sexp === exp and not exp.sexp_type.nil? then
+        begin
+          result.sexp_type = exp.sexp_type
+        rescue Exception
+          # nothing to do, on purpose
+        end
+      else
+        msg = "Bug! Unknown node-type #{type.inspect} to #{self.class}"
+        msg += " in #{exp_orig.inspect} from #{caller.inspect}" if $DEBUG
+        raise UnknownNodeError, msg
+      end
+    end
+
+    self.context.shift
+    result
+  end
+
+  ##
+  # Raises unless the Sexp type for +list+ matches +typ+
+
+  def assert_type(list, typ)
+    raise SexpTypeError, "Expected type #{typ.inspect} in #{list.inspect}" if
+      not Array === list or list.first != typ
+  end
+
+  def error_handler(type, exp=nil) # :nodoc:
+    begin
+      return yield
+    rescue StandardError => err
+      if @exceptions.has_key? type then
+        return @exceptions[type].call(self, exp, err)
+      else
+        warn "#{err.class} Exception thrown while processing #{type} for sexp #{exp.inspect} #{caller.inspect}" if $DEBUG
+        raise
+      end
+    end
+  end
+
+  ##
+  # Registers an error handler for +node+
+
+  def on_error_in(node_type, &block)
+    @exceptions[node_type] = block
+  end
+
+  ##
+  # A fairly generic processor for a dummy node. Dummy nodes are used
+  # when your processor is doing a complicated rewrite that replaces
+  # the current sexp with multiple sexps.
+  #
+  # Bogus Example:
+  #
+  #   def process_something(exp)
+  #     return s(:dummy, process(exp), s(:extra, 42))
+  #   end
+
+  def process_dummy(exp)
+    result = @expected.new(:dummy) rescue @expected.new
+
+    until exp.empty? do
+      result << self.process(exp.shift)
+    end
+
+    result
+  end
+
+  ##
+  # Add a scope level to the current env. Eg:
+  #
+  #   def process_defn exp
+  #     name = exp.shift
+  #     args = process(exp.shift)
+  #     scope do
+  #       body = process(exp.shift)
+  #       # ...
+  #     end
+  #   end
+  #
+  #   env[:x] = 42
+  #   scope do
+  #     env[:x]       # => 42
+  #     env[:y] = 24
+  #   end
+  #   env[:y]         # => nil
+
+  def scope &block
+    env.scope(&block)
+  end
+
+  ##
+  # I really hate this here, but I hate subdirs in my lib dir more...
+  # I guess it is kinda like shaving... I'll split this out when it
+  # itches too much...
+
+  class Environment
+    def initialize
+      @env = []
+      @env.unshift({})
+    end
+
+    def all
+      @env.reverse.inject { |env, scope| env.merge scope }
+    end
+
+    def depth
+      @env.length
+    end
+
+    # TODO: depth_of
+
+    def [] name
+      hash = @env.find { |closure| closure.has_key? name }
+      hash[name] if hash
+    end
+
+    def []= name, val
+      hash = @env.find { |closure| closure.has_key? name } || @env.first
+      hash[name] = val
+    end
+
+    def scope
+      @env.unshift({})
+      begin
+        yield
+      ensure
+        @env.shift
+        raise "You went too far unextending env" if @env.empty?
+      end
+    end
+  end
+end
+
+class Object
+
+  ##
+  # deep_clone is the usual Marshalling hack to make a deep copy.
+  # It is rather slow, so use it sparingly. Helps with debugging
+  # SexpProcessors since you usually shift off sexps.
+
+  def deep_clone
+    Marshal.load(Marshal.dump(self))
+  end
+end
+
+##
+# SexpProcessor base exception class.
+
+class SexpProcessorError < StandardError; end
+
+##
+# Raised by SexpProcessor if it sees a node type listed in its
+# unsupported list.
+
+class UnsupportedNodeError < SexpProcessorError; end
+
+##
+# Raised by SexpProcessor if it is in strict mode and sees a node for
+# which there is no processor available.
+
+class UnknownNodeError < SexpProcessorError; end
+
+##
+# Raised by SexpProcessor if a processor did not process every node in
+# a sexp and @require_empty is true.
+
+class NotEmptyError < SexpProcessorError; end
+
+##
+# Raised if assert_type encounters an unexpected sexp type.
+
+class SexpTypeError < SexpProcessorError; end