heist 0.1.0 → 0.2.0

data/lib/parser/scheme.tt

@@ -5,15 +5,25 @@ module Heist
     end
     
     rule cell
-      ignore quote? ignore (list / atom) ignore <Cell>
+      ignore quote cell <QuotedCell> /
+      ignore (list / atom) ignore <Cell>
     end
     
     rule quote
       "'" / "`" / ",@" / ","
     end
     
+    rule dot
+      "."
+    end
+    
     rule list
-      ("(" cell* ")" / "[" cell* "]") <List>
+      ("(" cells ")" / "[" cells "]") <List>
+    end
+    
+    rule cells
+      cell+ (dot space cell) /
+      cell* ignore
     end
     
     rule atom
@@ -33,7 +43,7 @@ module Heist
     end
     
     rule complex
-      real "+" real "i" <Complex>
+      real:(real / integer) "+" imaginary:(real / integer) "i" <Complex>
     end
     
     rule real
@@ -53,15 +63,23 @@ module Heist
     end
     
     rule identifier
-      (!delimiter .)+ <Identifier>
+      ((!delimiter .) (!delimiter .)+ / (!reserved .)) <Identifier>
     end
     
     rule digit
       [0-9]
     end
     
+    rule reserved
+      dot / delimiter
+    end
+    
     rule delimiter
-      [\(\)\[\]] / space
+      quote / paren / space
+    end
+    
+    rule paren
+      [\(\)\[\]]
     end
     
     rule space

data/lib/repl.rb

@@ -10,7 +10,7 @@ module Heist
     end
     
     def run
-      Heist.info(@runtime)
+      puts @runtime.info
       
       Readline.completion_append_character = nil
       Readline.completion_proc = lambda do |prefix|
@@ -27,22 +27,29 @@ module Heist
       end
       
       loop do
-        input  = Readline.readline(prompt)
-        exit if input.nil?
-        
-        push(input)
-        tree = Heist.parse(@buffer * ' ')
-        next if tree.nil?
-        
-        reset!
         begin
+          input  = Readline.readline(prompt)
+          exit if input.nil?
+          
+          push(input)
+          tree = Heist.parse(@buffer * "\n")
+          next if tree.nil?
+          
+          reset!
           result = @scope.eval(tree)
           puts "; => #{ result }\n\n" unless result.nil?
+          
         rescue Exception => ex
           return if SystemExit === ex
-          puts "; [error] #{ ex.message }\n\n"
-          puts "; backtrace: " + ex.backtrace.join("\n;            ") +
-            "\n\n" unless Heist::RuntimeError === ex
+          if Interrupt === ex
+            reset! and puts "\n\n"
+          else
+            puts "; [error] #{ ex.message }\n\n"
+          end
+          
+        # debugging
+        #  puts "; backtrace: " + ex.backtrace.join("\n;            ") +
+        #    "\n\n" unless Heist::RuntimeError === ex
         end
       end
     end
@@ -73,10 +80,6 @@ module Heist
       while code == code.gsub!(/\([^\(\)\[\]]*\)|\[[^\(\)\[\]]*\]/, ''); end
       calls = code.scan(/[\(\[](?:[^\(\)\[\]\s]*\s*)/).flatten
       
-      calls.pop while calls.size > 1 and
-                      SPECIAL.include?(calls.last[1..-1].strip) and
-                      SPECIAL.include?(calls[-2][1..-1].strip)
-      
       offsets  = calls.inject([]) do |list, call|
         index  = list.empty? ? 0 : list.last.last - @indent
         index  = @indent + (line.index(call, index) || 0)

data/lib/runtime/binding.rb

@@ -1,26 +1,48 @@
 module Heist
   class Runtime
     
+    # A +Binding+ is a simple data object that couples an +Expression+ to a
+    # +Scope+ so the expression can be evaluated at some later time. Evaluation
+    # is typically memoized, though this is not always the case. <tt>Binding</tt>s
+    # are analogous to the notion of 'promises' in Scheme, though in Heist
+    # promises are implemented (as suggested in R5RS) using a <tt>(delay)</tt>
+    # macro that produces memoized closures. <tt>Binding</tt>s are used to
+    # implement lazy evaluation, and to maintain lexical scope in hygienic
+    # macro expansions by tying variables in the expansion to the lexical
+    # scope of the macro.
+    #
+    # The +Binding+ class mixes in the +Expression+ module purely as a flag
+    # to indicate to the evaluator that it can be evaluated.
+    #
     class Binding
       include Expression
       
       attr_reader :expression, :scope
       
+      # To initialize a +Binding+ supply an +Expression+ and a +Scope+ in
+      # which to evaluate it. An optional third argument states whether the
+      # evaluation should be memoized.
       def initialize(expression, scope, memoized = true)
         @expression = expression
         @scope      = scope
         @memoized   = !!memoized
       end
       
-      def extract
+      # Evaluates the +Binding+ and returns the result. The +Expression+
+      # is only ever evaluated once if the +Binding+ has been memoized.
+      def force!
         return @value if defined?(@value) and @memoized
         @value = Heist.evaluate(@expression, @scope)
       end
       
+      # This method is provided as a convenience so that a +Binding+ may
+      # be treated like any other expression during evaluation. All it
+      # does is return the result of calling <tt>force!</tt>.
       def eval(scope)
-        extract
+        force!
       end
       
+      # Returns a string representation of the binding's +Expression+.
       def to_s
         @expression.to_s
       end

data/lib/runtime/callable/continuation.rb

@@ -1,19 +1,40 @@
 module Heist
   class Runtime
     
+    # The +Continuation+ class encapsulates the first-class continuations as
+    # generated by the Scheme <tt>(call/cc)</tt> procedure. It inherits from
+    # +Function+ so as to be recognised as a callable object by the +Stack+
+    # execution model. Each +Continuation+ contains a snapshot of the state
+    # of the +Stack+ at the moment it was created, and this state is restored,
+    # overriding whatever state is then in effect, when the +Continuation+ is
+    # called.
     class Continuation < Function
+
+      # A +Continuation+ is initialized with a +Stack+ object, which it saves a
+      # snapshot of. The topmost frame on the stack will typically be a call
+      # to <tt>(call/cc)</tt> or some other +Continuation+-generating function,
+      # and is thus discarded as this call site marks the hole that should be
+      # filled when the +Continuation+ is called and the saved stack is resumed.
       def initialize(stack)
         @stack  = stack.copy(false)
         @target = stack.last.target
       end
       
+      # Calls the +Continuation+ with the current +Scope+ and a +Cons+ list of
+      # parameters to the continuation. Returns a copy of the saved +Stack+
+      # with the innermost hole filled with the value of the parameter to the
+      # continuation call. The currently running +Stack+ will see this return
+      # value as an instruction to throw out the current state of the stack and
+      # replace it with the stack returned by this method before continuing
+      # execution.
       def call(scope, cells)
-        filler = Heist.evaluate(cells.first, scope)
         stack = @stack.copy
-        stack.fill!(@target, filler)
+        stack.fill!(@target, cells.car)
         stack
       end
       
+      # Returns a string representation of the +Continuation+ for console
+      # output.
       def to_s
         "#<continuation>"
       end

data/lib/runtime/callable/function.rb

@@ -1,53 +1,154 @@
 module Heist
   class Runtime
     
+    # There are several types of callable object in Heist: +Function+, +Syntax+,
+    # +Macro+ and +Continuation+. For convenience of type detection, +Function+
+    # is treated as a the base class for all these, though they all typically
+    # override <tt>Function</tt>'s core methods. The intention is to treat all
+    # these types as uniformly as possible, though this does introduce some
+    # API problems which still need resolving. In any case, the current state
+    # of things:
+    #
+    #                      ============
+    #                      | Function |
+    #                      ============
+    #                           |
+    #               ---------------------------
+    #               |                         |
+    #          ==========              ================
+    #          | Syntax |              | Continuation |
+    #          ==========              ================
+    #               |
+    #           =========
+    #           | Macro |
+    #           =========
+    #
+    # Heist is designed to be reasonably modular in that none of Scheme's
+    # special forms or built-in functions are mentioned in the parser; instead
+    # they are all implemented as first-class functions and stored in the
+    # global scope when you initialize a Heist +Runtime+. The Heist runtime
+    # does implement a fair amount of Scheme-specific functionality but you're
+    # not forced to use any of it: at its core it is designed as a general
+    # processor for s-expressions and could be used as a backend for any
+    # Lisp-style language. The only hard-and-fast rule is that, to evaluate
+    # an expression, you evaluate its first element, and call the resulting
+    # function with the remaining elements.
+    #
+    # The Heist callable types -- +Function+, +Syntax+, +Macro+ and
+    # +Continuation+ -- differ primarily in terms of what they do with their
+    # arguments when called. All are passed the current +Scope+ and a +Cons+
+    # list containing the rest of the current expression when they are
+    # called; it is up to the type of function being called what to do with
+    # the expression.
+    #
+    # The +Function+ class is used for representing the basic idea of a Lisp
+    # procedure; a reusable block of code that can accept zero, one or more
+    # arguments, return output values and cause side effects on the state of
+    # the running program. Heist functions can be implemented using Ruby
+    # blocks or Scheme lists.
+    #
+    # For information on other callable types, see +Syntax+, +Macro+ and
+    # +Continuation+.
+    #
     class Function
       attr_reader :body, :name
       
-      def initialize(scope, formals = [], body = nil, &block)
-        @scope   = scope
-        @body    = body ? List.from(body) : block
-        @formals = formals.map { |id| id.to_s }
-        @lazy    = scope.runtime.lazy?
-        @eager   = !scope.runtime.stackless?
+      # A +Function+ can be initialized in one of two ways. Both types
+      # require a +Scope+ as the first parameter: this is the lexical
+      # scope in which the +Function+ is defined. The other parameters
+      # should consist either of a Ruby block that accepts the procedure's
+      # arguments and returns its result:
+      #
+      #   env = Scope.new
+      #   env['plus'] = Function.new(env) { |a,b| a + b }
+      #
+      # Or, the parameters should be a list of formal argument names for
+      # the procedure, and a list containing zero or more Scheme expressions
+      # that make up the procedure's body. If the list of formals is an
+      # improper list, the tail name will be assigned a list of any
+      # parameters remaining after all the preceeding formals have been
+      # assigned values. If the list of formals is not a list but a single
+      # identifier, that identifier will be assigned a list of all the
+      # parameters when the procedure is called.
+      #
+      def initialize(scope, formals = nil, body = nil, &block)
+        @formals, @rest = [], formals
+        if Cons === formals
+          tail     = formals.tail.cdr
+          @formals = formals.map { |id| id.to_s }
+          @rest    = (Identifier === tail) ? tail : nil
+        end
+        @scope = scope
+        @body  = body || block
+        @lazy  = scope.runtime.lazy?
+        @eager = !scope.runtime.stackless?
       end
       
+      # Assigns a name to the +Function+, used primarily for console output.
+      # A +Function+ keeps the first name it is bound to. This is really only
+      # of use to REPL users; functions are first-class data and do not
+      # need names in order to operate correctly.
       def name=(name)
         @name ||= name.to_s
       end
       
+      # Calls the +Function+ with a calling +Scope+ (in which to evaluate the
+      # parameters), and a +Cons+ list containing the parameter expressions.
+      # In lazy mode, the parameter expressions are wrapped in +Binding+
+      # objects for later execution. If using the continuation-supporting
+      # +Stack+ interpreter (as indicated by the <tt>@eager</tt> flag), the
+      # parameters will already have been evaluated and should not be
+      # re-evaluated lest we try to interpret data lists as code. Returns the
+      # result of calling +apply+ with the evaluated parameters.
       def call(scope, cells)
-        params, closure = [], Scope.new(@scope)
-        cells.each_with_index do |arg, i|
-          params[i] = closure[@formals[i]] = lazy? ?
-              Binding.new(arg, scope) :
-              (@eager ? arg : Heist.evaluate(arg, scope))
+        params = cells.map do |arg|
+          lazy? ? Binding.new(arg, scope) :
+                  (@eager ? arg : Heist.evaluate(arg, scope))
         end
+        apply(params)
+      end
+      
+      # Returns the result of applying the +Function+ to the given collection
+      # of +params+. If the procedure is primitive, this will return a value
+      # immediately. If it's a Scheme function, this simply binds the +params+
+      # to the procedure's formal arguments and returns a +Body+ with the
+      # procedure's body expressions and the newly allocated +Scope+. We use
+      # a trampoline to call <tt>Function</tt>s iteratively to allow tail
+      # call optimisation. (See +Stackless+, +Stack+, +Frame+ and +Body+ for
+      # more information.)
+      def apply(params)
         return @body.call(*params) if primitive?
+        closure = Scope.new(@scope)
+        idx = 0
+        @formals.each do |name|
+          closure[name] = params[idx]
+          idx += 1
+        end
+        closure[@rest] = Cons.construct(params[idx..-1]) if @rest
         Body.new(@body, closure)
       end
       
+      # Returns +true+ iff the +Function+ is a primitive, that is to say it
+      # is implemented in Ruby rather than Scheme.
       def primitive?
         Proc === @body
       end
       
+      # Returns +true+ iff the +Function+ is lazy, i.e. it does not
+      # evaluate its arguments unless it needs their values. Only non-
+      # primitive functions can be lazy since we have no way of telling
+      # when Ruby needs to access a value. Besides, primitive functions
+      # will typically use all their parameters.
       def lazy?
         @lazy and not primitive?
       end
       
+      # Returns a string placeholder for the +Function+, containing its
+      # name if it has one.
       def to_s
         "#<procedure:#{ @name }>"
       end
-    end
-    
-    class Syntax < Function
-      def call(scope, cells)
-        @body.call(scope, *cells)
-      end
-      
-      def to_s
-        "#<syntax:#{ @name }>"
-      end
+      alias :inspect :to_s
     end
     
   end

data/lib/runtime/callable/macro.rb

@@ -1,168 +1,214 @@
-# Quotations taken from the R5RS spec
-# http://www.schemers.org/Documents/Standards/R5RS/HTML/r5rs-Z-H-7.html
 module Heist
   class Runtime
     
+    # The +Macro+ class is a type of +Function+ (it inherits from +Syntax+ to
+    # let the evaluator know it consumes syntax rather than values) that is
+    # used to represent non-primitive syntax as defined by the Scheme
+    # <tt>(syntax-rules)</tt> macro system. Calling a +Macro+ involves passing
+    # it an input +Expression+, which it will either convert into another
+    # +Expression+ according to user-defined rules, or it will raise an
+    # exception if the input is syntactically invalid. On success, a +Macro+
+    # will return an +Expansion+, which contains an +Expression+ that can be
+    # directly evaluated or expanded further.
+    #
+    # Heist does not have distinct macro expansion and runtime phases; its
+    # macros are first-class runtime objects and all expansion takes place
+    # as macros are encountered while the program is running. Once a macro
+    # call is processed, the expansion is inlined into the source tree,
+    # replacing the macro call to avoid further unnecessary expansions.
+    #
+    # +Macro+ uses several auxiliary classes to do its work. See +Matches+,
+    # +Tree+ and +Expansion+ for more information.
+    #
     class Macro < Syntax
-      ELLIPSIS = '...'
       
-      %w[expansion splice matches].each do |klass|
+      # The ellipsis identifier, used to indicate repeated patterns
+      ELLIPSIS = Identifier.new('...')
+      
+      # Array of reserved symbols that cannot be used as pattern vars
+      RESERVED = %w[_ ...]
+      
+      %w[expansion matches tree].each do |klass|
         require RUNTIME_PATH + 'callable/macro/' + klass
       end
       
-      def initialize(scope, *args)
-        super
-        @hygienic = scope.runtime.hygienic?
+      # Takes an s-expression and returns an array of the pattern variables
+      # it contains. The optional second argument should be an array, and
+      # specifies a list of names to exclude, for example to exclude the formal
+      # keywords of a macro transformer. The final argument is for internal use
+      # only; we call this method recursively but pass down a single array to
+      # push results onto to avoid lots of array joins.
+      def self.pattern_vars(pattern, excluded = [], results = [])
+        return results if Cons::NULL == pattern
+        case pattern
+          when Identifier then
+            name = pattern.to_s
+            return if excluded.include?(name) or RESERVED.include?(name)
+            results << name unless results.include?(name)
+          when Cons then
+            tail = pattern.each { |cell| pattern_vars(cell, excluded, results) }
+            pattern_vars(tail.cdr, excluded, results)
+        end
+        results
       end
       
+      # Calls the +Macro+ with the current +Scope+ and the +Cons+ list of the
+      # rest of the expression, i.e. the syntax tree for the macro to operate
+      # on. Returns an +Expansion+ if the given syntax is found to be valid,
+      # otherwise raises an exception.
       def call(scope, cells)
-        @calling_scope = scope
-        rule, matches = *rule_for(cells)
-        
-        return Expansion.new(expand_template(rule.last, matches)) if rule
-        
-        # TODO include macro name in error message,
-        # and be more specific about which pattern failed
-        input = cells.map { |c| c.to_s } * ' '
+        rule, matches = *rule_for(cells, scope)
+        return Expansion.new(@scope, scope, rule.cdr.car, matches) if rule
         raise SyntaxError.new(
-          "Bad syntax: no macro expansion found for (#{@name} #{input})")
+          "Bad syntax: no macro expansion found for #{Cons.new(@name, cells)}")
       end
       
+      # Returns a string placeholder for the +Macro+, containing its
+      # name if it has one.
       def to_s
         "#<macro:#{ @name }>"
       end
       
-    private
-      
-      def rule_for(cells)
+      # Takes a +Cons+ expression and a +Scope+ (required for determining
+      # the binding of macro keywords), and returns a tuple containing an
+      # expansion template and a set of pattern match data for the first
+      # rule in the macro that matches the input. If no such rule is found,
+      # returns +nil+.
+      def rule_for(cells, scope)
         @body.each do |rule|
-          matches = rule_matches(rule.first.rest, cells)
+          matches = rule_matches(scope, rule.car.cdr, cells)
           return [rule, matches] if matches
         end
-        nil
+        return nil
       end
       
+      # Takes a +Scope+ (the scope from which the macro is being called,
+      # required for determining keyword bindings), an +Expression+
+      # representing a macro pattern, and an input +Expression+ from
+      # the expression the macro was called with. If the pattern matches
+      # the input, a +Matches+ object is returned, otherwise we return
+      # +nil+. The +matches+ and +depth+ arguments are for internal use
+      # only and are passed down as the match algorithm recurses over
+      # the pattern and input expressions.
+      #
+      # +matches+ is a +Matches+ instance that stores data about which
+      # input expressions correspond to which pattern variables, and how
+      # often they repeat. +depth+ indicates the repetition depth, that
+      # is how many ellipses appear following the current pattern.
+      #
+      # From the R5RS spec
+      # http://www.schemers.org/Documents/Standards/R5RS/HTML/r5rs-Z-H-7.html
+      #
       # More formally, an input form F matches a pattern P if and only if:
-      # 
-      #     * P is a non-literal identifier; or
-      #     * P is a literal identifier and F is an identifier with the
-      #       same binding; or
-      #     * P is a list (P1 ... Pn) and F is a list of n forms that match
-      #       P1 through Pn, respectively; or
-      #     * P is an improper list (P1 P2 ... Pn . Pn+1) and F is a list
-      #       or improper list of n or more forms that match P1 through Pn,
-      #       respectively, and whose nth 'cdr' matches Pn+1; or
-      #     * P is of the form (P1 ... Pn Pn+1 <ellipsis>) where <ellipsis>
-      #       is the identifier '...' and F is a proper list of at least n forms,
-      #       the first n of which match P1 through Pn, respectively, and
-      #       each remaining element of F matches Pn+1; or
-      #     * P is a vector of the form #(P1 ... Pn) and F is a vector of n
-      #       forms that match P1 through Pn; or
-      #     * P is of the form #(P1 ... Pn Pn+1 <ellipsis>) where <ellipsis>
-      #       is the identifier '...' and F is a vector of n or more forms the
-      #       first n of which match P1 through Pn, respectively, and each
-      #       remaining element of F matches Pn+1; or
-      #     * P is a datum and F is equal to P in the sense of the 'equal?'
-      #       procedure.
-      # 
+      #
+      # * P is a non-literal identifier; or
+      # * P is a literal identifier and F is an identifier with the
+      #   same binding; or
+      # * P is a list (P1 ... Pn) and F is a list of n forms that match
+      #   P1 through Pn, respectively; or
+      # * P is an improper list (P1 P2 ... Pn . Pn+1) and F is a list
+      #   or improper list of n or more forms that match P1 through Pn,
+      #   respectively, and whose nth 'cdr' matches Pn+1; or
+      # * P is of the form (P1 ... Pn Pn+1 <ellipsis>) where <ellipsis>
+      #   is the identifier '...' and F is a proper list of at least n forms,
+      #   the first n of which match P1 through Pn, respectively, and
+      #   each remaining element of F matches Pn+1; or
+      # * P is a vector of the form #(P1 ... Pn) and F is a vector of n
+      #   forms that match P1 through Pn; or
+      # * P is of the form #(P1 ... Pn Pn+1 <ellipsis>) where <ellipsis>
+      #   is the identifier '...' and F is a vector of n or more forms the
+      #   first n of which match P1 through Pn, respectively, and each
+      #   remaining element of F matches Pn+1; or
+      # * P is a datum and F is equal to P in the sense of the 'equal?'
+      #   procedure.
+      #
       # It is an error to use a macro keyword, within the scope of its
       # binding, in an expression that does not match any of the patterns.
-      # 
-      def rule_matches(pattern, input, matches = Matches.new, depth = 0)
+      #
+      def rule_matches(scope, pattern, input, matches = nil, depth = 0)
+        matches ||= Matches.new(pattern, @formals)
         case pattern
         
-          when List then
-            return nil unless List === input
-            idx = 0
-            pattern.each_with_index do |token, i|
-              followed_by_ellipsis = (pattern[i+1].to_s == ELLIPSIS)
-              dx = followed_by_ellipsis ? 1 : 0
-              
-              matches.depth = depth + dx
-              next if token.to_s == ELLIPSIS
+          when Cons then
+            # If pattern is NULL, the input must also be NULL
+            return (Cons::NULL == input ? matches : nil) if pattern.null?
+            
+            # Fail if the pattern is a list and the input is not
+            return nil unless Cons === input
+            
+            # Iterate over the pattern, consuming input as we go
+            pattern_pair, input_pair = pattern, input
+            skip = lambda { pattern_pair = pattern_pair.cdr }
+            
+            while Cons === pattern_pair and not pattern_pair.null?
+              token = pattern_pair.car
               
-              consume = lambda { rule_matches(token, input[idx], matches, depth + dx) }
-              return nil unless value = consume[] or followed_by_ellipsis
-              next unless value
-              idx += 1
+              # Skip the current pattern token if it's an ellipsis
+              skip[] and next if token == ELLIPSIS
               
-              idx += 1 while idx < input.size and
-                             followed_by_ellipsis and
-                             consume[]
-            end
-            return nil unless idx == input.size
-        
-          when Identifier then
-            return (pattern.to_s == input.to_s) if @formals.include?(pattern.to_s)
-            matches.put(pattern, input)
-            return nil if input.nil?
-        
-          else
-            return pattern == input ? true : nil
-        end
-        matches
-      end
-      
-      # When a macro use is transcribed according to the template of the
-      # matching <syntax rule>, pattern variables that occur in the template
-      # are replaced by the subforms they match in the input. Pattern variables
-      # that occur in subpatterns followed by one or more instances of the
-      # identifier '...' are allowed only in subtemplates that are followed
-      # by as many instances of '...'. They are replaced in the output by all
-      # of the subforms they match in the input, distributed as indicated. It
-      # is an error if the output cannot be built up as specified.
-      # 
-      # Identifiers that appear in the template but are not pattern variables
-      # or the identifier '...' are inserted into the output as literal
-      # identifiers. If a literal identifier is inserted as a free identifier
-      # then it refers to the binding of that identifier within whose scope
-      # the instance of 'syntax-rules' appears. If a literal identifier is
-      # inserted as a bound identifier then it is in effect renamed to prevent
-      # inadvertent captures of free identifiers.
-      # 
-      def expand_template(template, matches, depth = 0, inspection = false)
-        case template
-        
-          when List then
-            result = List.new
-            template.each_with_index do |cell, i|
-              followed_by_ellipsis = (template[i+1].to_s == ELLIPSIS)
+              # Increment the repetition depth if the next pattern
+              # token is an ellipsis, and inform the +Matches+ object
+              # that the pattern vars in the current pattern have
+              # hit a repetition boundary. Note we do not increment
+              # +depth+ itself since this would persist for the remaining
+              # tokens in the pattern after we get past the ellipsis.
+              followed_by_ellipsis = (pattern_pair.cdr.car == ELLIPSIS rescue false)
               dx = followed_by_ellipsis ? 1 : 0
               
-              matches.inspecting(depth + 1) if followed_by_ellipsis and
-                                               not inspection
+              matches.descend!(Macro.pattern_vars(token, @formals),
+                               depth + dx) if followed_by_ellipsis
               
-              if cell.to_s == ELLIPSIS and not inspection
-                repeater = template[i-1]
-                matches.expand! { result << expand_template(repeater, matches, depth + 1) }
-                matches.depth = depth
-              else
-                inspect = inspection || (followed_by_ellipsis && depth + 1)
-                value = expand_template(cell, matches, depth + dx, inspect)
-                result << value unless inspect
+              # Set up a closure to consume input using the current
+              # pattern expression. Calls +rule_matches+ with the
+              # current scope, pattern, input, and +Matches+ object.
+              consume = lambda do
+                Cons === input_pair and not input_pair.null? and
+                rule_matches(scope, token, input_pair.car, matches, depth + dx)
               end
+              
+              # If the next pattern token is not an ellipsis, fail
+              # unless the pattern token matches the input token.
+              #
+              # If the next token is an ellipsis, consume input
+              # using the current pattern until the pattern no
+              # longer matches the current input
+              #
+              return nil unless consume[] or followed_by_ellipsis
+              input_pair = input_pair.cdr
+              input_pair = input_pair.cdr while followed_by_ellipsis and consume[]
+              
+              skip[]
             end
-            result
+            
+            # We're done iterating over the pattern, so the current
+            # pattern token will be NULL or some non-Cons object (if
+            # the pattern is an improper list). Fail unless the remaining
+            # input matches this object.
+            return nil unless rule_matches(scope, pattern_pair, input_pair, matches, depth)
         
+          # If the pattern is a formal keyword for the macro (a
+          # 'literal identifier' in the terms of the spec), return
+          # a boolean indicating whether the input is an identifier
+          # with the same binding, that is to say the two identifiers
+          # refer to the same location in memory (or both refer to
+          # no location). If it's a normal pattern variable, store
+          # the current input, whatever it is, in the +matches+.
           when Identifier then
-            return matches.get(template) if matches.defined?(template)
-            return Identifier.new(template) unless @hygienic
-            
-            @scope.defined?(template) ?
-                Binding.new(template, @scope, false) :
-                rename(template)
+            if @formals.include?(pattern.to_s)
+              return pattern == input && @scope.innermost_binding(pattern) ==
+                                         scope.innermost_binding(input)
+            else
+              matches.put(pattern, input)
+            end
         
+          # If all above type checks on the pattern fail, assume the
+          # pattern is literal data and make sure the input matches.
           else
-            template
+            return pattern == input ? matches : nil
         end
+        matches
       end
       
-      def rename(id)
-        return id unless @calling_scope.defined?(id)
-        i = 1
-        i += 1 while @calling_scope.defined?("#{id}#{i}")
-        Identifier.new("#{id}#{i}")
-      end
     end
     
   end