heist 0.1.0 → 0.2.0

data/lib/runtime/callable/macro/expansion.rb

@@ -2,10 +2,145 @@ module Heist
   class Runtime
     class Macro
       
+      # +Expansion+ is responsible for expanding syntactic forms matched by
+      # successful +Macro+ calls. Any successful +Macro+ call returns an
+      # +Expansion+ object, which is used as a signal to the evaluator that
+      # the +Expression+ contained in the expansion should be inlined into
+      # the syntax tree.
       class Expansion
         attr_reader :expression
-        def initialize(expression)
-          @expression = expression
+        
+        # An +Expansion+ is initialized using the lexical +Scope+ of the
+        # +Macro+ and the +Scope+ of the macro call site (both required
+        # for hygiene purposes), plus a +Cons+ representing the expansion
+        # template and a +Matches+ object containing the input expressions
+        # to be transcribed using the template. After initialization the
+        # expanded +Expression+ is available via the +Expansion+ object's
+        # +expression+ attribute.
+        def initialize(lexical_scope, calling_scope, template, matches)
+          @lexical_scope = lexical_scope
+          @calling_scope = calling_scope
+          @hygienic      = lexical_scope.runtime.hygienic?
+          @expression    = expand(template, matches)
+        end
+        
+        # Accepts a template object (a +Cons+, +Identifier+ or similar) and a
+        # +Matches+ instance and returns the result of expanding the +template+
+        # using the +matches+. The +depth+ and +ignoring_ellipses+ arguments are
+        # for internal state maintainance as we recursively expand the template
+        # forms. +depth+ indicates the current repetition depth, i.e. how
+        # many ellipses follow the current subtemplate, and +ignoring_ellipses+
+        # is +true+ iff we're expanding a template in which ellipses should be
+        # transcribed verbatim. This is an R6RS feature; if a template opens
+        # with an ellipsis, we transcribe the rest of the template as normal
+        # except that any ellipses in the template are inserted as ellipses
+        # into the output, without causing their preceeding forms to repeat.
+        #
+        # From the R5RS spec
+        # http://www.schemers.org/Documents/Standards/R5RS/HTML/r5rs-Z-H-7.html
+        #
+        # When a macro use is transcribed according to the template of the
+        # matching <syntax rule>, pattern variables that occur in the template
+        # are replaced by the subforms they match in the input. Pattern variables
+        # that occur in subpatterns followed by one or more instances of the
+        # identifier '...' are allowed only in subtemplates that are followed
+        # by as many instances of '...'. They are replaced in the output by all
+        # of the subforms they match in the input, distributed as indicated. It
+        # is an error if the output cannot be built up as specified.
+        #
+        # Identifiers that appear in the template but are not pattern variables
+        # or the identifier '...' are inserted into the output as literal
+        # identifiers. If a literal identifier is inserted as a free identifier
+        # then it refers to the binding of that identifier within whose scope
+        # the instance of 'syntax-rules' appears. If a literal identifier is
+        # inserted as a bound identifier then it is in effect renamed to prevent
+        # inadvertent captures of free identifiers.
+        #
+        def expand(template, matches, depth = 0, ignoring_ellipses = false)
+          case template
+          
+            when Cons then
+              # If the template is a list opening with an ellipsis, expand
+              # the rest of the list, transcribing ellipses verbatim
+              return expand(template.cdr.car,
+                            matches, depth, true) if template.car == ELLIPSIS
+              
+              result, last, repeater, template_pair = nil, nil, nil, template
+              
+              # Set up a closure to push forms onto the output. Needs to
+              # track both the head (+result+, for returning) and the tail
+              # (+last+, for appending new forms). Links each inserted form
+              # to its containing +Cons+ to enable further expansions to
+              # be inlined.
+              push = lambda do |value|
+                pair = Cons.new(value)
+                pair.hosts(value)
+                result ||= pair
+                last.cdr = pair if last
+                last = pair
+              end
+              
+              # Iterate over the template, inserting matches as we go
+              while not template_pair.null?
+                cell = template_pair.car
+                
+                # Increment the repetition depth if the current subtemplate
+                # is followed by an ellipsis and we are not treating ellipses
+                # as literals
+                followed_by_ellipsis = (template_pair.cdr.car == ELLIPSIS) && !ignoring_ellipses
+                dx = followed_by_ellipsis ? 1 : 0
+                
+                repeater = cell if followed_by_ellipsis
+                
+                # Once we reach an ellipsis, expand the preceeding form
+                # the correct number of times depending on the +matches+
+                if cell == ELLIPSIS and not ignoring_ellipses
+                  matches.expand!(repeater, depth + 1) do
+                    push[expand(repeater, matches, depth + 1)]
+                  end
+                
+                # If the current subtemplate is not an ellipsis and is
+                # not followed by an ellipsis, expand it and push the
+                # result onto the output
+                else
+                  push[expand(cell, matches, depth + dx,
+                              ignoring_ellipses)] unless followed_by_ellipsis
+                end
+                
+                template_pair = template_pair.cdr
+              end
+              result
+          
+            when Identifier then
+              # If the template is a pattern variable, return the current
+              # match for that variable. See +Matches+ to see how repeated
+              # patterns are handled.
+              return matches.get(template) if matches.has?(template)
+              
+              # Otherwise, if using unhygienic macros, return the template
+              # verbatim as a new symbol.
+              return Identifier.new(template) unless @hygienic
+              
+              # If using hygienic macros: bind the identifier to the macro's
+              # lexical scope if it is defined there, otherwise rename it
+              # as appropriate to avoid clashes with variables in the
+              # calling scope.
+              @lexical_scope.defined?(template) ?
+                  Binding.new(template, @lexical_scope, false) :
+                  rename(template)
+          
+            else
+              template
+          end
+        end
+        
+        # Returns a new +Identifier+ that does clash with any of the names
+        # visible in the <tt>Expansion</tt>'s calling scope.
+        def rename(id)
+          return id unless @calling_scope.defined?(id)
+          i = 1
+          i += 1 while @calling_scope.defined?("#{id}#{i}")
+          Identifier.new("#{id}#{i}")
         end
       end
       

data/lib/runtime/callable/macro/matches.rb

@@ -2,71 +2,155 @@ module Heist
   class Runtime
     class Macro
       
+      # +Matches+ instances, with help from the +Tree+ class, are data structures
+      # that represent the way in which syntactic expressions match patterns found
+      # in +Macro+ rules. They provide an API for storing and retrieving such data,
+      # with the aim of removing some clutter from the macro parsing and expansion
+      # routines.
+      #
+      # At a pure Ruby level, a +Matches+ is a wrapper around a hash that maps
+      # pattern variables to +Tree+ objects, which themselves are wrappers around
+      # nested arrays that represent how repeated pattern matches are grouped
+      # together. For example, given the pattern
+      #
+      #   (do ([variable init step ...] ...)
+      #       (test expression ...)
+      #       command ...)
+      #
+      # and the expression
+      #
+      #   (do ([x 6 (- x 1)]
+      #        [acc 1])
+      #       ((zero? x) acc)
+      #     (display x) (newline)
+      #     (set! acc (* acc x)))
+      #
+      # the +Matches+ object would contain the following:
+      #
+      #   @data = {
+      #       "variable"   => [ x,
+      #                         acc
+      #                       ],
+      #       
+      #       "init"       => [ 6,
+      #                         1
+      #                       ],
+      #       
+      #       "step"       => [ [ (- x 1)
+      #                         ],
+      #                         []
+      #                       ],
+      #       
+      #       "test"       => (zero? x),
+      #       
+      #       "expression" => [ acc
+      #                       ],
+      #       
+      #       "command"    => [ (display x),
+      #                         (newline),
+      #                         (set! acc (* acc x))
+      #                       ]
+      #   }
+      #
+      # Breaking this down, we see +test+ is not followed by an ellipsis in the
+      # pattern, and can thus only consume one item from the input expression.
+      # So, its match data is a single +Expression+. +variable+, +init+, +command+
+      # and +expression+ are all followed by a single ellipsis (+variable+ and
+      # +init+ appear in a list that is followed by an ellipsis), so can consume
+      # several values each; their match data are arrays of expressions.
+      #
+      # +step+, on the other hand, is followed by two ellipses: it itself is
+      # followed by an ellipsis, and <tt>step ...</tt> appears inside a list that
+      # is also followed by an ellipsis. If a pattern is followed by more than
+      # one ellipsis, the match data it generates is a tree of nested arrays
+      # that describe how the expressions are grouped. Here, we see that the
+      # expression <tt>[variable init step ...]</tt> appears twice in the input,
+      # so +step+'s root match element is an array of two elements. But, +step+
+      # does not match any data in the second appearance (<tt>[acc 1]</tt>), so
+      # the second element of this array is empty. The first element is an array
+      # containing the single match from the first appearance (<tt>(- x 1)</tt>
+      # in the expression <tt>[x 6 (- x 1)]</tt>).
+      #
+      # +Matches+ tries to hide many of these details so the macro routines can
+      # read and write to this data structure in the simplest possible terms.
+      #
       class Matches
-        def initialize
-          @data  = {}
-          @depth = 0
-          @names = []
+        
+        # A +Matches+ is initialized using a +Cons+ representing a macro pattern,
+        # and an array of formal keywords supplied by the +Macro+. Keywords do
+        # not need to store matches so they are ignored here.
+        def initialize(pattern, formals)
+          @data = {}
+          names = Macro.pattern_vars(pattern, formals)
+          names.each { |name| @data[name] = Tree.new(name) }
         end
         
-        def depth=(depth)
-          mark!(depth) if depth < @depth
-          @names[depth] = [] if depth >= @depth
-          @depth = depth
+        # Tells the +Matches+ object that the given pattern variables (the array
+        # +names+) have encountered a trailing ellipsis at the given repetition
+        # depth. This allows +Matches+ to group repeated patterns correctly.
+        def descend!(names, depth)
+          @data.each do |name, set|
+            set.descend!(depth) if names.include?(name)
+          end
         end
         
-        def put(name, expression)
+        # Writes an expression to the +Matches+ object under the variable +name+.
+        # The receiver deals with storing it in the correct repetition group.
+        def put(name, value)
           name = name.to_s
-          @names[@depth] << name
-          @data[name] ||= Splice.new(name, @depth)
-          @data[name] << expression unless expression.nil?
+          @data[name] << value if has?(name)
         end
         
-        def inspecting(depth)
-          @inspecting = true
-          self.depth = depth
+        # Returns +true+ iff the receiver has a pattern variable named +name+.
+        def has?(name)
+          @data.has_key?(name.to_s)
         end
         
+        # Retrieves an expression from the +Matches+ under the given +name+. The
+        # receiver deals with pulling the expression from the right point in the
+        # tree; see the <tt>expand!</tt>, <tt>iterate!</tt> and <tt>Tree#read</tt>
+        # and <tt>Tree#shift!</tt> methods.
         def get(name)
-          @inspecting ? @names[@depth] << name.to_s :
-                        @data[name.to_s].read
-        end
-        
-        def defined?(name)
-          @data.has_key?(name.to_s)
+          @data[name.to_s].read
         end
         
-        def expand!
-          @inspecting = false
-          size.times { yield and iterate! }
+        # Takes a +template+ +Expression+, a repetition +depth+ and a block, and
+        # calls the block +n+ times, where +n+ is the number of matches for the
+        # pattern variables in the template at the given depth and the current
+        # iteration point in the tree. After each block call, the +Matches+ object
+        # moves the pointer for all the applicable pattern variables along one
+        # place at the given depth -- see <tt>iterate!</tt> and <tt>Tree#shift!</tt>.
+        def expand!(template, depth)
+          names = Macro.pattern_vars(template)
+          size(names, depth).times { yield() and iterate!(names, depth) }
         end
         
       private
         
-        def mark!(depth)
-          d = @depth
-          while @names[d]
-            @names[d].uniq.each { |name| @data[name].mark!(depth) }
-            d += 1
+        # Returns the number of matched expressions available for the given set
+        # of pattern variables at the given depth, at the current iteration point.
+        # An exception is raised if the names do not all yield the same number
+        # of matches; this indicates a piece of mismatched syntax that cannot be
+        # expanded correctly.
+        def size(names, depth)
+          sizes = []
+          @data.each do |name, tree|
+            sizes << tree.size(depth) if names.include?(name)
           end
-        end
-        
-        def size
-          names   = @names[@depth].uniq
-          splices = @data.select { |k,v| names.include?(k.to_s) }
-          sizes   = splices.map { |pair| pair.last.size(@depth) }.uniq
           
-          return 0 if sizes.empty?
+          sizes.uniq!
           return sizes.first if sizes.size == 1
           
-          expressions = splices.map { |pair| '"' + pair.last.to_s(@depth) + '"' } * ', '
           raise MacroTemplateMismatch.new(
-            "Macro could not be expanded: expressions #{expressions} are of different sizes")
+            "Macro could not be expanded: mismatched repetition patterns")
         end
         
-        def iterate!
-          @data.each do |name, splice|
-            splice.shift!(@depth) if @names[@depth].include?(name)
+        # Shifts the tree pointer (see <tt>Tree#shift!</tt> for all the given
+        # +names+ along one place at the given +depth+. This is used while
+        # expanding repeated patterns using <tt>expand!</tt>.
+        def iterate!(names, depth)
+          @data.each do |name, tree|
+            tree.shift!(depth) if names.include?(name)
           end
         end
       end

data/lib/runtime/callable/macro/tree.rb

@@ -0,0 +1,141 @@
+module Heist
+  class Runtime
+    class Macro
+      
+      # <tt>Tree</tt>s are used by instances of +Matches+ to store expressions
+      # matched by macro patterns. Patterns may contain patterns that repeat
+      # (indicated by following the pattern with an ellipsis), and these repetitions
+      # may be nested. +Tree+ instances store expressions in a set of nested arrays
+      # that match the repetition structure of the macro pattern being matched.
+      # See +Matches+ for a fuller explanation.
+      #
+      # Every +Tree+ contains an array called <tt>@data</tt> that stores the
+      # expressions matched by a pattern variable, a variable <tt>@depth</tt> that
+      # stores the maximum repetition depth of the tree (a non-repeated pattern
+      # has depth zero), and an array <tt>@indexes</tt> which is used to maintain
+      # a list of array indexes that point to the current read position in the
+      # tree while a macro is being expanded. For example, taking the pattern from
+      # the +Matches+ example:
+      #
+      #   (do ([variable init step ...] ...)
+      #       (test expression ...)
+      #       command ...)
+      #
+      # Say we had the following expression (not entirely valid (do) syntax, but
+      # compatible with the above pattern):
+      #
+      #   (do ([x 6 (- x 1) (- acc 1)]
+      #        [y 5]
+      #        [acc 1 (* x acc)])
+      #       ((zero? x) acc)
+      #     (display x) (newline)
+      #     (set! acc (* acc x)))
+      #
+      # The resulting +Matches+ object would contain the following data for the
+      # variable +step+:
+      #
+      #   "step" => [ [ (- x 1),
+      #                 (- acc 1)
+      #               ],
+      #               
+      #               [],
+      #               
+      #               [ (* x acc)
+      #               ]
+      #             ]
+      #
+      # That is, the outermost repetition <tt>[variable init step ...]</tt>
+      # occurs three times; the first appearance includes two matches for
+      # <tt>step ...</tt>, the second no matches and the third one match. With
+      # this data, an <tt>@indexes</tt> state of <tt>[0,0]</tt> would read
+      # <tt>(- x 1)</tt>, a state of <tt>[0,1]</tt> would read <tt>(- acc 1)</tt>,
+      # and <tt>[2,0]</tt> would read <tt>(* x acc)</tt>; the latter instructing
+      # the +Tree+ to get the third element of the root array, then the first
+      # element of _that_ array to find the right value.
+      #
+      # In practise, all +Tree+ objects have an extra array around the data as
+      # presented above, to make the no-repetition case consistent with the
+      # representation for arbitrarily nested repetitions. That is, the methods
+      # in this class expect to read from an array in general, so the representation
+      # of a non-repeating pattern is just a single-element array to simplify
+      # the implementation of these methods in the general case. The first item
+      # in the <tt>@indexes</tt> array is always zero. We could remove this extra
+      # container and add a type check on <tt>@data</tt> when reading, but the
+      # current implementation seems more elegant for the moment.
+      #
+      class Tree
+        
+        # A +Tree+ is initialized using the name of the pattern variable it is
+        # associated with (for debugging purposes).
+        def initialize(name)
+          @name  = name
+          @data  = []
+          @depth = 0
+        end
+        
+        # Tells the receiving +Tree+ that its pattern variable has been visited
+        # at a repetition depth of +depth+ during pattern matching. This allocates
+        # a new empty array at an appropriate place in the tree to store matches
+        # (or groups of matches) if any are encountered. Calls to this method are
+        # also used to determine the tree's maximum depth.
+        def descend!(depth)
+          tail(depth-1) << []
+          @depth = depth if depth > @depth
+        end
+        
+        # Pushes an expression onto the end of the final branch of the tree. All
+        # expressions should exist at the same depth (the tree's maximum depth),
+        # seeing as the pattern should be followed by the same number of ellipses
+        # every time it is encountered.
+        def <<(value)
+          return if Cons::NULL == value
+          tail(@depth) << value
+        end
+        
+        # Returns the expression at the current read position as instructed by
+        # the <tt>@indexes</tt> list.
+        def read
+          current(@depth)[indexes[@depth]]
+        end 
+        
+        # Shifts the read position at the given +depth+ along by one, by adding 1
+        # to one of the values in <tt>@indexes</tt>. The macro expander calls this
+        # while walking a template to iterate over repetition branches.
+        def shift!(depth)
+          indexes[depth] += 1
+          indexes[depth] = 0 if indexes[depth] >= current(depth).size
+        end
+        
+        # Returns the number of matches (or groups of matches) on the current
+        # read branch at the given +depth+. Returns zero if no branch exists at
+        # the given indexes.
+        def size(depth)
+          current(depth).size rescue 0
+        end
+        
+      private
+        
+        # Returns the rightmost branch of the tree at the given +depth+. Used
+        # when allocating new branches as repetition blocks are entered.
+        def tail(depth)
+          (0...depth).inject(@data) { |list, d| list.last }
+        end
+        
+        # Returns the current read branch at the given +depth+, as instructed
+        # by the <tt>@indexes</tt> list.
+        def current(depth)
+          indexes[0...depth].inject(@data) { |list, i| list[i] }
+        end
+        
+        # Initializes the <tt>@indexes</tt> list once the maximum depth is
+        # known, and returns the list thereafter.
+        def indexes
+          @indexes ||= (0..@depth).map { 0 }
+          @indexes
+        end
+      end
+      
+    end
+  end
+end
+