furnace 0.2.6 → 0.3.1

data/.gitignore

@@ -1,6 +1,7 @@
-*.gem
 .bundle
 Gemfile.lock
 pkg/*
 .rbx/
-*.sublime-*
+*.sublime-*
+doc/
+.yardoc/

data/.yardopts

@@ -0,0 +1 @@
+-r {Furnace} -m markdown --protected

data/Gemfile

@@ -2,3 +2,4 @@ source "http://rubygems.org"
 
 # Specify your gem's dependencies in furnace.gemspec
 gemspec
+gem 'bacon', github: 'chneukirchen/bacon'

data/Rakefile

@@ -1 +1,23 @@
-require "bundler/gem_tasks"
+require 'bundler/gem_tasks'
+require 'bundler/setup'
+
+task :default => :test
+
+desc "Run test suite"
+task :test do
+  require 'bacon'
+  Bacon.summary_at_exit
+  Dir["test/**/*_test.rb"].each do |file|
+    load file
+  end
+end
+
+PAGES_REPO = 'git@github.com:whitequark/furnace'
+
+desc "Build and deploy documentation to GitHub pages"
+task :pages do
+  system "git clone #{PAGES_REPO} gh-temp/ -b gh-pages; rm gh-temp/* -rf; touch gh-temp/.nojekyll" or abort
+  system "yardoc -o gh-temp/; cp gh-temp/frames.html gh-temp/index.html; sed s,index.html,_index.html, -i gh-temp/index.html" or abort
+  system "cd gh-temp/; git add -A; git commit -m 'Updated pages.'; git push -f origin gh-pages" or abort
+  FileUtils.rm_rf 'gh-temp'
+end

data/furnace.gemspec

@@ -16,4 +16,9 @@ Gem::Specification.new do |s|
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
+
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'bacon', '~> 1.1'
+  s.add_development_dependency 'yard'
+  s.add_development_dependency 'redcarpet'
 end

data/lib/furnace/ast/node.rb

@@ -1,36 +1,121 @@
 module Furnace::AST
+  # Node is an immutable class, instances of which represent abstract
+  # syntax tree nodes. It combines semantic information (i.e. anything
+  # that affects the algorithmic properties of a program) with
+  # meta-information (line numbers or compiler intermediates).
+  #
+  # Notes on inheritance
+  # ====================
+  #
+  # The distinction between semantics and metadata is important. Complete
+  # semantic information should be contained within just the {#type} and
+  # {#children} of a Node instance; in other words, if an AST was to be
+  # stripped of all meta-information, it should remain a valid AST which
+  # could be successfully processed to yield a result with the same
+  # algorithmic properties.
+  #
+  # Thus, Node should never be inherited in order to define methods which
+  # affect or return semantic information, such as getters for `class_name`,
+  # `superclass` and `body` in the case of a hypothetical `ClassNode`. The
+  # correct solution is to use a generic Node with a {#type} of `:class`
+  # and three children. See also {Processor} for tips on working with such
+  # ASTs.
+  #
+  # On the other hand, Node can and should be inherited to define
+  # application-specific metadata (see also {#initialize}) or customize the
+  # printing format. It is expected that an application would have one or two
+  # such classes and use them across the entire codebase.
+  #
+  # The rationale for this pattern is extensibility and maintainability.
+  # Unlike static ones, dynamic languages do not require the presence of a
+  # predefined, rigid structure, nor does it improve dispatch efficiency,
+  # and while such a structure can certainly be defined, it does not add
+  # any value but incurs a maintaining cost.
+  # For example, extending the AST even with a transformation-local
+  # temporary node type requires making globally visible changes to
+  # the codebase.
+  #
   class Node
-    attr_accessor :type, :children, :metadata
-
-    def initialize(type, children=[], metadata={})
-      @type, @children, @metadata = type.to_sym, children, metadata
+    # Returns the type of this node.
+    # @return [Symbol]
+    attr_reader :type
+
+    # Returns the children of this node.
+    # The returned value is frozen.
+    # @return [Array]
+    attr_reader :children
+
+    # Constructs a new instance of Node.
+    #
+    # The arguments `type` and `children` are converted with `to_sym` and
+    # `to_a` respectively. Additionally, the result of converting `children`
+    # is frozen. While mutating the arguments is generally considered harmful,
+    # the most common case is to pass an array literal to the constructor. If
+    # your code does not expect the argument to be frozen, use `#dup`.
+    #
+    # The `properties` hash is passed to {#assign_properties}.
+    def initialize(type, children=[], properties={})
+      @type, @children = type.to_sym, children.to_a.freeze
+
+      assign_properties(properties)
+
+      freeze
     end
 
-    def update(type, children=nil, metadata={})
-      @type     = type
-      @children = children || @children
-
-      # If something non-nil is passed, including default value, then merge.
-      # Else, clear metadata store.
-      if metadata
-        @metadata.merge!(metadata)
-      else
-        @metadata = {}
+    # By default, each entry in the `properties` hash is assigned to
+    # a local variable in this instance of Node. A subclass should define
+    # attribute readers for such variables. The values passed in the hash
+    # are not frozen or whitelisted; such behavior can also be implemented\
+    # by subclassing Node and overriding this method.
+    #
+    # @return [nil]
+    def assign_properties(properties)
+      properties.each do |name, value|
+        instance_variable_set :"@#{name}", value
       end
 
-      self
+      nil
     end
-
-    def dup
-      node = super
-      node.children = @children.dup
-      node.metadata = @metadata.dup
-      node
+    protected :assign_properties
+
+    protected :dup
+
+    # Returns a new instance of Node where non-nil arguments replace the
+    # corresponding fields of `self`.
+    #
+    # For example, `Node.new(:foo, [ 1, 2 ]).updated(:bar)` would yield
+    # `(bar 1 2)`, and `Node.new(:foo, [ 1, 2 ]).updated(nil, [])` would
+    # yield `(foo)`.
+    #
+    # If the resulting node would be identical to `self`, does nothing.
+    #
+    # @param  [Symbol, nil] type
+    # @param  [Array, nil]  children
+    # @param  [Hash, nil]   properties
+    # @return [AST::Node]
+    def updated(type=nil, children=nil, properties=nil)
+      new_type       = type       || @type
+      new_children   = children   || @children
+      new_properties = properties || {}
+
+      if @type == new_type &&
+          @children == new_children &&
+          properties.nil?
+        self
+      else
+        dup.send :initialize, new_type, new_children, new_properties
+      end
     end
 
+    # Compares `self` to `other`, possibly converting with `to_ast`. Only
+    # `type` and `children` are compared; metadata is deliberately ignored.
+    #
+    # @return [Boolean]
     def ==(other)
-      if other.respond_to? :to_astlet
-        other = other.to_astlet
+      if equal?(other)
+        true
+      elsif other.respond_to? :to_ast
+        other = other.to_ast
         other.type == self.type &&
           other.children == self.children
       else
@@ -38,63 +123,65 @@ module Furnace::AST
       end
     end
 
+    # Converts `self` to a concise s-expression, omitting any children.
+    #
+    # @return [String]
     def to_s
       "(#{fancy_type} ...)"
     end
 
+    # Converts `self` to a pretty-printed s-expression.
+    #
+    # @param  [Integer] indent Base indentation level.
+    # @return [String]
     def to_sexp(indent=0)
-      str = "#{"  " * indent}(#{fancy_type}"
-
-      if @metadata[:ellipsis]
-        str << " <omitted>)"
-
-        return str
-      end
-
-      children.each do |child|
-        if (!children[0].is_a?(Node) && child.is_a?(Node)) ||
-            (children[0].is_a?(Node) && child.is_a?(Node) &&
-              child.children.any? { |c| c.is_a?(Node) || c.is_a?(Array) }) ||
-            (child.is_a?(Node) && child.metadata[:label])
-          str << "\n#{child.to_sexp(indent + 1)}"
+      indented = "  " * indent
+      sexp = "#{indented}(#{fancy_type}"
+
+      first_node_child = children.index do |child|
+        child.is_a?(Node) || child.is_a?(Array)
+      end || children.count
+
+      children.each_with_index do |child, idx|
+        if child.is_a?(Node) && idx >= first_node_child
+          sexp << "\n#{child.to_sexp(indent + 1)}"
+        elsif child.is_a?(Hash)
+          sexp << " {\n"
+          child.each do |key, value|
+            if value.is_a?(Node)
+              pretty_value = value.to_sexp(indent + 2).lstrip
+            else
+              pretty_value = value.inspect
+            end
+
+            sexp << "#{indented}    #{key.inspect} => #{pretty_value}\n"
+          end
+          sexp << "#{indented}  }"
         else
-          str << " #{child.inspect}"
+          sexp << " #{child.inspect}"
         end
       end
 
-      str << ")"
+      sexp << ")"
 
-      str
+      sexp
     end
     alias :inspect :to_sexp
 
-    def to_astlet
+    # @return [AST::Node] self
+    def to_ast
       self
     end
 
     protected
 
+    # Returns `@type` with all underscores replaced by dashes. This allows
+    # to write symbol literals without quotes in Ruby sources and yet have
+    # nicely looking s-expressions.
+    #
+    # @return [String]
     def fancy_type
-      dasherized = @type.to_s.gsub('_', '-')
-
-      if @metadata.any?
-        metainfo = @metadata.dup
-        metainfo.delete :label
-        metainfo.delete :origin
-        metainfo.delete :ellipsis
-
-        if metainfo.any?
-          metainfo = "#{metainfo.inspect}:"
-        else
-          metainfo = nil
-        end
-      end
-
-      if @metadata[:label]
-        "#{@metadata[:label]}:#{metainfo}#{dasherized}"
-      else
-        "#{metainfo}#{dasherized}"
-      end
+      @type.to_s.gsub('_', '-')
     end
   end
 end

data/lib/furnace/ast/processor.rb

@@ -0,0 +1,265 @@
+module Furnace::AST
+  # Processor is a class which helps transforming one AST into another.
+  # In a nutshell, the {#process} method accepts a {Node} and dispatches
+  # it to a handler corresponding to its type, and returns a (possibly)
+  # updated variant of the node.
+  #
+  # Processor has a set of associated design patterns. They are best
+  # explained with a concrete example. Let's define a simple arithmetic
+  # language and an AST format for it:
+  #
+  # Terminals (AST nodes which do not have other AST nodes inside):
+  #
+  #   * `(integer <int-literal>)`,
+  #
+  # Nonterminals (AST nodes with other nodes as children):
+  #
+  #   * `(add <node> <node>)`,
+  #   * `(multiply <node> <node>)`,
+  #   * `(divide <node> <node>)`,
+  #   * `(negate <node>)`,
+  #   * `(store <node> <string-literal>)`: stores value of `<node>` into a variable named `<string-literal>`,
+  #   * `(load <string-literal>)`: loads value of a variable named `<string-literal>`,
+  #   * `(each <node> ...): computes each of the `<node>`s and prints the result.
+  #
+  # Furnace AST nodes all have the same Ruby class, and therefore they don't
+  # know how to traverse themselves. (A solution which dynamically checks the
+  # type of children is possible, but is slow and error-prone.) So, a subclass
+  # of Processor which knows how to traverse the entire tree should be defined.
+  # Such subclass has a handler for each nonterminal node which recursively
+  # processes children nodes:
+  #
+  #     require 'furnace'
+  #     include Furnace
+  #
+  #     class ArithmeticsProcessor < AST::Processor
+  #       # This method traverses any binary operators such as (add) or (multiply).
+  #       def process_binary_op(node)
+  #         # Children aren't decomposed automatically; it is suggested to use Ruby
+  #         # multiple assignment expansion, as it is very convenient here.
+  #         left_expr, right_expr = node.children
+  #
+  #         # AST::Node#updated won't change node type if nil is passed as a first
+  #         # argument, which allows to reuse the same handler for multiple node types
+  #         # using `alias' (below).
+  #         node.updated(nil, [
+  #           process(left_expr),
+  #           process(right_expr)
+  #         ])
+  #       end
+  #       alias on_add      process_binary_op
+  #       alias on_multiply process_binary_op
+  #       alias on_divide   process_binary_op
+  #
+  #       def on_negate(node)
+  #         # It is also possible to use #process_all for more compact code
+  #         # if every child is a Node.
+  #         node.updated(nil, process_all(node.children))
+  #       end
+  #
+  #       def on_store(node)
+  #         expr, variable_name = node.children
+  #
+  #         # Note that variable_name is not a Node and thus isn't passed to #process.
+  #         node.updated(nil, [
+  #           process(expr),
+  #           variable_name
+  #         ])
+  #       end
+  #
+  #       # (load) is effectively a terminal node, and so it does not need
+  #       # an explicit handler, as the following is the default behavior.
+  #       def on_load(node)
+  #         nil
+  #       end
+  #
+  #       def on_each(node)
+  #         node.updated(nil, process_all(node.children))
+  #       end
+  #     end
+  #
+  # Let's test our ArithmeticsProcessor:
+  #
+  #     include AST::Sexp
+  #     expr = s(:add, s(:integer, 2), s(:integer, 2))
+  #
+  #     p ArithmeticsProcessor.new.process(expr) == expr # => true
+  #
+  # As expected, it does not change anything at all. This isn't actually
+  # very useful, so let's now define a Calculator, which will compute the
+  # expression values:
+  #
+  #     # This Processor folds nonterminal nodes and returns an (integer)
+  #     # terminal node.
+  #     class ArithmeticsCalculator < ArithmeticsProcessor
+  #       def compute_op(node)
+  #         # First, node children are processed and then unpacked to local
+  #         # variables.
+  #         nodes = process_all(node.children)
+  #
+  #         if nodes.all? { |node| node.type == :integer }
+  #           # If each of those nodes represents a literal, we can fold this
+  #           # node!
+  #           values = nodes.map { |node| node.children.first }
+  #           AST::Node.new(:integer, [
+  #             yield(values)
+  #           ])
+  #         else
+  #           # Otherwise, we can just leave the current node in the tree and
+  #           # only update it with processed children nodes, which can be
+  #           # partially folded.
+  #           node.updated(nil, nodes)
+  #         end
+  #       end
+  #
+  #       def on_add(node)
+  #         compute_op(node) { |left, right| left + right }
+  #       end
+  #
+  #       def on_multiply(node)
+  #         compute_op(node) { |left, right| left * right }
+  #       end
+  #     end
+  #
+  # Let's check:
+  #
+  #     p ArithmeticsCalculator.new.process(expr) # => (integer 4)
+  #
+  # Excellent, the calculator works! Now, a careful reader could notice that
+  # the ArithmeticsCalculator does not know how to divide numbers. What if we
+  # pass an expression with division to it?
+  #
+  #     expr_with_division = \
+  #       s(:add,
+  #         s(:integer, 1),
+  #         s(:divide,
+  #           s(:add, s(:integer, 8), s(:integer, 4)),
+  #           s(:integer, 3))) # 1 + (8 + 4) / 3
+  #
+  #     folded_expr_with_division = ArithmeticsCalculator.new.process(expr_with_division)
+  #     p folded_expr_with_division
+  #     # => (add
+  #     #      (integer 1)
+  #     #      (divide
+  #     #        (integer 12)
+  #     #        (integer 3)))
+  #
+  # As you can see, the expression was folded _partially_: the inner `(add)` node which
+  # could be computed was folded to `(integer 12)`, the `(divide)` node is left as-is
+  # because there is no computing handler for it, and the root `(add)` node was also left
+  # as it is because some of its children were not literals.
+  #
+  # Note that this partial folding is only possible because the _data_ format, i.e.
+  # the format in which the computed values of the nodes are represented, is the same as
+  # the AST itself.
+  #
+  # Let's extend our ArithmeticsCalculator class further.
+  #
+  #     class ArithmeticsCalculator
+  #       def on_divide(node)
+  #         compute_op(node) { |left, right| left / right }
+  #       end
+  #
+  #       def on_negate(node)
+  #         # Note how #compute_op works regardless of the operator arity.
+  #         compute_op(node) { |value| -value }
+  #       end
+  #     end
+  #
+  # Now, let's apply our renewed ArithmeticsCalculator to a partial result of previous
+  # evaluation:
+  #
+  #     p ArithmeticsCalculator.new.process(expr_with_division) # => (integer 5)
+  #
+  # Five! Excellent. This is also pretty much how CRuby 1.8 executed its programs.
+  #
+  # Now, let's do some automated bug searching. Division by zero is an error, right?
+  # So if we could detect that someone has divided by zero before the program is even
+  # run, that could save some debugging time.
+  #
+  #     class DivisionByZeroVerifier < ArithmeticsProcessor
+  #       class VerificationFailure < Exception; end
+  #
+  #       def on_divide(node)
+  #         # You need to process the children to handle nested divisions
+  #         # such as:
+  #         # (divide
+  #         #   (integer 1)
+  #         #   (divide (integer 1) (integer 0))
+  #         left, right = process_all(node.children)
+  #
+  #         if right.type == :integer &&
+  #            right.children.first == 0
+  #           raise VerificationFailure, "Ouch! This code divides by zero."
+  #         end
+  #       end
+  #
+  #       def divides_by_zero?(ast)
+  #         process(ast)
+  #         false
+  #       rescue VerificationFailure
+  #         true
+  #       end
+  #     end
+  #
+  #     nice_expr = \
+  #       s(:divide,
+  #         s(:add, s(:integer, 10), s(:integer, 2)),
+  #         s(:integer, 4))
+  #
+  #     p DivisionByZeroVerifier.new.divides_by_zero?(nice_expr)
+  #     # => false. Good.
+  #
+  #     bad_expr = \
+  #       s(:add, s(:integer, 10),
+  #         s(:divide, s(:integer, 1), s(:integer, 0)))
+  #
+  #     p DivisionByZeroVerifier.new.divides_by_zero?(bad_expr)
+  #     # => true. WHOOPS. DO NOT RUN THIS.
+  #
+  # Of course, this won't detect more complex cases... unless you use some partial
+  # evaluation before! The possibilites are endless. Have fun.
+  class Processor
+    # Dispatches `node`. If a node has type `:foo`, then a handler named
+    # `on_foo` is invoked with one argument, the `node`; if there isn't
+    # such a handler, {#handler_missing} is invoked with the same argument.
+    #
+    # If the handler returns `nil`, `node` is returned; otherwise, the return
+    # value of the handler is passed along.
+    #
+    # @param  [AST::Node, nil] node
+    # @return [AST::Node]
+    def process(node)
+      node = node.to_ast
+
+      # Invoke a specific handler
+      on_handler = :"on_#{node.type}"
+      if respond_to? on_handler
+        new_node = send on_handler, node
+      else
+        new_node = handler_missing(node)
+      end
+
+      node = new_node if new_node
+
+      node
+    end
+
+    # {#process}es each node from `nodes` and returns an array of results.
+    #
+    # @param  [Array<AST::Node>] nodes
+    # @return [Array<AST::Node>]
+    def process_all(nodes)
+      nodes.map do |node|
+        process node
+      end
+    end
+
+    # Default handler. Does nothing.
+    #
+    # @param  [AST::Node] node
+    # @return [AST::Node, nil]
+    def handler_missing(node)
+    end
+  end
+end

data/lib/furnace/ast/sexp.rb

@@ -0,0 +1,30 @@
+module Furnace::AST
+  # This simple module is very useful in the cases where one needs
+  # to define deeply nested ASTs from Ruby code, for example, in
+  # tests. It should be used like this:
+  #
+  #     describe YourLanguage::AST do
+  #       include Sexp
+  #
+  #       it "should correctly parse expressions" do
+  #         YourLanguage.parse("1 + 2 * 3").should ==
+  #             s(:add,
+  #               s(:integer, 1),
+  #               s(:multiply,
+  #                 s(:integer, 2),
+  #                 s(:integer, 3)))
+  #       end
+  #     end
+  #
+  # This way the amount of boilerplate code is greatly reduced.
+  module Sexp
+    # Creates a {Node} with type `type` and children `children`.
+    # Note that the resulting node is of the type AST::Node and not a
+    # subclass.
+    # This would not pose a problem with comparisons, as {Node#==}
+    # ignores metadata.
+    def s(type, *children)
+      Node.new(type, children)
+    end
+  end
+end

data/lib/furnace/ast.rb

@@ -1,9 +1,19 @@
-require "furnace"
+module Furnace
+  # Furnace::AST is a library for manipulating abstract syntax trees.
+  #
+  # It embraces immutability; each AST node is inherently frozen at
+  # creation, and updating a child node requires recreating that node
+  # and its every parent, recursively.
+  # This is a design choice. It does create significant pressure on
+  # garbage collector, but completely eliminates all concurrency
+  # and aliasing problems.
+  #
+  # See also {Node}, {Processor} and {Sexp} for additional
+  # recommendations and design patterns.
+  module AST
+  end
 
-require "furnace/ast/node"
-require "furnace/ast/visitor"
-require "furnace/ast/strict_visitor"
-
-require "furnace/ast/matcher/special"
-require "furnace/ast/matcher/dsl"
-require "furnace/ast/matcher"
+  require_relative "ast/node"
+  require_relative "ast/processor"
+  require_relative "ast/sexp"
+end