comp_tree 0.5.0

data/lib/comp_tree/driver.rb

@@ -0,0 +1,267 @@
+
+require 'comp_tree/bucket_ipc'
+require 'comp_tree/quix/diagnostic'
+require 'comp_tree/quix/kernel'
+require 'comp_tree/algorithm'
+require 'comp_tree/node'
+require 'comp_tree/task_node'
+require 'comp_tree/error'
+
+require 'thread'
+
+module CompTree
+  #
+  # Driver is the main interface to the computation tree.  It is
+  # responsible for defining nodes and running computations.
+  #
+  class Driver
+    DEFAULTS = {
+      :threads => 1,
+      :fork => false,
+      :timeout => 5.0,
+      :wait_interval => 0.02,
+    }
+
+    include Quix::Diagnostic #:nodoc:
+    include Quix::Kernel #:nodoc:
+    
+    #
+    # Begin a new computation tree.
+    #
+    # Options hash:
+    #
+    # <tt>:node_class</tt> -- (Class) CompTree::Node subclass from
+    # which nodes are created.
+    #
+    # <tt>:discard_result</tt> -- (boolean) If you are <em>not</em>
+    # interested in the final answer, but only in the actions which
+    # complete the computation, then set this to +true+.  This is
+    # equivalent to saying <tt>:node_class => CompTree::TaskNode</tt>.
+    # (If you are forking processes, it is good to know that IPC is
+    # not needed to communicate the result.)
+    #
+    def initialize(opts = nil)
+      if opts and opts[:node_class] and opts[:discard_result]
+        raise(
+          Error::ArgumentError,
+          "#{self.class.name}.new: :discard_result and :node_class " +
+          "are mutually exclusive")
+      end
+
+      @node_class =
+        if opts and opts[:node_class]
+          opts[:node_class]
+        elsif opts and opts[:discard_result]
+          TaskNode
+        else
+          Node
+        end
+
+      @nodes = Hash.new
+
+      if block_given?
+        yield self
+      end
+    end
+
+    #
+    # Name-to-node hash.
+    #
+    attr_reader :nodes
+
+    #
+    # Define a computation node.
+    #
+    # There are three distinct forms of a node definition.  In each of
+    # the following examples, a computation node named +area+ is
+    # defined which depends on the nodes +height+, +width+, +offset+.
+    #
+    # The method_missing form:
+    #  driver.define_area(:width, :height, :offset) { |width, height, offset|
+    #    width*height - offset
+    #  }
+    #
+    # The eval form:
+    #  driver.define_area :width, :height, :offset, %{
+    #    width*height - offset
+    #  }
+    # (Note the '%' before the brace.)
+    #
+    # The raw form:
+    #  driver.define(:area, :width, :height, :offset) { |width, height, offset|
+    #    width*height - offset
+    #  }
+    #
+    def define(*args, &block)
+      parent_name = args.first
+      children_names = args[1..-1]
+      
+      unless parent_name
+        raise Error::ArgumentError, "No name given for node"
+      end
+      
+      #
+      # retrieve or create parent and children
+      #
+      parent =
+        if t = @nodes[parent_name]
+          t
+        else 
+          @nodes[parent_name] = @node_class.new(parent_name)
+        end
+
+      if parent.function
+        raise Error::RedefinitionError, "Node #{parent.name} already defined."
+      end
+      parent.function = block
+      
+      children = children_names.map { |child_name|
+        if t = @nodes[child_name]
+          t
+        else
+          @nodes[child_name] = @node_class.new(child_name)
+        end
+      }
+
+      #
+      # link
+      #
+      parent.children = children
+      children.each { |child|
+        child.parents << parent
+      }
+    end
+
+    #
+    # parsing/evaling helper
+    #
+    def evaling_define(*args) #:nodoc:
+      function_name = args[0]
+      function_arg_names = args[1..-2]
+      function_string = args.last.to_str
+      
+      comma_separated = function_arg_names.map { |name|
+        name.to_s
+      }.join(",")
+
+      eval_me = %{ 
+        lambda { |#{comma_separated}|
+          #{function_string}
+        }
+      }
+
+      function = eval(eval_me, TOPLEVEL_BINDING)
+
+      define(function_name, *function_arg_names, &function)
+    end
+
+    def method_missing(symbol, *args, &block) #:nodoc:
+      if match = symbol.to_s.match(%r!\Adefine_(\w+)\Z!)
+        method_name = match.captures.first.to_sym
+        if block
+          define(method_name, *args, &block)
+        else
+          evaling_define(method_name, *args)
+        end
+      else
+        super(symbol, *args, &block)
+      end
+    end
+
+    #
+    # Mark this node and all its children as uncomputed.
+    #
+    # Arguments:
+    #
+    # +name+ -- (Symbol) node name.
+    #
+    def reset(name)
+      @nodes[name].reset
+    end
+
+    #
+    # Check for a cyclic graph below the given node.  Raises
+    # CompTree::Error::CircularError if found.
+    #
+    # Arguments:
+    #
+    # +name+ -- (Symbol) node name.
+    #
+    def check_circular(name)
+      helper = lambda { |root, chain|
+        if chain.include? root
+          raise Error::CircularError,
+            "Circular dependency detected: #{root} => #{chain.last} => #{root}"
+        end
+        @nodes[root].children.each { |child|
+          helper.call(child.name, chain + [root])
+        }
+      }
+      helper.call(name, [])
+    end
+
+    #
+    # Compute this node.
+    #
+    # Arguments:
+    #
+    # +name+ -- (Symbol) node name.
+    #
+    # Options hash:
+    #
+    # <tt>:threads</tt> -- (Integer) Number of parallel threads.
+    #
+    # <tt>:fork</tt> -- (boolean) Whether to fork each computation
+    # node into its own process.
+    #
+    # Defaults options are taken from Driver::DEFAULTS.
+    #
+    def compute(name, opts = nil)
+      #
+      # Undocumented options:
+      #
+      # <tt>:wait_interval</tt> -- (seconds) (Obscure) How long to
+      # wait after an IPC failure.
+      #
+      # <tt>:timeout</tt> -- (seconds) (Obscure) Give up after this
+      # period of persistent IPC failures.
+      #
+
+      abort_on_exception {
+        compute_private(name, opts || Hash.new)
+      }
+    end
+
+    private
+    
+    def compute_private(name, opts_in)
+      opts = DEFAULTS.merge(opts_in)
+      root = @nodes[name]
+
+      if opts[:threads] < 1
+        raise Error::ArgumentError, "threads is #{opts[:threads]}"
+      end
+
+      if opts[:threads] == 1
+        root.result = root.compute_now
+      elsif opts[:fork] and not @node_class.discard_result?
+        #
+        # Use buckets to send results across forks.
+        #
+        result = nil
+        BucketIPC::Driver.new(opts[:threads], opts) { |buckets|
+          result =
+            Algorithm.compute_multithreaded(
+              root, opts[:threads], opts[:fork], buckets)
+        }
+        result
+      else
+        #
+        # Multithreaded computation without fork.
+        #
+        Algorithm.compute_multithreaded(
+          root, opts[:threads], opts[:fork], nil)
+      end
+    end
+  end
+end

data/lib/comp_tree/error.rb

@@ -0,0 +1,27 @@
+
+module CompTree
+  module Error
+    # Base class for CompTree errors.
+    class Base < StandardError ; end
+    
+    # Internal error inside CompTree.  Please send a bug report.
+    class AssertionFailed < Base ; end
+    
+    # Bad arguments were passed to a method.
+    class ArgumentError < Base ; end
+    
+    #
+    # Attempt to redefine a Node.
+    #
+    # If you wish to only replace the function, set
+    #   driver.nodes[name].function = some_new_lambda
+    #
+    class RedefinitionError < Base ; end
+    
+    # A Cyclic graph was detected.
+    class CircularError < Base ; end
+
+    # No function was defined for this node.
+    class NoFunctionError < Base ; end
+  end
+end

data/lib/comp_tree/node.rb

@@ -0,0 +1,165 @@
+
+require 'comp_tree/quix/diagnostic'
+require 'thread'
+
+module CompTree
+  #
+  # Base class for nodes in the computation tree.
+  # 
+  class Node
+    include Quix::Diagnostic #:nodoc:
+
+    attr_reader :name                   #:nodoc:
+
+    attr_accessor :parents              #:nodoc:
+    attr_accessor :children             #:nodoc:
+    attr_accessor :function             #:nodoc:
+    attr_accessor :result               #:nodoc:
+    attr_accessor :shared_lock          #:nodoc:
+
+    #
+    # Create a node
+    #
+    def initialize(name) #:nodoc:
+      @name = name
+      @mutex = Mutex.new
+      @children = []
+      @parents = []
+      reset_self
+    end
+
+    #
+    # Reset the computation for this node.
+    #
+    def reset_self #:nodoc:
+      @shared_lock = 0
+      @children_results = nil
+      @result = nil
+    end
+
+    #
+    # Reset the computation for this node and all children.
+    #
+    def reset #:nodoc:
+      each_downward { |node|
+        node.reset_self
+      }
+    end
+
+    def each_downward(&block) #:nodoc:
+      block.call(self)
+      @children.each { |child|
+        child.each_downward(&block)
+      }
+    end
+
+    def each_upward(&block) #:nodoc:
+      block.call(self)
+      @parents.each { |parent|
+        parent.each_upward(&block)
+      }
+    end
+
+    def each_child #:nodoc:
+      @children.each { |child|
+        yield(child)
+      }
+    end
+
+    #
+    # Force computation of all children; intended for
+    # single-threaded mode.
+    #
+    def compute_now #:nodoc:
+      unless @children_results
+        @children_results = @children.map { |child|
+          child.compute_now
+        }
+      end
+      compute
+    end
+    
+    #
+    # If all children have been computed, return their results;
+    # otherwise return nil.
+    #
+    def children_results #:nodoc:
+      if @children_results
+        @children_results
+      else
+        results = @children.map { |child|
+          if child_result = child.result
+            child_result
+          else
+            return nil
+          end
+        }
+        @children_results = results
+      end
+    end
+
+    def trace_compute #:nodoc:
+      debug {
+        # --- own mutex
+        trace "Computing #{@name}"
+        raise Error::AssertionFailed if @result
+        raise Error::AssertionFailed unless @mutex.locked?
+        raise Error::AssertionFailed unless @children_results
+      }
+    end
+
+    #
+    # Compute this node; children must be computed and lock must be
+    # already acquired.
+    #
+    def compute #:nodoc:
+      unless defined?(@function) and @function
+        raise Error::NoFunctionError,
+          "No function was defined for node '#{@name.inspect}'"
+      end
+      @function.call(*@children_results)
+    end
+
+    def try_lock #:nodoc:
+      # --- shared tree mutex and own mutex
+      if @shared_lock == 0 and @mutex.try_lock
+        trace "Locking #{@name}"
+        each_upward { |node|
+          node.shared_lock += 1
+          trace "#{node.name} locked by #{@name}: level: #{node.shared_lock}"
+        }
+        true
+      else
+        false
+      end
+    end
+
+    def unlock #:nodoc:
+      # --- shared tree mutex and own mutex
+      debug {
+        raise Error::AssertionFailed unless @mutex.locked?
+        trace "Unlocking #{@name}"
+      }
+      each_upward { |node|
+        node.shared_lock -= 1
+        debug {
+          if node.shared_lock == 0
+            trace "#{node.name} unlocked by #{@name}"
+          end
+        }
+      }
+      @mutex.unlock
+    end
+
+    class << self
+      #
+      # Throw away the computation result?
+      #
+      # This Node base class always returns false.
+      #
+      def discard_result?
+        false
+      end
+    end
+  end
+end

data/lib/comp_tree/quix/builtin/kernel/tap.rb

@@ -0,0 +1,33 @@
+
+
+
+######################################################
+# 
+# **** DO NOT EDIT ****
+# 
+# **** THIS IS A GENERATED FILE *****
+# 
+######################################################
+
+
+
+unless respond_to? :tap
+  module Kernel
+    def tap
+      yield self
+      self
+    end
+  end
+end
+
+
+
+######################################################
+# 
+# **** DO NOT EDIT ****
+# 
+# **** THIS IS A GENERATED FILE *****
+# 
+######################################################
+
+

data/lib/comp_tree/quix/diagnostic.rb

@@ -0,0 +1,68 @@
+
+
+
+######################################################
+# 
+# **** DO NOT EDIT ****
+# 
+# **** THIS IS A GENERATED FILE *****
+# 
+######################################################
+
+
+
+require 'comp_tree/quix/builtin/kernel/tap'
+
+module CompTree::Quix
+  module Diagnostic
+    def show(desc = nil, stream = STDOUT, &block)
+      if desc
+        stream.puts(desc)
+      end
+      if block
+        expression = block.call
+        eval(expression, block.binding).tap { |result|
+          stream.printf("%-16s => %s\n", expression, result.inspect)
+        }
+      end
+    end
+
+    if $DEBUG
+      def debug
+        yield
+      end
+
+      def debugging?
+        true
+      end
+
+      def trace(desc = nil, &block)
+        if desc
+          show("#{desc}.".sub(%r!\.\.+\Z!, ""), STDERR, &block)
+        else
+          show(nil, STDERR, &block)
+        end
+      end
+    else
+      # non-$DEBUG
+      def debug ; end
+      def debugging? ; end
+      def trace(*args) ; end
+    end
+
+    extend self
+  end
+end
+
+
+
+
+######################################################
+# 
+# **** DO NOT EDIT ****
+# 
+# **** THIS IS A GENERATED FILE *****
+# 
+######################################################
+
+

data/lib/comp_tree/quix/kernel.rb

@@ -0,0 +1,85 @@
+
+
+
+######################################################
+# 
+# **** DO NOT EDIT ****
+# 
+# **** THIS IS A GENERATED FILE *****
+# 
+######################################################
+
+
+
+require 'thread'
+
+module CompTree::Quix
+  module Kernel
+    def let
+      yield self
+    end
+
+    def singleton_class
+      class << self
+        self
+      end
+    end
+
+    module Gensym
+      @mutex = Mutex.new
+      @count = 0
+
+      def gensym(prefix = nil)
+        count = Gensym.module_eval {
+          @mutex.synchronize {
+            @count += 1
+          }
+        }
+        "#{prefix || :G}_#{count}_#{rand}".to_sym
+      end
+    end
+    include Gensym
+
+    def call_private(method, *args, &block)
+      instance_eval { send(method, *args, &block) }
+    end
+
+    def with_warnings(value = true)
+      previous = $VERBOSE
+      $VERBOSE = value
+      begin
+        yield
+      ensure
+        $VERBOSE = previous
+      end
+    end
+
+    def no_warnings(&block)
+      with_warnings(false, &block)
+    end
+
+    def abort_on_exception(value = true)
+      previous = Thread.abort_on_exception
+      Thread.abort_on_exception = value
+      begin
+        yield
+      ensure
+        Thread.abort_on_exception = previous
+      end
+    end
+
+    extend self
+  end
+end
+
+
+
+######################################################
+# 
+# **** DO NOT EDIT ****
+# 
+# **** THIS IS A GENERATED FILE *****
+# 
+######################################################
+
+

data/lib/comp_tree/retriable_fork.rb

@@ -0,0 +1,42 @@
+
+module CompTree
+  module RetriableFork
+    HAVE_FORK = lambda {
+      begin
+        process_id = fork { }
+        Process.wait(process_id)
+      rescue NotImplementedError
+        return false
+      end
+      true
+    }.call
+
+    def fork(retry_wait = 10, retry_max = 10, &block)
+      num_retries = 0
+      begin
+        Process.fork(&block)
+      rescue Errno::EAGAIN
+        num_retries += 1
+        if num_retries == retry_max
+          message = %Q{
+            ****************************************************************
+            Maximum number of EAGAIN signals reached (#{retry_max})
+            ****************************************************************
+
+            Either increase your process limit permission (consult your
+            OS manual) or run this script as superuser.
+  
+            ****************************************************************
+          }
+          STDERR.puts(message.gsub(%r!^[ \t]+!, ""))
+          raise
+        end
+        STDERR.puts "Caught EGAIN. Retrying in #{retry_wait} seconds."
+        sleep(retry_wait)
+        retry
+      end
+    end
+    module_function :fork
+  end
+end
+

data/lib/comp_tree/task_node.rb

@@ -0,0 +1,22 @@
+
+module CompTree
+  #
+  # TaskNode is a Node which discards its results
+  #
+  class TaskNode < Node
+    def compute #:nodoc:
+      @function.call
+      true
+    end
+
+    class << self
+      #
+      # TaskNode always returns true.
+      #
+      def discard_result?
+        true
+      end
+    end
+  end
+end
+