comp_tree 0.5.0

data/contrib/quix/test/test_struct.rb

@@ -0,0 +1,48 @@
+$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../lib"
+
+require 'test/unit'
+require 'quix/lazy_struct'
+require 'quix/hash_struct'
+
+class TestLazyStruct < Test::Unit::TestCase
+  def common(s)
+    s.f = 33
+    assert_equal(33, s.f)
+
+    n = 0
+    s.attribute(:f) {
+      n += 1
+      44
+    }
+    
+    3.times {
+      assert_equal(44, s.f)
+    }
+    assert_equal(1, n)
+  end
+
+  def test_1
+    common(Quix::LazyStruct.new)
+  end
+
+  def test_2
+    s = OpenStruct.new
+    class << s
+      include Quix::LazyStruct::Mixin
+    end
+    common(s)
+  end
+
+  def test_3
+    s = Quix::HashStruct.new
+    class << s
+      include Quix::LazyStruct::Mixin
+    end
+    common(s)
+    s[:g] = 55
+    assert_equal(
+      { :f => 33, :g => 55 },
+      s.keys.inject(Hash.new) { |acc, e| acc.merge(e => s[e]) })
+  end
+end
+

data/contrib/quix/test/test_vars.rb

@@ -0,0 +1,187 @@
+$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../lib"
+
+require 'test/unit'
+require 'quix/vars'
+require 'quix/hash_struct'
+
+class TestVars < Test::Unit::TestCase
+  include Quix::Vars
+
+  def test_locals_to_hash
+    a = 33
+    b = Object.new
+    c = lambda { a + 11 }
+
+    hash = locals_to_hash {%{a, b, c}}
+
+    assert_equal(a.object_id, hash[:a].object_id)
+    assert_equal(b.object_id, hash[:b].object_id)
+    assert_equal(c.object_id, hash[:c].object_id)
+
+    assert_equal(hash[:c].call, 44)
+  end
+
+  def test_hash_to_locals
+    a = nil
+    b = nil
+    c = nil
+    
+    hash = {
+      :a => 33,
+      :b => Object.new,
+      :c => lambda { hash[:a] + 11 },
+    }
+
+    hash_to_locals { hash }
+
+    assert_equal(a.object_id, hash[:a].object_id)
+    assert_equal(b.object_id, hash[:b].object_id)
+    assert_equal(c.object_id, hash[:c].object_id)
+
+    assert_equal(hash[:c].call, 44)
+    assert_nothing_raised { hash_to_locals { nil } }
+  end
+
+  def test_with_readers
+    hash = {
+      :a => 33,
+      :b => Object.new,
+      :c => lambda { hash[:a] + 11 },
+    }
+
+    assert_raise(NameError) { a }
+    assert_raise(NameError) { b }
+    assert_raise(NameError) { c }
+
+    with_readers(hash) {
+      assert_equal(a.object_id, hash[:a].object_id)
+      assert_equal(b.object_id, hash[:b].object_id)
+      assert_equal(c.object_id, hash[:c].object_id)
+    }
+
+    assert_raise(NameError) { a }
+    assert_raise(NameError) { b }
+    assert_raise(NameError) { c }
+
+    with_readers(hash, :a, :b) {
+      assert_equal(a.object_id, hash[:a].object_id)
+      assert_equal(b.object_id, hash[:b].object_id)
+      assert_raise(NameError) { c }
+    }
+  end
+
+  def test_locals_to_ivs
+    a = 33
+    b = Object.new
+    c = lambda { a + 11 }
+
+    assert(!defined?(@a))
+    assert(!defined?(@b))
+    assert(!defined?(@c))
+    
+    locals_to_ivs {%{a, b, c}}
+
+    assert_equal(a.object_id, @a.object_id)
+    assert_equal(b.object_id, @b.object_id)
+    assert_equal(c.object_id, @c.object_id)
+
+    assert_equal(@c.call, 44)
+  end
+
+  def test_hash_to_ivs
+    hash = {
+      :d => 33,
+      :e => Object.new,
+      :f => lambda { hash[:d] + 11 },
+    }
+
+    assert(!defined?(@d))
+    assert(!defined?(@e))
+    assert(!defined?(@f))
+    
+    hash_to_ivs { hash }
+
+    assert_equal(hash[:d].object_id, @d.object_id)
+    assert_equal(hash[:e].object_id, @e.object_id)
+    assert_equal(hash[:f].object_id, @f.object_id)
+
+    assert_equal(hash[:f].call, 44)
+    assert_nothing_raised { hash_to_ivs { nil } }
+  end
+
+  def test_config_to_hash
+    config = %q{
+      a = 33
+      b = a + 11
+      c = 5*(a - 22)
+      d = (1..3).map { |n| n*n }
+      e = "moo"
+      f = lambda { a + 66 }
+
+      a_object_id = a.object_id
+      b_object_id = b.object_id
+      c_object_id = c.object_id
+      d_object_id = d.object_id
+      e_object_id = e.object_id
+      f_object_id = f.object_id
+    }
+
+    hash = config_to_hash(config)
+
+    assert_equal(hash[:a], 33)
+    assert_equal(hash[:b], 44)
+    assert_equal(hash[:c], 55)
+    assert_equal(hash[:d], [1, 4, 9])
+    assert_equal(hash[:e], "moo")
+    assert_equal(hash[:f].call, 99)
+
+    assert_equal(hash[:a].object_id, hash[:a_object_id])
+    assert_equal(hash[:b].object_id, hash[:b_object_id])
+    assert_equal(hash[:c].object_id, hash[:c_object_id])
+    assert_equal(hash[:d].object_id, hash[:d_object_id])
+    assert_equal(hash[:e].object_id, hash[:e_object_id])
+    assert_equal(hash[:f].object_id, hash[:f_object_id])
+  end
+
+  def test_hash_struct
+    hash = {
+      :a => {
+        :b => :c,
+        :d => :e,
+        :f => {
+          :g => :h,
+          :i => :j,
+        },
+      },
+      :k => :l,
+      :m => [ :n, :o, :p ],
+      :q => {
+        :r => {},
+        :s => [],
+      },
+      :t => [
+        {
+          :u => :v,       
+          :w => :x,       
+        },
+      ],
+      :w => {
+        :x => {
+          :y => :z,
+        },
+      },
+    }
+    
+    s = Quix::HashStruct.recursive_new(hash)
+    assert_equal(s.a.b, :c)
+    assert_equal(s.a.d, :e)
+    assert_equal(s.a.f.g, :h)
+    assert_equal(s.a.f.i, :j)
+    assert_equal(s.k, :l)
+    assert_equal(s.m, [:n, :o, :p])
+    assert_equal(s.q.r, OpenStruct.new)
+    assert_equal(s.q.s, [])
+    assert_equal(s.t, [{ :u => :v, :w => :x }])
+    assert_equal(s.w.x.y, :z)
+  end
+end

data/install.rb

@@ -0,0 +1,3 @@
+$LOAD_PATH.unshift "./contrib/quix/lib"
+require 'quix/simple_installer'
+Quix::SimpleInstaller.new.run

data/lib/comp_tree/algorithm.rb

@@ -0,0 +1,210 @@
+
+require 'comp_tree/quix/diagnostic'
+require 'comp_tree/retriable_fork'
+
+module CompTree
+  module Algorithm
+    include Quix::Diagnostic
+
+    def compute_multithreaded(root, num_threads, use_fork, buckets)
+      trace "Computing #{root.name} with #{num_threads} threads"
+      result = nil
+      mutex = Mutex.new
+      node_finished_condition = ConditionVariable.new
+      thread_wake_condition = ConditionVariable.new
+      threads = []
+      
+      # workaround: jruby gives "run" status for waiting on
+      # condition variable
+      num_threads_ready = 0
+
+      num_threads.times { |thread_index|
+        threads << Thread.new {
+          #
+          # wait for main thread
+          #
+          mutex.synchronize {
+            trace "Thread #{thread_index} waiting to start"
+            num_threads_ready += 1
+            thread_wake_condition.wait(mutex)
+          }
+
+          while true
+            trace "Thread #{thread_index} node search"
+
+            #
+            # Done! Thread will exit.
+            #
+            break if mutex.synchronize {
+              result
+            }
+
+            #
+            # Lock the tree and find a node.  The node we
+            # obtain, if any, is already locked.
+            #
+            node = mutex.synchronize {
+              find_node(root)
+            }
+
+            if node
+              trace "Thread #{thread_index} found node #{node.name}"
+
+              node_result =
+                compute_node(
+                  node,
+                  use_fork,
+                  buckets ? buckets[thread_index] : nil)
+              
+              mutex.synchronize {
+                node.result = node_result
+              }
+
+              #
+              # remove locks for this node (shared lock and own lock)
+              #
+              mutex.synchronize {
+                node.unlock
+                if node == root
+                  #
+                  # Root node was computed; we are done.
+                  #
+                  trace "Thread #{thread_index} got final answer"
+                  result = root.result
+                end
+                node_finished_condition.signal
+              }
+            else
+              trace "Thread #{thread_index}: no node found; sleeping."
+              mutex.synchronize {
+                thread_wake_condition.wait(mutex)
+              }
+            end
+          end
+          trace "Thread #{thread_index} exiting"
+        }
+      }
+
+      trace "Main: waiting for threads to launch and block."
+      while true
+        break if mutex.synchronize {
+          num_threads_ready == num_threads
+        }
+        Thread.pass
+      end
+      
+      trace "Main: entering main loop"
+      mutex.synchronize {
+        while true
+          trace "Main: waking threads"
+          thread_wake_condition.broadcast
+
+          if result
+            trace "Main: detected finish."
+            break
+          end
+
+          trace "Main: waiting for a node"
+          node_finished_condition.wait(mutex)
+          trace "Main: got a node"
+        end
+      }
+
+      trace "Main: waiting for threads to finish."
+      catch(:done) {
+        while true
+          mutex.synchronize {
+            throw :done if threads.all? { |thread|
+              thread.status == false
+            }
+            thread_wake_condition.broadcast
+          }
+          Thread.pass
+        end
+      }
+
+      trace "Main: computation done."
+      result
+    end
+
+    def find_node(node)
+      # --- only called inside mutex
+      trace "Looking for a node, starting with #{node.name}"
+      if node.result
+        #
+        # already computed
+        #
+        trace "#{node.name} has been computed"
+        nil
+      elsif node.children_results and node.try_lock
+        #
+        # Node is not computed and its children are computed;
+        # and we have the lock.  Ready to compute.
+        #
+        node
+      else
+        #
+        # locked or children not computed; recurse to children
+        #
+        trace "Checking #{node.name}'s children"
+        node.each_child { |child|
+          if next_node = find_node(child)
+            return next_node
+          end
+        }
+        nil
+      end
+    end
+
+    def compute_node(node, use_fork, bucket)
+      if use_fork
+        trace "About to fork for node #{node.name}"
+        if bucket
+          #
+          # Use our assigned bucket to transfer the result.
+          #
+          fork_node(node) {
+            node.trace_compute
+            bucket.contents = node.compute
+          }
+          bucket.contents
+        else
+          #
+          # No bucket -- discarding result
+          #
+          fork_node(node) {
+            node.trace_compute
+            node.compute
+          }
+          true
+        end
+      else
+        #
+        # No fork
+        #
+        node.trace_compute
+        node.compute
+      end
+    end
+
+    def fork_node(node)
+      trace "About to fork for node #{node.name}"
+      process_id = RetriableFork.fork {
+        trace "Fork: process #{Process.pid}"
+        node.trace_compute
+        yield
+        trace "Fork: computation done"
+      }
+      trace "Waiting for process #{process_id}"
+      Process.wait(process_id)
+      trace "Process #{process_id} finished"
+      exitstatus = $?.exitstatus
+      if exitstatus != 0
+        trace "Process #{process_id} returned #{exitstatus}; exiting."
+        exit(1)
+      end
+    end
+    
+    extend self
+  end
+end

data/lib/comp_tree/bucket_ipc.rb

@@ -0,0 +1,151 @@
+
+require 'drb'
+require 'thread'
+
+require 'comp_tree/retriable_fork'
+require 'comp_tree/quix/diagnostic'
+require 'comp_tree/quix/builtin/kernel/tap'
+
+module CompTree
+  module BucketIPC
+    class Bucket
+      include Quix::Diagnostic
+      include RetriableFork
+
+      def initialize(address, timeout, wait_interval)
+        trace "Making bucket with address #{address}"
+        
+        @remote_pid = fork {
+          own_object = Class.new {
+            attr_accessor(:contents)
+          }.new
+          server = DRb.start_service(address, own_object)
+          debug {
+            server.verbose = true
+          }
+          DRb.thread.join
+        }
+
+        @remote_object = DRbObject.new_with_uri(address)
+        @address = address
+        @timeout = timeout
+        @wait_interval = wait_interval
+      end
+      
+      attr_accessor(:timeout, :wait_interval)
+      attr_reader(:address)
+      
+      def contents=(new_contents)
+        connect {
+          @remote_object.contents = new_contents
+        }
+      end
+      
+      def contents
+        connect {
+          @remote_object.contents
+        }
+      end
+      
+      def stop
+        Process.kill("TERM", @remote_pid)
+      end
+      
+      private
+      
+      def connect
+        begin
+          return yield
+        rescue DRb::DRbConnError
+          start = Time.now
+          begin
+            Kernel.sleep(@wait_interval)
+            return yield
+          rescue DRb::DRbConnError
+            if Time.now - start > @timeout
+              raise
+            end
+            retry
+          end
+        end
+      end
+    end
+
+    class DriverBase
+      def initialize(addresses, timeout, wait_interval)
+        begin
+          @buckets = addresses.map { |address|
+            Bucket.new(address, timeout, wait_interval)
+          }
+          if block_given?
+            yield @buckets
+          end
+        ensure
+          if block_given?
+            stop
+          end
+        end
+      end
+      
+      def stop
+        if defined?(@buckets)
+          @buckets.each { |bucket|
+            bucket.stop
+          }
+        end
+      end
+    end
+
+    class Driver < DriverBase
+      DEFAULTS = {
+        :timeout => 0.5,
+        :wait_interval => 0.05,
+        :port_start => 18181,
+      }
+
+      module BucketCounter
+        @mutex = Mutex.new
+        @count = 0
+        class << self
+          def increment_count
+            @mutex.synchronize {
+              @count += 1
+            }
+          end
+
+          def map_indexes(num_buckets)
+            Array.new.tap { |result|
+              num_buckets.times {
+                result << yield(increment_count)
+              }
+            }
+          end
+        end
+      end
+
+      def initialize(num_buckets, opts_in = {})
+        opts = DEFAULTS.merge(opts_in)
+
+        addresses = 
+          if RetriableFork::HAVE_FORK
+            #
+            # Assume the existence of fork implies a unix machine.
+            #
+            require 'drb/unix'
+            basename = "drbunix://#{Dir.tmpdir}/bucket.#{Process.pid}.#{rand}"
+            BucketCounter.map_indexes(num_buckets) { |index|
+              "#{basename}.#{index}"
+            }
+          else
+            #
+            # Fallback: use the default socket.
+            #
+            BucketCounter.map_indexes(num_buckets) { |index|
+              "druby://localhost:#{opts[:port_start] + index}"
+            }
+          end
+        super(addresses, opts[:timeout], opts[:wait_interval])
+      end
+    end
+  end
+end