skynet 0.9.1 → 0.9.2

data/test/test_skynet_manager.rb

@@ -0,0 +1,142 @@
+require File.dirname(__FILE__) + '/test_helper.rb'
+
+class SkynetManagerTest < Test::Unit::TestCase
+
+  PIDS = []
+  PIDFILE = []
+  
+  def setup
+    Skynet.configure(
+      :ENABLE                         => false,
+      # :SKYNET_PIDS_FILE               => File.expand_path("#{RAILS_ROOT}/log/skynet_#{RAILS_ENV}.pids"),
+      :SKYNET_LOG_FILE                => STDOUT,
+      :SKYNET_LOG_LEVEL               => 4,
+      :MESSAGE_QUEUE_ADAPTER          => "Skynet::MessageQueueAdapter::Mysql",
+      :MYSQL_TEMPERATURE_CHANGE_SLEEP => 1,
+      :MYSQL_NEXT_TASK_TIMEOUT        => 1
+    )      
+    ActiveRecord::Base.establish_connection(
+        :adapter  => "mysql",
+        :host     => "localhost",
+        :username => "root",
+        :password => "",
+        :database => "skynet_test"
+      )
+
+    mq.clear_outstanding_tasks
+    mq.clear_worker_status
+
+    @pids = []
+    @pidfile = []
+    @worker_queue = []
+    
+    PIDS.delete_if {true}
+    PIDFILE.delete_if {true}
+
+    Process.stubs(:detatch).returns(true)    
+    Process.stubs(:kill).returns(true)
+
+    @minstance = Skynet::Manager.any_instance
+    @hostname = Socket.gethostname
+   
+    Skynet::Worker.any_instance.stubs(:max_memory_reached?).returns(false)
+  end                                                                     
+  
+  def setup_manager
+    @manager = Skynet::Manager.new(:script_path => "path", :workers => 2)
+    @manager.extend(Functor)
+
+    mq = functor
+    @manager.stubs(:mq).returns(mq)
+
+    mq.get_worker_version = 1
+    
+    worker_queue = @worker_queue
+    mq.read_all_worker_statuses = lambda do |hostname|
+      worker_queue
+    end
+
+    mq.take_worker_status = lambda do |options, timeout|
+      worker_queue.delete_if {|w|w.process_id == options[:process_id] }.first
+    end
+    
+    pids     = @pids                   
+    hostname = @hostname
+    @manager.define_method(:fork) do
+      newpid = pids.size + 1
+      pids << newpid
+      worker_info = {
+        :hostname     => hostname,
+        :process_id   => newpid,
+        :worker_type  => :task, 
+        :worker_id    => newpid,
+        :version      => 1
+      }
+            
+      worker_queue << Skynet::WorkerStatusMessage.new(worker_info)
+      newpid
+    end
+  end
+  
+  def test_manager_start  
+    setup_manager
+    @manager.start_workers
+    assert_equal 2, @manager.worker_pids.size
+    assert_equal @pids.sort, @manager.worker_pids.sort
+    assert_equal @pids.sort, @manager.worker_queue.collect {|q|q.process_id}.sort
+  end
+
+  def test_check_workers
+    setup_manager
+    Skynet::Manager.any_instance.expects(:worker_alive?).times(2).returns(true)
+    @manager.start_workers
+    @manager.check_workers
+    assert_equal 2, @manager.worker_pids.size
+    assert_equal @pids.sort, @manager.worker_pids.sort
+    assert_equal @pids.sort, @manager.worker_queue.collect {|q|q.process_id}.sort
+  end
+  
+  def test_running_pids
+    setup_manager
+    Skynet::Manager.any_instance.expects(:worker_alive?).with(1).returns(true)
+    Skynet::Manager.any_instance.expects(:worker_alive?).with(2).returns(false)
+    @manager.start_workers
+    @manager.check_workers
+    assert_equal 1, @manager.worker_pids.size
+    assert_equal [1], @manager.worker_pids.sort
+    assert_equal [1], @manager.worker_queue.collect {|q|q.process_id}.sort
+  end
+
+  ## XXX FIXME.  What happens if there's a worker missing from the pidfile, but was running?
+  def test_more_in_pidfile_than_queue_alive
+    setup_manager
+    @manager.expects(:worker_alive?).with(1).returns(false)
+    Skynet.configure(:SKYNET_LOG_LEVEL => 4) do
+      @manager.start_workers
+
+      @manager.expects(:worker_alive?).with(2).returns(true)
+      @worker_queue.delete_if {|w| w.process_id == 1}
+      @manager.check_workers
+    end
+    assert_equal 2, @manager.worker_pids.size
+    assert_equal [2,3], @manager.worker_pids.sort
+    assert_equal [2,3], @manager.worker_queue.collect {|q|q.process_id}.sort
+  end
+
+  def test_dead_workers
+    setup_manager
+    Skynet::Manager.any_instance.expects(:worker_alive?).times(1).with(1).returns(false)
+    # Skynet::Manager.any_instance.expects(:worker_alive?).with(2).returns(true)
+    @manager.start_workers     
+    @manager.check_workers
+    assert_equal [2], @manager.worker_pids.sort
+    assert_equal [2], @manager.worker_queue.collect {|q|q.process_id}.sort
+  end
+
+
+  private
+
+  def mq
+		Skynet::MessageQueueAdapter::Mysql.new
+  end
+end

data/test/test_skynet_message.rb

@@ -0,0 +1,229 @@
+require File.dirname(__FILE__) + '/test_helper.rb'
+
+class MysqlMessageQueueTest < Test::Unit::TestCase
+  
+  def setup
+    Skynet.configure(
+      :ENABLE                         => false,
+      :SKYNET_LOG_FILE                => STDOUT,
+      :SKYNET_LOG_LEVEL               => Logger::ERROR
+    )              
+  end      
+
+  def setup_worker_message
+    @worker_message = Skynet::WorkerStatusMessage.new(
+      :tasktype     => :task, 
+      :drburi       => "localhost", 
+      :job_id       => 1,
+      :task_id      => 2,
+      :payload      => "payload",
+      :payload_type => "task",
+      :expiry       => 20, 
+      :expire_time  => 1095108406.9251,
+      :iteration    => 0,
+      :name         => "name",       
+      :version      => 1,
+      :queue_id     => 0
+    )
+  end    
+
+  def setup_task_message
+    @task_message = Skynet::Message.new(
+      :tasktype     => :task, 
+      :drburi       => "localhost", 
+      :job_id       => 1,
+      :task_id      => 2,
+      :payload      => "payload",
+      :payload_type => "task",
+      :expiry       => 20, 
+      :expire_time  => 0,
+      :iteration    => 0,
+      :name         => "name",       
+      :version      => 1,
+      :retry        => 2,
+      :queue_id     => 0
+    )     
+  end    
+  
+  def test_worker_status_message
+    setup_worker_message
+    {
+       :queue_id      => 0,
+       :tasksubtype   => :worker,
+       :tasktype      => :status,
+       :name          => "name",
+       :hostname      => nil,
+       :worker_id     => nil,
+       :map_or_reduce => nil,
+       :process_id    => nil,
+       :started_at    => nil,
+       :job_id        => 1,
+       :iteration     => 0,
+       :processed     => nil,
+       :task_id       => 2,
+       :version       => 1
+    }.each do |key, val|
+       assert_equal val, @worker_message.send(key), key
+     end
+     assert_equal @worker_message.to_a, Skynet::WorkerStatusMessage.new([:status, :worker, nil, nil, nil, 1, 2, 0, "name", nil, nil, 1, nil, 0]).to_a
+  end              
+
+  def test_task_message
+    setup_task_message
+    {:payload=>"payload",
+     :queue_id=>0,
+     :tasktype=>:task,
+     :name=>"name",
+     :payload_type=>:task,
+     :expiry=>20,
+     :drburi=>"localhost",
+     :expire_time=>0,
+     :job_id=>1,
+     :iteration=>0,
+     :retry=>2,
+     :task_id=>2,
+     :version=>1}.each do |key, val|
+       assert_equal val, @task_message.send(key)
+     end
+     assert_equal @task_message.to_a, Skynet::Message.new([:task, "localhost", 2, 1, "payload", :task, "name", 20, 0, 0, 1, 2, 0]).to_a
+  end              
+  
+
+  def test_worker_status_template
+    template = Skynet::WorkerStatusMessage.worker_status_template(:hostname => "localhost", :process_id => $$)
+    [:status,:worker, nil, "localhost", :skip, nil, nil, nil, nil, nil, nil, nil, nil, nil].each_with_index do |field, ii|
+      next if field == :skip
+      assert_equal field, template[ii]
+    end
+  end
+  
+  def test_fallback_message
+    setup_task_message
+    fb = @task_message.fallback_task_message
+    test_fb = {:payload=>"payload",
+     :queue_id=>0,
+     :tasktype=>:task,
+     :name=>"name",
+     :payload_type=>:task,
+     :expiry=>20,
+     :drburi=>"localhost",
+     :expire_time=>1200599892,
+     :job_id=>1,
+     :iteration=>1,
+     :retry=>2,
+     :task_id=>2,
+     :version=>1}
+    
+    test_fb.each do |key, val| 
+      next if key == :expire_time
+      assert_equal val, fb.send(key)
+    end
+    
+  end
+  
+  def test_retry
+    setup_task_message
+    test_fb = {:payload=>"payload",
+     :queue_id=>0,
+     :tasktype=>:task,
+     :name=>"name",
+     :payload_type=>:task,
+     :expiry=>20,
+     :drburi=>"localhost",
+     :job_id=>1,
+     :retry=>2,
+     :task_id=>2,
+     :version=>1}
+     
+    fb1 = @task_message.fallback_task_message
+    test_fb.merge(:iteration => 1).each do |k,v| 
+      assert_equal v, fb1.send(k)
+    end
+
+    fb2 = fb1.fallback_task_message
+    test_fb.merge(:iteration => 2).each do |k,v| 
+      assert_equal v, fb2.send(k)
+    end
+
+    fb3 = fb2.fallback_task_message
+    test_fb.merge(:iteration => -1).each do |k,v| 
+      assert_equal v, fb3.send(k)
+    end
+  end
+  
+  def test_next_task_template
+    template = Skynet::Message.next_task_template(0,:any, 99)
+    [:task, nil, nil, nil, nil, :any, nil, nil, :skip, 0..6, 0, nil, 99].each_with_index do |field, ii|
+      next if field == :skip
+      assert_equal field, template[ii]
+    end
+  end      
+  
+  def test_result_template
+    template = Skynet::Message.result_template(88,:task)
+    [:task, nil, nil, 88, nil, nil, nil, nil, nil, nil, nil, nil, nil].each_with_index do |field, ii|
+      assert_equal field, template[ii]
+    end
+  end            
+  
+  def test_result_message
+    setup_task_message
+    result = @task_message.result_message("result")
+    {
+       :payload=>"result",
+       :queue_id=>0,
+       :tasktype=>:result,
+       :name=>"name",
+       :payload_type=>:result,
+       :expiry=>20,
+       :drburi=>"localhost",
+       :expire_time=>0,
+       :job_id=>1,
+       :iteration=>0,
+       :retry=>2,
+       :task_id=>2,
+       :version=>1
+     }.each do |key, val| 
+       assert_equal val, result.send(key)
+     end
+     
+  end
+  
+  def test_fallback_template
+    setup_task_message
+    template = @task_message.fallback_template
+    [:task, "localhost", 2, nil, nil, nil, nil, nil, nil, 1..6, 1, nil, 0].each_with_index do |field, ii|
+      assert_equal field, template[ii]
+    end
+  end
+  
+  def test_no_retry
+    setup_task_message
+    @task_message.retry = 0
+    fb1 = @task_message.fallback_task_message
+    {
+      :payload=>"payload",
+       :queue_id=>0,
+       :tasktype=>:task,
+       :name=>"name",
+       :payload_type=>:task,
+       :expiry=>20,
+       :drburi=>"localhost",
+       :job_id=>1,
+       :iteration=>-1,
+       :retry=>0,
+       :task_id=>2,
+       :version=>1
+     }.each do |k,v| 
+       assert_equal v, fb1.send(k)
+     end
+  end
+  
+  def test_payload
+    setup_task_message
+    @task_message.payload = "hi"
+    assert_equal YAML::dump("hi"), @task_message.raw_payload
+  end
+  
+  
+end

data/test/test_skynet_task.rb

@@ -0,0 +1,24 @@
+require File.dirname(__FILE__) + '/test_helper.rb'
+
+class SkynetTaskTest < Test::Unit::TestCase
+  
+  def setup
+    Skynet.configure(
+      :ENABLE                         => false,
+      :SKYNET_LOG_FILE                => STDOUT,
+      :SKYNET_LOG_LEVEL               => Logger::ERROR
+    )              
+  end      
+  
+  def test_master_task
+    job = Skynet::Job.new(:map_reduce_class => self.class, :async => true)
+    master_task = Skynet::Task.master_task(job)
+    master_job = Skynet::Job.new(master_task.process)
+    assert_equal self.class.to_s, master_job.map
+    assert ! master_job.async
+    assert_equal true, master_job.local_master
+  end
+
+  def self.map(datas)
+  end
+end

data/test/{tuplespace_message_queue_test.rb → test_tuplespace_message_queue.rb}

@@ -1,41 +1,36 @@
-ENV["RAILS_ENV"] = "test"
+require File.dirname(__FILE__) + '/test_helper.rb'
 
-require 'test/unit'
-require 'pp'        
-require '../lib/skynet.rb'
-
-
-# Tests for the partitioners.
-#
 class TuplespaceMessageQueueTest < Test::Unit::TestCase
   
   def setup
     Skynet.configure(
-      :ENABLE                         => false,
-      # :SKYNET_PIDS_FILE               => File.expand_path("#{RAILS_ROOT}/log/skynet_#{RAILS_ENV}.pids"),
-      :SKYNET_LOG_FILE                => STDOUT,
-      :SKYNET_LOG_LEVEL               => Logger::ERROR,
-      :SKYNET_LOCAL_MANAGER_URL       => "druby://localhost:40000",
-      :MESSAGE_QUEUE_ADAPTER          => "Skynet::MessageQueueAdapter::TupleSpace",
-      :TUPLESPACE_DRBURIS              => ["druby://localhost:77647"],
-      :USE_RINGSERVER                 => false
+      :ENABLE                   => false,
+      # :SKYNET_PIDS_FILE       => File.expand_path("#{RAILS_ROOT}/log/skynet_#{RAILS_ENV}.pids"),
+      :SKYNET_LOG_FILE          => STDOUT,
+      :SKYNET_LOG_LEVEL         => Logger::ERROR,
+      :SKYNET_LOCAL_MANAGER_URL => "druby://localhost:40000",
+      :MESSAGE_QUEUE_ADAPTER    => "Skynet::MessageQueueAdapter::TupleSpace",
+      :TS_DRBURIS               => ["druby://localhost:77647"],
+      :TS_USE_RINGSERVER        => false
     )      
     @ts ||= Rinda::TupleSpace.new
-    @@tss ||= DRb.start_service(Skynet::CONFIG[:TUPLESPACE_DRBURIS].first, @ts)
+    @@tss ||= DRb.start_service(Skynet::CONFIG[:TS_DRBURIS].first, @ts)
     mq.clear_outstanding_tasks
     
     @worker_message = Skynet::Message.new(
-      :tasktype=> :task, 
-      :drburi=> "localhost", 
-      :job_id => 1,
-      :task_id => 2,
-      :payload => "task",
+      :tasktype     => :task, 
+      :drburi       => "localhost", 
+      :job_id       => 1,
+      :task_id      => 2,
+      :payload      => "task",
       :payload_type => "master",
-      :expiry => 20, 
-      :expire_time => 0,
-      :iteration => 0,
-      :name => "test",       
-      :version => 1
+      :expiry       => 20, 
+      :expire_time  => 0,
+      :iteration    => 0,
+      :name         => "test",       
+      :version      => 1,
+      :retry        => 3,
+      :queue_id     => 0
     )
     
   end
@@ -44,7 +39,7 @@ class TuplespaceMessageQueueTest < Test::Unit::TestCase
     @ts.write([:test])
     assert_equal [:test], @ts.take([:test])
     
-    rts = DRbObject.new(nil, Skynet::CONFIG[:TUPLESPACE_DRBURIS].first)
+    rts = DRbObject.new(nil, Skynet::CONFIG[:TS_DRBURIS].first)
     rts.write([:test])
     assert_equal [:test], rts.take([:test])
   end
@@ -84,8 +79,8 @@ class TuplespaceMessageQueueTest < Test::Unit::TestCase
     message.expiry=0.4
     assert mq.write_message(message)
     assert_equal message.to_a, mq.take_next_task(1).to_a
-    sleep 0.6
-    ntt = Skynet::Message.next_task_template(1)
+    sleep 1
+    ntt = Skynet::Message.next_task_template(1,nil,0)
     assert_equal 2, mq.take_next_task(1,0.00001).task_id
   end