timocratic-skynet 0.9.4

data/test/test_skynet_manager.rb

@@ -0,0 +1,157 @@
+require File.dirname(__FILE__) + '/test_helper.rb'
+require 'functor'
+
+class SkynetManagerTest < Test::Unit::TestCase
+
+  PIDS = []
+  PIDFILE = []
+  
+  def setup
+    connection = ActiveRecord::Base.establish_connection(
+        :adapter  => "mysql",
+        :host     => "localhost",
+        :username => "root",
+        :password => "",
+        :database => "skynet_test"
+      )
+      
+    ActiveRecord::Base.connection.reconnect!
+     
+    Skynet.configure(
+      :ENABLE                         => false,
+      # :SKYNET_PIDS_FILE               => File.expand_path("#{RAILS_ROOT}/log/skynet_#{RAILS_ENV}.pids"),
+      :SKYNET_LOG_FILE                => STDOUT,
+      :SKYNET_LOG_LEVEL               => 4,
+      :MESSAGE_QUEUE_ADAPTER          => "Skynet::MessageQueueAdapter::Mysql",
+      :MYSQL_TEMPERATURE_CHANGE_SLEEP => 1,
+      :MYSQL_NEXT_TASK_TIMEOUT        => 1,
+      :MYSQL_QUEUE_DATABASE           => nil
+    )      
+
+    mq.clear_outstanding_tasks
+    mq.clear_worker_status
+
+    @pids = []
+    @pidfile = []
+    @worker_queue = []
+    
+    PIDS.delete_if {true}
+    PIDFILE.delete_if {true}
+
+    Process.stubs(:detatch).returns(true)    
+    Process.stubs(:kill).returns(true)
+
+    @minstance = Skynet::Manager.any_instance
+    @hostname = Socket.gethostname
+   
+    Skynet::Worker.any_instance.stubs(:max_memory_reached?).returns(false)
+  end                                                                     
+  
+  def setup_manager
+    @manager = Skynet::Manager.new(:script_path => "path", :workers => 2)
+    @manager.extend(Functor)
+
+    mq = functor
+    @manager.stubs(:mq).returns(mq)
+
+    mq.get_worker_version = 1
+    
+    worker_queue = @worker_queue
+    mq.read_all_worker_statuses = lambda do |hostname|
+      worker_queue
+    end
+
+    mq.take_worker_status = lambda do |options, timeout|
+      worker_queue.delete_if {|w|w.process_id == options[:process_id] }.first
+    end
+    
+    pids     = @pids                   
+    hostname = @hostname
+    Skynet.extend(Functor)
+    Skynet.define_method(:close_files) do
+      true
+    end
+
+    Skynet.define_method(:close_console) do
+      true
+    end
+
+    
+    Skynet.define_method(:fork_and_exec) do |cmd|
+      newpid = pids.size + 1
+      pids << newpid
+      worker_info = {
+        :hostname     => hostname,
+        :process_id   => newpid,
+        :worker_type  => :task, 
+        :worker_id    => newpid,
+        :version      => 1
+      }
+            
+      worker_queue << Skynet::WorkerStatusMessage.new(worker_info)
+      newpid
+    end
+  end
+  
+  def test_manager_start  
+    setup_manager
+    @manager.start_workers
+    assert_equal 2, @manager.worker_pids.size
+    assert_equal @pids.sort, @manager.worker_pids.sort
+    assert_equal @pids.sort, @manager.worker_queue.collect {|q|q.process_id}.sort
+  end
+
+  def test_check_workers
+    setup_manager
+    Skynet::Manager.any_instance.expects(:worker_alive?).times(2).returns(true)
+    @manager.start_workers
+    @manager.check_workers
+    assert_equal 2, @manager.worker_pids.size
+    assert_equal @pids.sort, @manager.worker_pids.sort
+    assert_equal @pids.sort, @manager.worker_queue.collect {|q|q.process_id}.sort
+  end
+  
+  def test_running_pids
+    setup_manager
+    Skynet::Manager.any_instance.expects(:worker_alive?).with(1).returns(true)
+    Skynet::Manager.any_instance.expects(:worker_alive?).with(2).returns(false)
+    @manager.start_workers
+    @manager.check_workers
+    assert_equal 1, @manager.worker_pids.size
+    assert_equal [1], @manager.worker_pids.sort
+    assert_equal [1], @manager.worker_queue.collect {|q|q.process_id}.sort
+  end
+
+  ## XXX FIXME.  What happens if there's a worker missing from the pidfile, but was running?
+  def test_more_in_pidfile_than_queue_alive
+    setup_manager
+    @manager.expects(:worker_alive?).with(1).returns(false)
+    Skynet.configure(:SKYNET_LOG_LEVEL => 4) do
+      @manager.start_workers
+
+      @manager.expects(:worker_alive?).with(2).returns(true)
+      @worker_queue.delete_if {|w| w.process_id == 1}
+      @manager.check_workers
+    end
+    assert_equal 2, @manager.worker_pids.size
+    assert_equal [2,3], @manager.worker_pids.sort
+    assert_equal [2,3], @manager.worker_queue.collect {|q|q.process_id}.sort
+  end
+
+  def test_dead_workers
+    setup_manager
+    Skynet::Manager.any_instance.expects(:worker_alive?).times(1).with(1).returns(false)
+    # Skynet::Manager.any_instance.expects(:worker_alive?).with(2).returns(true)
+    @manager.start_workers     
+    @manager.check_workers
+    assert_equal [2], @manager.worker_pids.sort
+    assert_equal [2], @manager.worker_queue.collect {|q|q.process_id}.sort
+  end
+
+
+  private
+
+  def mq
+		Skynet::MessageQueueAdapter::Mysql.new
+  end
+end

data/test/test_skynet_message.rb

@@ -0,0 +1,229 @@
+require File.dirname(__FILE__) + '/test_helper.rb'
+
+class MysqlMessageQueueTest < Test::Unit::TestCase
+  
+  def setup
+    Skynet.configure(
+      :ENABLE                         => false,
+      :SKYNET_LOG_FILE                => STDOUT,
+      :SKYNET_LOG_LEVEL               => Logger::ERROR
+    )              
+  end      
+
+  def setup_worker_message
+    @worker_message = Skynet::WorkerStatusMessage.new(
+      :tasktype     => :task, 
+      :drburi       => "localhost", 
+      :job_id       => 1,
+      :task_id      => 2,
+      :payload      => "payload",
+      :payload_type => "task",
+      :expiry       => 20, 
+      :expire_time  => 1095108406.9251,
+      :iteration    => 0,
+      :name         => "name",       
+      :version      => 1,
+      :queue_id     => 0
+    )
+  end    
+
+  def setup_task_message
+    @task_message = Skynet::Message.new(
+      :tasktype     => :task, 
+      :drburi       => "localhost", 
+      :job_id       => 1,
+      :task_id      => 2,
+      :payload      => "payload",
+      :payload_type => "task",
+      :expiry       => 20, 
+      :expire_time  => 0,
+      :iteration    => 0,
+      :name         => "name",       
+      :version      => 1,
+      :retry        => 2,
+      :queue_id     => 0
+    )     
+  end    
+  
+  def test_worker_status_message
+    setup_worker_message
+    {
+       :queue_id      => 0,
+       :tasksubtype   => :worker,
+       :tasktype      => :status,
+       :name          => "name",
+       :hostname      => nil,
+       :worker_id     => nil,
+       :map_or_reduce => nil,
+       :process_id    => nil,
+       :started_at    => nil,
+       :job_id        => 1,
+       :iteration     => 0,
+       :processed     => nil,
+       :task_id       => 2,
+       :version       => 1
+    }.each do |key, val|
+       assert_equal val, @worker_message.send(key), key
+     end
+     assert_equal @worker_message.to_a, Skynet::WorkerStatusMessage.new([:status, :worker, nil, nil, nil, 1, 2, 0, "name", nil, nil, 1, nil, 0]).to_a
+  end              
+
+  def test_task_message
+    setup_task_message
+    {:payload=>"payload",
+     :queue_id=>0,
+     :tasktype=>:task,
+     :name=>"name",
+     :payload_type=>:task,
+     :expiry=>20,
+     :drburi=>"localhost",
+     :expire_time=>0,
+     :job_id=>1,
+     :iteration=>0,
+     :retry=>2,
+     :task_id=>2,
+     :version=>1}.each do |key, val|
+       assert_equal val, @task_message.send(key)
+     end
+     assert_equal @task_message.to_a, Skynet::Message.new([:task, "localhost", 2, 1, "payload", :task, "name", 20, 0, 0, 1, 2, 0]).to_a
+  end              
+  
+
+  def test_worker_status_template
+    template = Skynet::WorkerStatusMessage.worker_status_template(:hostname => "localhost", :process_id => $$)
+    [:status,:worker, nil, "localhost", :skip, nil, nil, nil, nil, nil, nil, nil, nil, nil].each_with_index do |field, ii|
+      next if field == :skip
+      assert_equal field, template[ii]
+    end
+  end
+  
+  def test_fallback_message
+    setup_task_message
+    fb = @task_message.fallback_task_message
+    test_fb = {:payload=>"payload",
+     :queue_id=>0,
+     :tasktype=>:task,
+     :name=>"name",
+     :payload_type=>:task,
+     :expiry=>20,
+     :drburi=>"localhost",
+     :expire_time=>1200599892,
+     :job_id=>1,
+     :iteration=>1,
+     :retry=>2,
+     :task_id=>2,
+     :version=>1}
+    
+    test_fb.each do |key, val| 
+      next if key == :expire_time
+      assert_equal val, fb.send(key)
+    end
+    
+  end
+  
+  def test_retry
+    setup_task_message
+    test_fb = {:payload=>"payload",
+     :queue_id=>0,
+     :tasktype=>:task,
+     :name=>"name",
+     :payload_type=>:task,
+     :expiry=>20,
+     :drburi=>"localhost",
+     :job_id=>1,
+     :retry=>2,
+     :task_id=>2,
+     :version=>1}
+     
+    fb1 = @task_message.fallback_task_message
+    test_fb.merge(:iteration => 1).each do |k,v| 
+      assert_equal v, fb1.send(k)
+    end
+
+    fb2 = fb1.fallback_task_message
+    test_fb.merge(:iteration => 2).each do |k,v| 
+      assert_equal v, fb2.send(k)
+    end
+
+    fb3 = fb2.fallback_task_message
+    test_fb.merge(:iteration => -1).each do |k,v| 
+      assert_equal v, fb3.send(k)
+    end
+  end
+  
+  def test_next_task_template
+    template = Skynet::Message.next_task_template(0,:any, 99)
+    [:task, nil, nil, nil, nil, :any, nil, nil, :skip, 0..6, 0, nil, 99].each_with_index do |field, ii|
+      next if field == :skip
+      assert_equal field, template[ii]
+    end
+  end      
+  
+  def test_result_template
+    template = Skynet::Message.result_template(88,:task)
+    [:task, nil, nil, 88, nil, nil, nil, nil, nil, nil, nil, nil, nil].each_with_index do |field, ii|
+      assert_equal field, template[ii]
+    end
+  end            
+  
+  def test_result_message
+    setup_task_message
+    result = @task_message.result_message("result")
+    {
+       :payload=>"result",
+       :queue_id=>0,
+       :tasktype=>:result,
+       :name=>"name",
+       :payload_type=>:result,
+       :expiry=>20,
+       :drburi=>"localhost",
+       :expire_time=>0,
+       :job_id=>1,
+       :iteration=>0,
+       :retry=>2,
+       :task_id=>2,
+       :version=>1
+     }.each do |key, val| 
+       assert_equal val, result.send(key)
+     end
+     
+  end
+  
+  def test_fallback_template
+    setup_task_message
+    template = @task_message.fallback_template
+    [:task, "localhost", 2, nil, nil, nil, nil, nil, nil, 1..6, 1, nil, 0].each_with_index do |field, ii|
+      assert_equal field, template[ii]
+    end
+  end
+  
+  def test_no_retry
+    setup_task_message
+    @task_message.retry = 0
+    fb1 = @task_message.fallback_task_message
+    {
+      :payload=>"payload",
+       :queue_id=>0,
+       :tasktype=>:task,
+       :name=>"name",
+       :payload_type=>:task,
+       :expiry=>20,
+       :drburi=>"localhost",
+       :job_id=>1,
+       :iteration=>-1,
+       :retry=>0,
+       :task_id=>2,
+       :version=>1
+     }.each do |k,v| 
+       assert_equal v, fb1.send(k)
+     end
+  end
+  
+  def test_payload
+    setup_task_message
+    @task_message.payload = "hi"
+    assert_equal YAML::dump("hi"), @task_message.raw_payload
+  end
+  
+  
+end

data/test/test_skynet_task.rb

@@ -0,0 +1,24 @@
+require File.dirname(__FILE__) + '/test_helper.rb'
+
+class SkynetTaskTest < Test::Unit::TestCase
+  
+  def setup
+    Skynet.configure(
+      :ENABLE                         => false,
+      :SKYNET_LOG_FILE                => STDOUT,
+      :SKYNET_LOG_LEVEL               => Logger::ERROR
+    )              
+  end      
+  
+  def test_master_task
+    job = Skynet::Job.new(:map_reduce_class => self.class, :async => true, :local_master => false, :solo => true)
+    master_task = Skynet::Task.master_task(job)
+    master_job = Skynet::Job.new(master_task.process)
+    assert_equal self.class.to_s, master_job.map
+    assert ! master_job.async
+    assert_equal true, master_job.local_master
+  end
+
+  def self.map(datas)
+  end
+end

data/test/test_tuplespace_message_queue.rb

@@ -0,0 +1,174 @@
+require File.dirname(__FILE__) + '/test_helper.rb'
+
+class TuplespaceMessageQueueTest < Test::Unit::TestCase
+  
+  def setup
+    Skynet.configure(
+      :ENABLE                   => false,
+      # :SKYNET_PIDS_FILE       => File.expand_path("#{RAILS_ROOT}/log/skynet_#{RAILS_ENV}.pids"),
+      :SKYNET_LOG_FILE          => STDOUT,
+      :SKYNET_LOG_LEVEL         => Logger::ERROR,
+      :SKYNET_LOCAL_MANAGER_URL => "druby://localhost:40000",
+      :MESSAGE_QUEUE_ADAPTER    => "Skynet::MessageQueueAdapter::TupleSpace",
+      :TS_DRBURIS               => ["druby://localhost:77647"],
+      :TS_USE_RINGSERVER        => false
+    )      
+    @ts ||= Rinda::TupleSpace.new
+    @@tss ||= DRb.start_service(Skynet::CONFIG[:TS_DRBURIS].first, @ts)
+    mq.clear_outstanding_tasks
+    
+    @worker_message = Skynet::Message.new(
+      :tasktype     => :task, 
+      :drburi       => "localhost", 
+      :job_id       => 1,
+      :task_id      => 2,
+      :payload      => "task",
+      :payload_type => "master",
+      :expiry       => 20, 
+      :expire_time  => 0,
+      :iteration    => 0,
+      :name         => "test",       
+      :version      => 1,
+      :retry        => 3,
+      :queue_id     => 0
+    )
+    
+  end
+                   
+  def test_tuple_space
+    @ts.write([:test])
+    assert_equal [:test], @ts.take([:test])
+    
+    rts = DRbObject.new(nil, Skynet::CONFIG[:TS_DRBURIS].first)
+    rts.write([:test])
+    assert_equal [:test], rts.take([:test])
+  end
+
+  def test_tuplespace_ranges
+    tuple = [:test,10]
+    @ts.write(tuple)
+    assert_equal tuple, @ts.take([:test,0...11],0.00001)
+    write_tuple = [:task,"localhost", 2,1,"task","master","test",0.001,1195078587.001,1,1]
+    read_tuple = [:task, nil, nil, nil, nil, nil, nil, nil, 0...1195078588, 0..6, 1]
+    assert_equal write_tuple.size, read_tuple.size
+    @ts.write(write_tuple)
+    assert_equal write_tuple, @ts.take(read_tuple,0.00001)    
+    
+    write_tuple = [:task, "localhost", 2, 1, "task", "master", "test", 0.001, 1195079557.001, 1, 1]
+    read_tuple = [:task, nil, nil, nil, nil, nil, nil, nil, 0..1195079558, 0..6, 1]
+    assert_equal write_tuple.size, read_tuple.size
+    @ts.write(write_tuple)
+    assert_equal write_tuple, @ts.take(read_tuple,0.00001)        
+  end
+
+### THE FOLLOWING TESTS THE MessageQueueProxy API
+  
+  def test_write_message    
+    assert mq.write_message(@worker_message,10)
+    assert_equal 1, mq.list_tasks.size
+  end
+
+  def test_take_next_task
+    assert mq.write_message(@worker_message,10)
+    assert_equal @worker_message.to_a, mq.take_next_task(1).to_a
+  end
+
+  
+  def test_task_failover
+    message = @worker_message.clone
+    message.expiry=0.4
+    assert mq.write_message(message)
+    assert_equal message.to_a, mq.take_next_task(1).to_a
+    sleep 1
+    ntt = Skynet::Message.next_task_template(1,nil,0)
+    assert_equal 2, mq.take_next_task(1,0.00001).task_id
+  end
+
+  def test_write_and_take_result
+    assert mq.write_message(@worker_message,10)  
+    message = mq.take_next_task(1)
+    set_result = {:blah => ['hi']}
+    
+    mq.write_result(message,set_result,20)
+    assert mq.list_tasks.empty?
+    assert_equal 1, mq.list_results.size
+    result = mq.take_result(1)
+    assert_equal set_result, result.payload
+  end
+  
+  def test_write_error
+    mq.write_message(@worker_message,10)  
+    message = mq.take_next_task(1)
+    mq.write_error(message,"something_bad_happened",10)
+    result = mq.take_result(1)          
+    assert_equal "something_bad_happened", result.payload
+  end
+
+  def test_write_complex_error
+    mq.write_message(@worker_message,10)  
+    message = mq.take_next_task(1)
+    error = {:error => "something_bad_happened"}
+    mq.write_error(message,error,10)
+    result = mq.take_result(1)          
+    assert_equal error, result.payload
+  end                      
+  
+  def test_write_worker_status
+    assert mq.write_worker_status({
+      :worker_id  => 5,
+      :hostname   => 'localhost',
+      :process_id => $$,
+      :name       => "waiting for master or tasks",
+      :processed  => 0,
+      :version    => 1,
+      :started_at => Time.now.to_i
+    })        
+  end
+  
+  def test_take_worker_status
+    assert mq.write_worker_status({
+      :worker_id  => 5,
+      :hostname   => 'localhost',
+      :process_id => $$,
+      :name       => "waiting for master or tasks",
+      :processed  => 0,
+      :version    => 1,
+      :started_at => Time.now.to_i
+    })        
+    
+    status = mq.take_worker_status({:hostname   => 'localhost',:process_id => $$ })
+    assert_equal $$, status.process_id        
+  end
+  
+  def test_read_all_worker_statuses
+    assert mq.write_worker_status({
+      :worker_id  => 5,
+      :hostname   => 'localhost',
+      :process_id => $$,
+      :name       => "waiting for master or tasks",
+      :processed  => 0,
+      :version    => 1,
+      :started_at => Time.now.to_i
+    })        
+    status = mq.read_all_worker_statuses.first
+
+    # status = mq.take_worker_status({:hostname   => 'localhost',:process_id => $$ })
+    assert_equal $$, status.process_id        
+  end
+
+  def test_worker_version
+    mq.set_worker_version(2)
+    assert_equal 2, mq.get_worker_version
+    mq.set_worker_version(10)
+    assert_equal 10, mq.get_worker_version
+  end
+  
+
+  
+
+private
+  def mq
+		Skynet::MessageQueueAdapter::TupleSpace.new
+  end
+    
+end