brendan-skynet 0.9.33 → 0.9.303

data/test/test_skynet_manager.rb

@@ -1,157 +0,0 @@
-require File.dirname(__FILE__) + '/test_helper.rb'
-require 'functor'
-
-class SkynetManagerTest < Test::Unit::TestCase
-
-  PIDS = []
-  PIDFILE = []
-  
-  def setup
-    connection = ActiveRecord::Base.establish_connection(
-        :adapter  => "mysql",
-        :host     => "localhost",
-        :username => "root",
-        :password => "",
-        :database => "skynet_test"
-      )
-      
-    ActiveRecord::Base.connection.reconnect!
-     
-    Skynet.configure(
-      :ENABLE                         => false,
-      # :SKYNET_PIDS_FILE               => File.expand_path("#{RAILS_ROOT}/log/skynet_#{RAILS_ENV}.pids"),
-      :SKYNET_LOG_FILE                => STDOUT,
-      :SKYNET_LOG_LEVEL               => 4,
-      :MESSAGE_QUEUE_ADAPTER          => "Skynet::MessageQueueAdapter::Mysql",
-      :MYSQL_TEMPERATURE_CHANGE_SLEEP => 1,
-      :MYSQL_NEXT_TASK_TIMEOUT        => 1,
-      :MYSQL_QUEUE_DATABASE           => nil
-    )      
-
-    mq.clear_outstanding_tasks
-    mq.clear_worker_status
-
-    @pids = []
-    @pidfile = []
-    @worker_queue = []
-    
-    PIDS.delete_if {true}
-    PIDFILE.delete_if {true}
-
-    Process.stubs(:detatch).returns(true)    
-    Process.stubs(:kill).returns(true)
-
-    @minstance = Skynet::Manager.any_instance
-    @hostname = Socket.gethostname
-   
-    Skynet::Worker.any_instance.stubs(:max_memory_reached?).returns(false)
-  end                                                                     
-  
-  def setup_manager
-    @manager = Skynet::Manager.new(:script_path => "path", :workers => 2)
-    @manager.extend(Functor)
-
-    mq = functor
-    @manager.stubs(:mq).returns(mq)
-
-    mq.get_worker_version = 1
-    
-    worker_queue = @worker_queue
-    mq.read_all_worker_statuses = lambda do |hostname|
-      worker_queue
-    end
-
-    mq.take_worker_status = lambda do |options, timeout|
-      worker_queue.delete_if {|w|w.process_id == options[:process_id] }.first
-    end
-    
-    pids     = @pids                   
-    hostname = @hostname
-    Skynet.extend(Functor)
-    Skynet.define_method(:close_files) do
-      true
-    end
-
-    Skynet.define_method(:close_console) do
-      true
-    end
-
-    
-    Skynet.define_method(:fork_and_exec) do |cmd|
-      newpid = pids.size + 1
-      pids << newpid
-      worker_info = {
-        :hostname     => hostname,
-        :process_id   => newpid,
-        :worker_type  => :task, 
-        :worker_id    => newpid,
-        :version      => 1
-      }
-            
-      worker_queue << Skynet::WorkerStatusMessage.new(worker_info)
-      newpid
-    end
-  end
-  
-  def test_manager_start  
-    setup_manager
-    @manager.start_workers
-    assert_equal 2, @manager.worker_pids.size
-    assert_equal @pids.sort, @manager.worker_pids.sort
-    assert_equal @pids.sort, @manager.worker_queue.collect {|q|q.process_id}.sort
-  end
-
-  def test_check_workers
-    setup_manager
-    Skynet::Manager.any_instance.expects(:worker_alive?).times(2).returns(true)
-    @manager.start_workers
-    @manager.check_workers
-    assert_equal 2, @manager.worker_pids.size
-    assert_equal @pids.sort, @manager.worker_pids.sort
-    assert_equal @pids.sort, @manager.worker_queue.collect {|q|q.process_id}.sort
-  end
-  
-  def test_running_pids
-    setup_manager
-    Skynet::Manager.any_instance.expects(:worker_alive?).with(1).returns(true)
-    Skynet::Manager.any_instance.expects(:worker_alive?).with(2).returns(false)
-    @manager.start_workers
-    @manager.check_workers
-    assert_equal 1, @manager.worker_pids.size
-    assert_equal [1], @manager.worker_pids.sort
-    assert_equal [1], @manager.worker_queue.collect {|q|q.process_id}.sort
-  end
-
-  ## XXX FIXME.  What happens if there's a worker missing from the pidfile, but was running?
-  def test_more_in_pidfile_than_queue_alive
-    setup_manager
-    @manager.expects(:worker_alive?).with(1).returns(false)
-    Skynet.configure(:SKYNET_LOG_LEVEL => 4) do
-      @manager.start_workers
-
-      @manager.expects(:worker_alive?).with(2).returns(true)
-      @worker_queue.delete_if {|w| w.process_id == 1}
-      @manager.check_workers
-    end
-    assert_equal 2, @manager.worker_pids.size
-    assert_equal [2,3], @manager.worker_pids.sort
-    assert_equal [2,3], @manager.worker_queue.collect {|q|q.process_id}.sort
-  end
-
-  def test_dead_workers
-    setup_manager
-    Skynet::Manager.any_instance.expects(:worker_alive?).times(1).with(1).returns(false)
-    # Skynet::Manager.any_instance.expects(:worker_alive?).with(2).returns(true)
-    @manager.start_workers     
-    @manager.check_workers
-    assert_equal [2], @manager.worker_pids.sort
-    assert_equal [2], @manager.worker_queue.collect {|q|q.process_id}.sort
-  end
-
-
-  private
-
-  def mq
-		Skynet::MessageQueueAdapter::Mysql.new
-  end
-end

data/test/test_skynet_message.rb

@@ -1,229 +0,0 @@
-require File.dirname(__FILE__) + '/test_helper.rb'
-
-class MysqlMessageQueueTest < Test::Unit::TestCase
-  
-  def setup
-    Skynet.configure(
-      :ENABLE                         => false,
-      :SKYNET_LOG_FILE                => STDOUT,
-      :SKYNET_LOG_LEVEL               => Logger::ERROR
-    )              
-  end      
-
-  def setup_worker_message
-    @worker_message = Skynet::WorkerStatusMessage.new(
-      :tasktype     => :task, 
-      :drburi       => "localhost", 
-      :job_id       => 1,
-      :task_id      => 2,
-      :payload      => "payload",
-      :payload_type => "task",
-      :expiry       => 20, 
-      :expire_time  => 1095108406.9251,
-      :iteration    => 0,
-      :name         => "name",       
-      :version      => 1,
-      :queue_id     => 0
-    )
-  end    
-
-  def setup_task_message
-    @task_message = Skynet::Message.new(
-      :tasktype     => :task, 
-      :drburi       => "localhost", 
-      :job_id       => 1,
-      :task_id      => 2,
-      :payload      => "payload",
-      :payload_type => "task",
-      :expiry       => 20, 
-      :expire_time  => 0,
-      :iteration    => 0,
-      :name         => "name",       
-      :version      => 1,
-      :retry        => 2,
-      :queue_id     => 0
-    )     
-  end    
-  
-  def test_worker_status_message
-    setup_worker_message
-    {
-       :queue_id      => 0,
-       :tasksubtype   => :worker,
-       :tasktype      => :status,
-       :name          => "name",
-       :hostname      => nil,
-       :worker_id     => nil,
-       :map_or_reduce => nil,
-       :process_id    => nil,
-       :started_at    => nil,
-       :job_id        => 1,
-       :iteration     => 0,
-       :processed     => nil,
-       :task_id       => 2,
-       :version       => 1
-    }.each do |key, val|
-       assert_equal val, @worker_message.send(key), key
-     end
-     assert_equal @worker_message.to_a, Skynet::WorkerStatusMessage.new([:status, :worker, nil, nil, nil, 1, 2, 0, "name", nil, nil, 1, nil, 0]).to_a
-  end              
-
-  def test_task_message
-    setup_task_message
-    {:payload=>"payload",
-     :queue_id=>0,
-     :tasktype=>:task,
-     :name=>"name",
-     :payload_type=>:task,
-     :expiry=>20,
-     :drburi=>"localhost",
-     :expire_time=>0,
-     :job_id=>1,
-     :iteration=>0,
-     :retry=>2,
-     :task_id=>2,
-     :version=>1}.each do |key, val|
-       assert_equal val, @task_message.send(key)
-     end
-     assert_equal @task_message.to_a, Skynet::Message.new([:task, "localhost", 2, 1, "payload", :task, "name", 20, 0, 0, 1, 2, 0]).to_a
-  end              
-  
-
-  def test_worker_status_template
-    template = Skynet::WorkerStatusMessage.worker_status_template(:hostname => "localhost", :process_id => $$)
-    [:status,:worker, nil, "localhost", :skip, nil, nil, nil, nil, nil, nil, nil, nil, nil].each_with_index do |field, ii|
-      next if field == :skip
-      assert_equal field, template[ii]
-    end
-  end
-  
-  def test_fallback_message
-    setup_task_message
-    fb = @task_message.fallback_task_message
-    test_fb = {:payload=>"payload",
-     :queue_id=>0,
-     :tasktype=>:task,
-     :name=>"name",
-     :payload_type=>:task,
-     :expiry=>20,
-     :drburi=>"localhost",
-     :expire_time=>1200599892,
-     :job_id=>1,
-     :iteration=>1,
-     :retry=>2,
-     :task_id=>2,
-     :version=>1}
-    
-    test_fb.each do |key, val| 
-      next if key == :expire_time
-      assert_equal val, fb.send(key)
-    end
-    
-  end
-  
-  def test_retry
-    setup_task_message
-    test_fb = {:payload=>"payload",
-     :queue_id=>0,
-     :tasktype=>:task,
-     :name=>"name",
-     :payload_type=>:task,
-     :expiry=>20,
-     :drburi=>"localhost",
-     :job_id=>1,
-     :retry=>2,
-     :task_id=>2,
-     :version=>1}
-     
-    fb1 = @task_message.fallback_task_message
-    test_fb.merge(:iteration => 1).each do |k,v| 
-      assert_equal v, fb1.send(k)
-    end
-
-    fb2 = fb1.fallback_task_message
-    test_fb.merge(:iteration => 2).each do |k,v| 
-      assert_equal v, fb2.send(k)
-    end
-
-    fb3 = fb2.fallback_task_message
-    test_fb.merge(:iteration => -1).each do |k,v| 
-      assert_equal v, fb3.send(k)
-    end
-  end
-  
-  def test_next_task_template
-    template = Skynet::Message.next_task_template(0,:any, 99)
-    [:task, nil, nil, nil, nil, :any, nil, nil, :skip, 0..6, 0, nil, 99].each_with_index do |field, ii|
-      next if field == :skip
-      assert_equal field, template[ii]
-    end
-  end      
-  
-  def test_result_template
-    template = Skynet::Message.result_template(88,:task)
-    [:task, nil, nil, 88, nil, nil, nil, nil, nil, nil, nil, nil, nil].each_with_index do |field, ii|
-      assert_equal field, template[ii]
-    end
-  end            
-  
-  def test_result_message
-    setup_task_message
-    result = @task_message.result_message("result")
-    {
-       :payload=>"result",
-       :queue_id=>0,
-       :tasktype=>:result,
-       :name=>"name",
-       :payload_type=>:result,
-       :expiry=>20,
-       :drburi=>"localhost",
-       :expire_time=>0,
-       :job_id=>1,
-       :iteration=>0,
-       :retry=>2,
-       :task_id=>2,
-       :version=>1
-     }.each do |key, val| 
-       assert_equal val, result.send(key)
-     end
-     
-  end
-  
-  def test_fallback_template
-    setup_task_message
-    template = @task_message.fallback_template
-    [:task, "localhost", 2, nil, nil, nil, nil, nil, nil, 1..6, 1, nil, 0].each_with_index do |field, ii|
-      assert_equal field, template[ii]
-    end
-  end
-  
-  def test_no_retry
-    setup_task_message
-    @task_message.retry = 0
-    fb1 = @task_message.fallback_task_message
-    {
-      :payload=>"payload",
-       :queue_id=>0,
-       :tasktype=>:task,
-       :name=>"name",
-       :payload_type=>:task,
-       :expiry=>20,
-       :drburi=>"localhost",
-       :job_id=>1,
-       :iteration=>-1,
-       :retry=>0,
-       :task_id=>2,
-       :version=>1
-     }.each do |k,v| 
-       assert_equal v, fb1.send(k)
-     end
-  end
-  
-  def test_payload
-    setup_task_message
-    @task_message.payload = "hi"
-    assert_equal YAML::dump("hi"), @task_message.raw_payload
-  end
-  
-  
-end

data/test/test_skynet_task.rb

@@ -1,24 +0,0 @@
-require File.dirname(__FILE__) + '/test_helper.rb'
-
-class SkynetTaskTest < Test::Unit::TestCase
-  
-  def setup
-    Skynet.configure(
-      :ENABLE                         => false,
-      :SKYNET_LOG_FILE                => STDOUT,
-      :SKYNET_LOG_LEVEL               => Logger::ERROR
-    )              
-  end      
-  
-  def test_master_task
-    job = Skynet::Job.new(:map_reduce_class => self.class, :async => true, :local_master => false, :solo => true)
-    master_task = Skynet::Task.master_task(job)
-    master_job = Skynet::Job.new(master_task.process)
-    assert_equal self.class.to_s, master_job.map
-    assert ! master_job.async
-    assert_equal true, master_job.local_master
-  end
-
-  def self.map(datas)
-  end
-end

data/test/test_tuplespace_message_queue.rb

@@ -1,174 +0,0 @@
-require File.dirname(__FILE__) + '/test_helper.rb'
-
-class TuplespaceMessageQueueTest < Test::Unit::TestCase
-  
-  def setup
-    Skynet.configure(
-      :ENABLE                   => false,
-      # :SKYNET_PIDS_FILE       => File.expand_path("#{RAILS_ROOT}/log/skynet_#{RAILS_ENV}.pids"),
-      :SKYNET_LOG_FILE          => STDOUT,
-      :SKYNET_LOG_LEVEL         => Logger::ERROR,
-      :SKYNET_LOCAL_MANAGER_URL => "druby://localhost:40000",
-      :MESSAGE_QUEUE_ADAPTER    => "Skynet::MessageQueueAdapter::TupleSpace",
-      :TS_DRBURIS               => ["druby://localhost:77647"],
-      :TS_USE_RINGSERVER        => false
-    )      
-    @ts ||= Rinda::TupleSpace.new
-    @@tss ||= DRb.start_service(Skynet::CONFIG[:TS_DRBURIS].first, @ts)
-    mq.clear_outstanding_tasks
-    
-    @worker_message = Skynet::Message.new(
-      :tasktype     => :task, 
-      :drburi       => "localhost", 
-      :job_id       => 1,
-      :task_id      => 2,
-      :payload      => "task",
-      :payload_type => "master",
-      :expiry       => 20, 
-      :expire_time  => 0,
-      :iteration    => 0,
-      :name         => "test",       
-      :version      => 1,
-      :retry        => 3,
-      :queue_id     => 0
-    )
-    
-  end
-                   
-  def test_tuple_space
-    @ts.write([:test])
-    assert_equal [:test], @ts.take([:test])
-    
-    rts = DRbObject.new(nil, Skynet::CONFIG[:TS_DRBURIS].first)
-    rts.write([:test])
-    assert_equal [:test], rts.take([:test])
-  end
-
-  def test_tuplespace_ranges
-    tuple = [:test,10]
-    @ts.write(tuple)
-    assert_equal tuple, @ts.take([:test,0...11],0.00001)
-    write_tuple = [:task,"localhost", 2,1,"task","master","test",0.001,1195078587.001,1,1]
-    read_tuple = [:task, nil, nil, nil, nil, nil, nil, nil, 0...1195078588, 0..6, 1]
-    assert_equal write_tuple.size, read_tuple.size
-    @ts.write(write_tuple)
-    assert_equal write_tuple, @ts.take(read_tuple,0.00001)    
-    
-    write_tuple = [:task, "localhost", 2, 1, "task", "master", "test", 0.001, 1195079557.001, 1, 1]
-    read_tuple = [:task, nil, nil, nil, nil, nil, nil, nil, 0..1195079558, 0..6, 1]
-    assert_equal write_tuple.size, read_tuple.size
-    @ts.write(write_tuple)
-    assert_equal write_tuple, @ts.take(read_tuple,0.00001)        
-  end
-
-### THE FOLLOWING TESTS THE MessageQueueProxy API
-  
-  def test_write_message    
-    assert mq.write_message(@worker_message,10)
-    assert_equal 1, mq.list_tasks.size
-  end
-
-  def test_take_next_task
-    assert mq.write_message(@worker_message,10)
-    assert_equal @worker_message.to_a, mq.take_next_task(1).to_a
-  end
-
-  
-  def test_task_failover
-    message = @worker_message.clone
-    message.expiry=0.4
-    assert mq.write_message(message)
-    assert_equal message.to_a, mq.take_next_task(1).to_a
-    sleep 1
-    ntt = Skynet::Message.next_task_template(1,nil,0)
-    assert_equal 2, mq.take_next_task(1,0.00001).task_id
-  end
-
-  def test_write_and_take_result
-    assert mq.write_message(@worker_message,10)  
-    message = mq.take_next_task(1)
-    set_result = {:blah => ['hi']}
-    
-    mq.write_result(message,set_result,20)
-    assert mq.list_tasks.empty?
-    assert_equal 1, mq.list_results.size
-    result = mq.take_result(1)
-    assert_equal set_result, result.payload
-  end
-  
-  def test_write_error
-    mq.write_message(@worker_message,10)  
-    message = mq.take_next_task(1)
-    mq.write_error(message,"something_bad_happened",10)
-    result = mq.take_result(1)          
-    assert_equal "something_bad_happened", result.payload
-  end
-
-  def test_write_complex_error
-    mq.write_message(@worker_message,10)  
-    message = mq.take_next_task(1)
-    error = {:error => "something_bad_happened"}
-    mq.write_error(message,error,10)
-    result = mq.take_result(1)          
-    assert_equal error, result.payload
-  end                      
-  
-  def test_write_worker_status
-    assert mq.write_worker_status({
-      :worker_id  => 5,
-      :hostname   => 'localhost',
-      :process_id => $$,
-      :name       => "waiting for master or tasks",
-      :processed  => 0,
-      :version    => 1,
-      :started_at => Time.now.to_i
-    })        
-  end
-  
-  def test_take_worker_status
-    assert mq.write_worker_status({
-      :worker_id  => 5,
-      :hostname   => 'localhost',
-      :process_id => $$,
-      :name       => "waiting for master or tasks",
-      :processed  => 0,
-      :version    => 1,
-      :started_at => Time.now.to_i
-    })        
-    
-    status = mq.take_worker_status({:hostname   => 'localhost',:process_id => $$ })
-    assert_equal $$, status.process_id        
-  end
-  
-  def test_read_all_worker_statuses
-    assert mq.write_worker_status({
-      :worker_id  => 5,
-      :hostname   => 'localhost',
-      :process_id => $$,
-      :name       => "waiting for master or tasks",
-      :processed  => 0,
-      :version    => 1,
-      :started_at => Time.now.to_i
-    })        
-    status = mq.read_all_worker_statuses.first
-
-    # status = mq.take_worker_status({:hostname   => 'localhost',:process_id => $$ })
-    assert_equal $$, status.process_id        
-  end
-
-  def test_worker_version
-    mq.set_worker_version(2)
-    assert_equal 2, mq.get_worker_version
-    mq.set_worker_version(10)
-    assert_equal 10, mq.get_worker_version
-  end
-  
-
-  
-
-private
-  def mq
-		Skynet::MessageQueueAdapter::TupleSpace.new
-  end
-    
-end