skynet 0.9.1

data/sometest.rb

@@ -0,0 +1,23 @@
+class SomeTest
+  include SkynetDebugger
+  
+  def self.map(datas)
+    results = {}
+    datas.each do |data|
+      results[data] ||= 0
+      results[data] += 1
+    end                 
+    [results]      
+  end
+  
+  def self.reduce(datas)
+    results = {}
+    datas.each do |hashes|
+      hashes.each do |key,value|
+        results[key] ||= 0
+        results[key] += value
+      end
+    end
+    results
+  end
+end

data/tasks/deployment.rake

@@ -0,0 +1,34 @@
+desc 'Release the website and new gem version'
+task :deploy => [:check_version, :website, :release] do
+  puts "Remember to create SVN tag:"
+  puts "svn copy svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/trunk " +
+    "svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/tags/REL-#{VERS} "
+  puts "Suggested comment:"
+  puts "Tagging release #{CHANGES}"
+end
+
+desc 'Runs tasks website_generate and install_gem as a local deployment of the gem'
+task :local_deploy => [:website_generate, :install_gem]
+
+task :check_version do
+  unless ENV['VERSION']
+    puts 'Must pass a VERSION=x.y.z release version'
+    exit
+  end
+  unless ENV['VERSION'] == VERS
+    puts "Please update your version.rb to match the release version, currently #{VERS}"
+    exit
+  end
+end
+
+desc 'Install the package as a gem, without generating documentation(ri/rdoc)'
+task :install_gem_no_doc => [:clean, :package] do
+  sh "#{'sudo ' unless Hoe::WINDOZE }gem install pkg/*.gem --no-rdoc --no-ri"
+end
+
+namespace :manifest do
+  desc 'Recreate Manifest.txt to include ALL files'
+  task :refresh do
+    `rake check_manifest | patch -p0 > Manifest.txt`
+  end
+end

data/tasks/environment.rake

@@ -0,0 +1,7 @@
+task :ruby_env do
+  RUBY_APP = if RUBY_PLATFORM =~ /java/
+    "jruby"
+  else
+    "ruby"
+  end unless defined? RUBY_APP
+end

data/tasks/website.rake

@@ -0,0 +1,17 @@
+desc 'Generate website files'
+task :website_generate => :ruby_env do
+  (Dir['website/**/*.txt'] - Dir['website/version*.txt']).each do |txt|
+    sh %{ #{RUBY_APP} script/txt2html #{txt} > #{txt.gsub(/txt$/,'html')} }
+  end
+end
+
+desc 'Upload website files to rubyforge'
+task :website_upload do
+  host = "#{rubyforge_username}@rubyforge.org"
+  remote_dir = "/var/www/gforge-projects/#{PATH}/"
+  local_dir = 'website'
+  sh %{rsync -aCv #{local_dir}/ #{host}:#{remote_dir}}
+end
+
+desc 'Generate and upload website files'
+task :website => [:website_generate, :website_upload, :publish_docs]

data/test/all_models_test.rb

@@ -0,0 +1,139 @@
+require File.dirname(__FILE__) + '/../../functional_test_helper'
+require 'mapreduce_lib/all_models'
+
+module MapreduceLib
+  class AllModelsTest < Test::Unit::TestCase
+
+    fixtures :profiles, :profile_details
+
+    def test_map
+      eachmeth = lambda do |profile|
+        profile.claimed = false
+        profile.save        
+      end
+      data = [1,10,{:conditions => "profiles.claimed = 1"},"Profile",eachmeth]
+      
+      assert Profile.find(2).claimed, "claimed"
+
+      MapreduceLib::AllModels.map([data])
+      
+      assert !Profile.find(2).claimed, "not claimed"
+    end
+
+    def test_each_with_proc
+      Skynet.solo do
+        MapreduceLib::AllModels.find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile", :limit => 5).each do |profile|
+          profile.suffix = 'z'
+          profile.save        
+        end
+      end      
+      assert_equal 'z', Profile.find(2).suffix
+    end
+
+    def test_big_batch
+      Skynet.solo do
+        MapreduceLib::AllModels.find(:all, :conditions => "claimed = 1", :batch_size => 200, :model_class => "Profile").each do |profile|
+          profile.suffix = 'z'
+          profile.save        
+        end
+      end      
+      assert_equal 'z', Profile.find(2).suffix
+    end
+
+    def test_joins
+      p = Profile.find(1)
+      p.details.zodiac_sign = 'hermit'
+      p.details.save
+      Skynet.solo do
+        MapreduceLib::AllModels.find(:all, :conditions => "profile_details.zodiac_sign='hermit'", :joins => "JOIN profile_details ON profiles.id = profile_details.profile_id", :batch_size => 2, :model_class => "Profile").each do |profile|
+          profile.suffix = 'z'
+          profile.save        
+        end
+      end      
+      assert_equal 'z', Profile.find(1).suffix
+    end
+
+    def test_stragglers
+      profiles = Profile.find(:all)
+      Skynet.solo do
+        MapreduceLib::AllModels.find(:all, :batch_size => profiles.size-1, :model_class => "Profile").each do |profile|
+          profile.suffix = 'z'
+          profile.save        
+        end
+      end                      
+      profiles.last.reload
+      assert_equal 'z', profiles.last.suffix      
+    end
+    
+    def test_small_limit
+      Skynet.solo do
+        MapreduceLib::AllModels.find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile", :limit => 3).each do |profile|
+          profile.suffix = 'z'
+          profile.save        
+        end
+      end      
+      assert_equal 'z', Profile.find(2).suffix
+    end
+
+    def test_each_with_proc_exception
+      Skynet.solo(:SKYNET_LOG_LEVEL => Logger::FATAL) do
+        MapreduceLib::AllModels.find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each do |profile|
+          raise "BUSTED" if profile.id == 6
+          profile.suffix = 'z'
+          profile.save        
+        end
+      end      
+      assert_equal 'z', Profile.find(2).suffix
+      assert_equal nil, Profile.find(6).suffix
+    end
+
+    def test_each_with_class_exception
+      Skynet.solo(:SKYNET_LOG_LEVEL => Logger::FATAL) do
+        MapreduceLib::AllModels.find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each(MapreduceLib::AllModelsExTest)
+      end      
+      assert_equal 'k', Profile.find(2).suffix
+      assert_equal nil, Profile.find(8).suffix
+    end
+
+    def test_each_with_class
+      Skynet.solo do
+        MapreduceLib::AllModels.find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each(self.class)
+      end      
+      assert_equal 'k', Profile.find(2).suffix
+    end
+    
+    def test_distributed_each
+      Skynet.solo do
+        Profile.distributed_find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each do |profile|
+            profile.suffix = 'gg'
+            profile.save        
+        end
+        assert_equal 'gg', Profile.find(2).suffix
+        Profile.distributed_find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each(self.class)
+        assert_equal 'k', Profile.find(2).suffix
+      end
+    end
+    
+    def test_distributed_each_with_symbol
+      Skynet.solo do
+        Profile.distributed_find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each(:mark_modified)
+      end
+      assert ModifiedProfile.find(:first, :conditions => "profile_id = 2")
+    end
+        
+    def self.each(profile)
+      profile.suffix = 'k'
+      profile.save        
+    end
+    
+  end
+
+  class AllModelsExTest
+    def self.each(profile)
+      raise "BUSTED" if profile.id == 8
+      profile.suffix = 'k'
+      profile.save        
+    end
+  end
+end
+

data/test/mysql_message_queue_adaptor_test.rb

@@ -0,0 +1,199 @@
+ENV["RAILS_ENV"] = "test"
+
+require 'test/unit'
+require '../lib/skynet.rb'
+
+
+# Tests for the partitioners.
+#
+class MysqlMessageQueueTest < Test::Unit::TestCase
+  
+  def setup
+    Skynet.configure(
+      :ENABLE                         => false,
+      # :SKYNET_PIDS_FILE               => File.expand_path("#{RAILS_ROOT}/log/skynet_#{RAILS_ENV}.pids"),
+      :SKYNET_LOG_FILE                => STDOUT,
+      :SKYNET_LOG_LEVEL               => Logger::ERROR,
+      :MESSAGE_QUEUE_ADAPTER          => "Skynet::MessageQueueAdapter::Mysql",
+      :NEXT_TASK_TIMEOUT              => 1
+    )      
+    mq.clear_outstanding_tasks
+    mq.clear_worker_status
+    
+    @worker_message = Skynet::Message.new(
+      :tasktype=> :task, 
+      :drburi=> "localhost", 
+      :job_id => 1,
+      :task_id => 2,
+      :payload => "payload",
+      :payload_type => "task",
+      :expiry => 20, 
+      :expire_time => 1095108406.9251,
+      :iteration => 0,
+      :name => "name",       
+      :version => 1
+    )
+    
+  end
+                   
+### THE FOLLOWING TESTS THE MessageQueueProxy API
+  
+  def test_write_message    
+    mq.write_message(@worker_message,10)
+    message = SkynetMessageQueue.find(:all).first
+    assert_equal @worker_message.expiry.to_f, 20
+  end
+                                                                 
+  def test_template_to_conditions
+    conditions = mq.template_to_conditions(Skynet::Message.next_task_template(1,:task))
+    assert_match "iteration BETWEEN 0 AND 6", conditions
+    assert_match "version = 1", conditions
+    assert_match "tasktype = 'task'", conditions
+    assert_match "expire_time BETWEEN 0 AND", conditions
+    assert_match "payload_type = 'task'", conditions
+  end
+
+  def test_message_to_conditions
+    conditions = mq.template_to_conditions(Skynet::Message.new(Skynet::Message.next_task_template(1)))
+    assert_match "iteration BETWEEN 0 AND 6", conditions
+    assert_match "version = 1", conditions
+    assert_match "tasktype = 'task'", conditions
+    assert_match "expire_time BETWEEN 0 AND", conditions
+  end
+  
+  def test_take_next_task
+    assert mq.write_message(@worker_message,10)                                   
+    task = mq.take_next_task(1,1,:task)                       
+    assert_equal @worker_message.payload, task.payload
+    
+    message = SkynetMessageQueue.find(:first)
+    assert_equal 1, message.iteration
+    assert @worker_message.expire_time < message.expire_time            
+    excep = nil
+    begin 
+      mq.take_next_task(1)
+    rescue Skynet::RequestExpiredError
+      excep = true
+    end
+    assert excep    
+  end
+  
+  def test_task_failover
+    message = @worker_message.clone
+    message.expiry=0.4
+    assert mq.write_message(message)
+    task = mq.take_next_task(1)
+    assert_equal 2, task.task_id
+    assert_equal 0, task.iteration
+    sleep 0.6
+    ntt = Skynet::Message.next_task_template(1)
+    next_task = mq.take_next_task(1,0.00001)
+    assert_equal 2, next_task.task_id
+    assert_equal 1, next_task.iteration
+  end
+
+  def test_write_and_take_result
+    assert mq.write_message(@worker_message,10)  
+    message = mq.take_next_task(1)
+    set_result = {:blah => ['hi']}
+    
+    mq.write_result(message,set_result,20)    
+
+    assert mq.list_tasks.empty?
+    assert_equal 1, mq.list_results.size
+    result = mq.take_result(1)
+    assert_equal set_result, result.payload
+  end
+  
+  def test_write_error
+    mq.write_message(@worker_message,10)  
+    message = mq.take_next_task(1)
+    mq.write_error(message,"something_bad_happened",10)
+    result = mq.take_result(1)          
+    assert_equal "something_bad_happened", result.payload
+  end
+
+  def test_write_complex_error
+    mq.write_message(@worker_message,10)  
+    message = mq.take_next_task(1)
+    error = {:error => "something_bad_happened"}
+    mq.write_error(message,error,10)
+    result = mq.take_result(1)          
+    assert_equal error, result.payload
+  end                      
+  
+  def test_write_worker_status
+    assert mq.write_worker_status({
+      :worker_id  => 5,
+      :hostname   => 'localhost',
+      :process_id => $$,
+      :name       => "waiting for master or tasks",
+      :iteration  => 0,
+      :processed  => 0,
+      :version    => 1,
+      :started_at => Time.now.to_i
+    })                 
+    ws = SkynetWorkerQueue.find_by_worker_id(5)
+    assert_equal ws.iteration, 0
+
+    assert mq.write_worker_status({
+      :worker_id  => 5,
+      :hostname   => 'localhost',
+      :process_id => $$,
+      :name       => "waiting for master or tasks",
+      :processed  => 0,
+      :version    => 1,
+      :started_at => Time.now.to_i
+    })                            
+    ws.reload
+    assert_equal ws.iteration, nil    
+  end
+  
+  def test_take_worker_status
+    assert mq.write_worker_status({
+      :worker_id  => 5,
+      :hostname   => 'localhost',
+      :process_id => $$,
+      :name       => "waiting for master or tasks",
+      :processed  => 0,
+      :version    => 1,
+      :started_at => Time.now.to_i
+    })        
+    
+    status = mq.take_worker_status({:hostname   => 'localhost',:process_id => $$ })
+    assert_equal $$, status.process_id        
+  end
+  
+  def test_read_all_worker_statuses
+    assert mq.write_worker_status({
+      :worker_id  => 5,
+      :hostname   => 'localhost',
+      :process_id => $$,
+      :name       => "waiting for master or tasks",
+      :processed  => 0,
+      :version    => 1,
+      :started_at => Time.now.to_i
+    })        
+    status = mq.read_all_worker_statuses.first
+
+    # status = mq.take_worker_status({:hostname   => 'localhost',:process_id => $$ })
+    assert_equal $$, status.process_id        
+  end
+
+  def test_worker_version
+    mq.set_worker_version(2)
+    assert_equal 2, mq.get_worker_version
+    mq.set_worker_version(10)
+    assert_equal 10, mq.get_worker_version
+    mq.set_worker_version(11)
+    mq.set_worker_version(12)
+    assert_equal 1, SkynetWorkerQueue.count(:id, :conditions => "tasktype = 'workerversion'")
+  end
+
+  private
+
+  def mq
+		Skynet::MessageQueueAdapter::Mysql.new
+  end
+
+end

data/test/skynet_manager_test.rb

@@ -0,0 +1,107 @@
+ENV["RAILS_ENV"] = "test"
+
+require 'test/unit'
+require 'pp'        
+
+
+# Tests for the partitioners.
+#
+
+
+class SkynetManagerTest < Test::Unit::TestCase
+
+  PIDS = []
+  PIDFILE = []
+  
+  def setup
+    Skynet.configure(
+      :ENABLE                         => false,
+      # :SKYNET_PIDS_FILE               => File.expand_path("#{RAILS_ROOT}/log/skynet_#{RAILS_ENV}.pids"),
+      :SKYNET_LOG_FILE                => STDOUT,
+      :SKYNET_LOG_LEVEL               => Logger::INFO,
+      :MESSAGE_QUEUE_ADAPTER          => "Skynet::MessageQueueAdapter::Mysql",
+      :NEXT_TASK_TIMEOUT              => 1,
+      :WORKER_CHECK_DELAY             => 0.5,
+      :WORKER_WARMUP_TIME             => 0.5
+    )      
+    mq.clear_outstanding_tasks
+    mq.clear_worker_status
+    @pids = []
+    @pidfile = []
+    
+    PIDS.delete_if {true}
+    PIDFILE.delete_if {true}
+
+    Process.stubs(:detatch).returns(true)    
+    Process.stubs(:kill).returns(true)
+
+    @minstance = Skynet::Manager.any_instance
+   
+    Skynet::Worker.any_instance.stubs(:max_memory_reached?).returns(false)
+
+    @manager = Skynet::Manager.new("path",2)
+    
+    def @manager.fork
+      newpid = SkynetManagerTest::PIDS.size + 1
+      SkynetManagerTest::PIDS << newpid
+      worker = Skynet::Worker.new()
+      worker.stubs(:process_id).returns(newpid)
+      worker.notify_worker_queue(:started)
+      newpid
+    end
+  end  
+  
+  def test_manager_start  
+    @manager.start_workers
+    assert_equal 2, @manager.worker_pids.size
+    assert_equal PIDS.sort, @manager.worker_pids.sort
+    assert_equal PIDS.sort, @manager.worker_queue.collect {|q|q.process_id}.sort
+  end
+
+  def test_check_workers
+    Skynet::Manager.any_instance.expects(:worker_alive?).times(2).returns(true)
+    @manager.start_workers
+    @manager.check_workers
+    assert_equal 2, @manager.worker_pids.size
+    assert_equal PIDS.sort, @manager.worker_pids.sort
+    assert_equal PIDS.sort, @manager.worker_queue.collect {|q|q.process_id}.sort
+  end
+  
+  def test_running_pids
+    Skynet::Manager.any_instance.expects(:worker_alive?).with(1).returns(true)
+    Skynet::Manager.any_instance.expects(:worker_alive?).with(2).returns(false)
+    @manager.start_workers
+    @manager.check_workers
+    assert_equal 1, @manager.worker_pids.size
+    assert_equal [1], @manager.worker_pids.sort
+    assert_equal [1], @manager.worker_queue.collect {|q|q.process_id}.sort
+  end
+
+  ## XXX FIXME.  What happens if there's a worker missing from the pidfile, but was running?
+  def test_more_in_pidfile_than_queue_alive
+    Skynet::Manager.any_instance.expects(:worker_alive?).with(1).returns(true)
+    Skynet::Manager.any_instance.expects(:worker_alive?).with(2).returns(true)
+    @manager.start_workers
+    SkynetWorkerQueue.find(:first, :conditions => "process_id = 1").destroy
+    @manager.check_workers
+    assert_equal 1, @manager.worker_pids.size
+    assert_equal [2], @manager.worker_pids.sort
+    assert_equal [2], @manager.worker_queue.collect {|q|q.process_id}.sort
+  end
+
+  def test_dead_workers
+    Skynet::Manager.any_instance.expects(:worker_alive?).times(1).with(1).returns(false)
+    Skynet::Manager.any_instance.expects(:worker_alive?).with(2).returns(true)
+    @manager.start_workers     
+    @manager.check_workers
+    assert_equal [2], @manager.worker_pids.sort
+    assert_equal [2], @manager.worker_queue.collect {|q|q.process_id}.sort
+  end
+
+
+  private
+
+  def mq
+		Skynet::MessageQueueAdapter::Mysql.new
+  end
+end