timocratic-skynet 0.9.4

data/tasks/deployment.rake

@@ -0,0 +1,34 @@
+desc 'Release the website and new gem version'
+task :deploy => [:check_version, :website, :release] do
+  puts "Remember to create SVN tag:"
+  puts "svn copy svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/trunk " +
+    "svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/tags/REL-#{VERS} "
+  puts "Suggested comment:"
+  puts "Tagging release #{CHANGES}"
+end
+
+desc 'Runs tasks website_generate and install_gem as a local deployment of the gem'
+task :local_deploy => [:website_generate, :install_gem]
+
+task :check_version do
+  unless ENV['VERSION']
+    puts 'Must pass a VERSION=x.y.z release version'
+    exit
+  end
+  unless ENV['VERSION'] == VERS
+    puts "Please update your version.rb to match the release version, currently #{VERS}"
+    exit
+  end
+end
+
+desc 'Install the package as a gem, without generating documentation(ri/rdoc)'
+task :install_gem_no_doc => [:clean, :package] do
+  sh "#{'sudo ' unless Hoe::WINDOZE }gem install pkg/*.gem --no-rdoc --no-ri"
+end
+
+namespace :manifest do
+  desc 'Recreate Manifest.txt to include ALL files'
+  task :refresh do
+    `rake check_manifest | patch -p0 > Manifest.txt`
+  end
+end

data/tasks/environment.rake

@@ -0,0 +1,7 @@
+task :ruby_env do
+  RUBY_APP = if RUBY_PLATFORM =~ /java/
+    "jruby"
+  else
+    "ruby"
+  end unless defined? RUBY_APP
+end

data/tasks/website.rake

@@ -0,0 +1,17 @@
+desc 'Generate website files'
+task :website_generate => :ruby_env do
+  (Dir['website/**/*.txt'] - Dir['website/version*.txt']).each do |txt|
+    sh %{ #{RUBY_APP} script/txt2html #{txt} > #{txt.gsub(/txt$/,'html')} }
+  end
+end
+
+desc 'Upload website files to rubyforge'
+task :website_upload do
+  host = "#{rubyforge_username}@rubyforge.org"
+  remote_dir = "/var/www/gforge-projects/#{PATH}/"
+  local_dir = 'website'
+  sh %{rsync -aCv #{local_dir}/ #{host}:#{remote_dir}}
+end
+
+desc 'Generate and upload website files'
+task :website => [:website_generate, :website_upload, :publish_docs]

data/test/test_active_record_extensions.rb

@@ -0,0 +1,138 @@
+require File.dirname(__FILE__) + '/test_helper.rb'
+
+class AllModelsTest < Test::Unit::TestCase
+  
+  def test_true
+    true
+  end
+
+  # def test_map
+  #   eachmeth = lambda do |profile|
+  #     profile.claimed = false
+  #     profile.save        
+  #   end
+  #   data = [1,10,{:conditions => "profiles.claimed = 1"},"Profile",eachmeth]
+  #   
+  #   assert Profile.find(2).claimed, "claimed"
+  # 
+  #   ActiveRecord::Mapreduce.map([data])
+  #   
+  #   assert !Profile.find(2).claimed, "not claimed"
+  # end
+  # 
+  # def test_each_with_proc
+  #   Skynet.solo do
+  #     ActiveRecord::Mapreduce.find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile", :limit => 5).each do |profile|
+  #       profile.suffix = 'z'
+  #       profile.save        
+  #     end
+  #   end      
+  #   assert_equal 'z', Profile.find(2).suffix
+  # end
+  # 
+  # def test_big_batch
+  #   Skynet.solo do
+  #     ActiveRecord::Mapreduce.find(:all, :conditions => "claimed = 1", :batch_size => 200, :model_class => "Profile").each do |profile|
+  #       profile.suffix = 'z'
+  #       profile.save        
+  #     end
+  #   end      
+  #   assert_equal 'z', Profile.find(2).suffix
+  # end
+  # 
+  # def test_joins
+  #   p = Profile.find(1)
+  #   p.details.zodiac_sign = 'hermit'
+  #   p.details.save
+  #   Skynet.solo do
+  #     ActiveRecord::Mapreduce.find(:all, :conditions => "profile_details.zodiac_sign='hermit'", :joins => "JOIN profile_details ON profiles.id = profile_details.profile_id", :batch_size => 2, :model_class => "Profile").each do |profile|
+  #       profile.suffix = 'z'
+  #       profile.save        
+  #     end
+  #   end      
+  #   assert_equal 'z', Profile.find(1).suffix
+  # end
+  # 
+  # def test_stragglers
+  #   profiles = Profile.find(:all)
+  #   Skynet.solo do
+  #     ActiveRecord::Mapreduce.find(:all, :batch_size => profiles.size-1, :model_class => "Profile").each do |profile|
+  #       profile.suffix = 'z'
+  #       profile.save        
+  #     end
+  #   end                      
+  #   profiles.last.reload
+  #   assert_equal 'z', profiles.last.suffix      
+  # end
+  # 
+  # def test_small_limit
+  #   Skynet.solo do
+  #     ActiveRecord::Mapreduce.find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile", :limit => 3).each do |profile|
+  #       profile.suffix = 'z'
+  #       profile.save        
+  #     end
+  #   end      
+  #   assert_equal 'z', Profile.find(2).suffix
+  # end
+  # 
+  # def test_each_with_proc_exception
+  #   Skynet.solo(:SKYNET_LOG_LEVEL => Logger::FATAL) do
+  #     ActiveRecord::Mapreduce.find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each do |profile|
+  #       raise "BUSTED" if profile.id == 6
+  #       profile.suffix = 'z'
+  #       profile.save        
+  #     end
+  #   end      
+  #   assert_equal 'z', Profile.find(2).suffix
+  #   assert_equal nil, Profile.find(6).suffix
+  # end
+  # 
+  # def test_each_with_class_exception
+  #   Skynet.solo(:SKYNET_LOG_LEVEL => Logger::FATAL) do
+  #     ActiveRecord::Mapreduce.find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each(ActiveRecord::MapreduceExTest)
+  #   end      
+  #   assert_equal 'k', Profile.find(2).suffix
+  #   assert_equal nil, Profile.find(8).suffix
+  # end
+  # 
+  # def test_each_with_class
+  #   Skynet.solo do
+  #     ActiveRecord::Mapreduce.find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each(self.class)
+  #   end      
+  #   assert_equal 'k', Profile.find(2).suffix
+  # end
+  # 
+  # def test_distributed_each
+  #   Skynet.solo do
+  #     Profile.distributed_find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each do |profile|
+  #         profile.suffix = 'gg'
+  #         profile.save        
+  #     end
+  #     assert_equal 'gg', Profile.find(2).suffix
+  #     Profile.distributed_find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each(self.class)
+  #     assert_equal 'k', Profile.find(2).suffix
+  #   end
+  # end
+  # 
+  # def test_distributed_each_with_symbol
+  #   Skynet.solo do
+  #     Profile.distributed_find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each(:mark_modified)
+  #   end
+  #   assert ModifiedProfile.find(:first, :conditions => "profile_id = 2")
+  # end
+  #     
+  # def self.each(profile)
+  #   profile.suffix = 'k'
+  #   profile.save        
+  # end
+
+end
+
+class AllModelsExTest
+  def self.each(profile)
+    raise "BUSTED" if profile.id == 8
+    profile.suffix = 'k'
+    profile.save        
+  end
+end
+

data/test/test_generator_helper.rb

@@ -0,0 +1,20 @@
+require File.dirname(__FILE__) + '/test_helper'
+require 'fileutils'
+
+# Must set before requiring generator libs.
+TMP_ROOT = File.dirname(__FILE__) + "/tmp" unless defined?(TMP_ROOT)
+PROJECT_NAME = "myproject" unless defined?(PROJECT_NAME)
+app_root = File.join(TMP_ROOT, PROJECT_NAME)
+if defined?(APP_ROOT)
+  APP_ROOT.replace(app_root)
+else
+  APP_ROOT = app_root
+end
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/helpers/generator_test_helper'

data/test/test_helper.rb

@@ -0,0 +1,10 @@
+ENV["RAILS_ENV"] = "test"
+$VERBOSE=false
+
+
+require 'test/unit'
+require 'rubygems'
+require File.dirname(__FILE__) + '/../lib/skynet'
+require 'pp'        
+require 'mocha'
+require 'functor'

data/test/test_mysql_message_queue_adapter.rb

@@ -0,0 +1,263 @@
+require File.dirname(__FILE__) + '/test_helper.rb'
+
+class MysqlMessageQueueTest < Test::Unit::TestCase
+  ENV["RAILS_ENV"] = "test"
+  
+  ActiveRecord::Base.establish_connection(
+    :adapter  => "mysql",
+    :host     => "localhost",
+    :username => "root",
+    :password => "",
+    :database => "skynet_test"
+  )
+  ActiveRecord::Base.connection.execute("delete from skynet_message_queues")
+  ActiveRecord::Base.connection.execute("delete from skynet_worker_queues")
+  
+  def setup
+    Skynet.configure(
+      :ENABLE                         => false,
+      # :SKYNET_PIDS_FILE               => File.expand_path("#{RAILS_ROOT}/log/skynet_#{RAILS_ENV}.pids"),
+      :SKYNET_LOG_FILE                => STDOUT,
+      :SKYNET_LOG_LEVEL               => Logger::ERROR,
+      :MESSAGE_QUEUE_ADAPTER          => "Skynet::MessageQueueAdapter::Mysql",
+      :MYSQL_NEXT_TASK_TIMEOUT        => 1,
+      :MYSQL_TEMPERATURE_CHANGE_SLEEP => 1
+    )      
+
+    ActiveRecord::Base.establish_connection(
+      :adapter  => "mysql",
+      :host     => "localhost",
+      :username => "root",
+      :password => "",
+      :database => "skynet_test"
+    )
+    ActiveRecord::Base.connection.execute("delete from skynet_message_queues")
+    ActiveRecord::Base.connection.execute("delete from skynet_worker_queues")
+    
+    @message_options = {
+      :tasktype     => "task", 
+      :job_id       => 1,
+      :task_id      => 2,
+      :payload      => "payload",
+      :payload_type => "task",
+      :expiry       => 20, 
+      :expire_time  => 1095108406.9251,
+      :iteration    => 0,
+      :name         => "name",       
+      :version      => 1,
+      :queue_id     => 0,
+      :retry        => 1
+    }
+    @worker_message = Skynet::Message.new(@message_options)
+    
+  end
+                   
+### THE FOLLOWING TESTS THE MessageQueueProxy API
+  
+  def test_write_message    
+    mq.write_message(@worker_message,10)
+    message = SkynetMessageQueue.find(:first, :conditions => "tasktype = 'task'")
+    assert_equal @worker_message.expiry.to_f, 20          
+    @message_options.each do |key,val|
+      next if key == :payload
+      assert_equal message.send(key), val
+    end    
+  end
+                                                                 
+  def test_template_to_conditions
+    conditions = mq.template_to_conditions(Skynet::Message.next_task_template(1,:task))
+    assert_match "iteration BETWEEN 0 AND 6", conditions
+    assert_match "version = 1", conditions
+    assert_match "tasktype = 'task'", conditions
+    assert_match "expire_time BETWEEN 0 AND", conditions
+    assert_match "payload_type = 'task'", conditions
+  end
+
+  def test_message_to_conditions
+    conditions = mq.template_to_conditions(Skynet::Message.new(Skynet::Message.next_task_template(1)))
+    assert_match "iteration BETWEEN 0 AND 6", conditions
+    assert_match "version = 1", conditions
+    assert_match "tasktype = 'task'", conditions
+    assert_match "expire_time BETWEEN 0 AND", conditions
+  end
+  
+  def test_take_next_task
+    assert mq.write_message(@worker_message,10)
+    message = SkynetMessageQueue.find(:first, :conditions => "tasktype = 'task'")
+    assert_equal 0, message.iteration
+    task = mq.take_next_task(1,1,:task)                       
+    assert_equal @worker_message.payload, task.payload
+
+    message = SkynetMessageQueue.find(:first, :conditions => "tasktype = 'task'")
+    assert_equal 1, message.iteration
+    assert @worker_message.expire_time < message.expire_time            
+    
+    excep = nil
+    begin 
+      mq.take_next_task(1)
+    rescue Skynet::RequestExpiredError
+      excep = true
+    end
+    assert excep    
+  end
+  
+  def test_task_failover
+    message = @worker_message.clone
+    message.expiry=0.4
+    assert mq.write_message(message)
+    task = mq.take_next_task(1)
+    assert_equal 2, task.task_id
+    assert_equal 0, task.iteration
+    ntt = Skynet::Message.next_task_template(1)
+    next_task = mq.take_next_task(1,1)
+    assert_equal 2, next_task.task_id
+    assert_equal 1, next_task.iteration
+  end               
+  
+  def test_take_race_condition
+    message = @worker_message.clone
+    message.expiry=0.4
+    assert mq.write_message(message)
+    template = Skynet::Message.next_task_template(1, :task, 0)
+
+    message_row = mq.send(:find_next_message,template,:task)
+    assert !message_row.tran_id
+
+    assert_equal 1, mq.send(:write_fallback_message,message_row, message)
+    assert_equal 0, mq.send(:write_fallback_message,message_row, message)
+    assert_equal 0, mq.send(:write_fallback_message,message_row, message)
+
+    template = Skynet::Message.next_task_template(1, :task, 0)
+    another_row = mq.send(:find_next_message,template,:task)
+    assert !another_row
+    sleep 1
+    template = Skynet::Message.next_task_template(1, :task, 0)
+    message_row2 = mq.send(:find_next_message,template,:task)
+    assert message_row2
+    assert_equal message_row.tran_id, message_row2.tran_id
+
+    message_row2.tran_id = 4
+    assert_equal 0, mq.send(:write_fallback_message,message_row2, message)
+  end
+
+  def test_write_and_take_result
+    assert mq.write_message(@worker_message,10)  
+    message = mq.take_next_task(1)
+    set_result = {:blah => ['hi']}
+    
+    mq.write_result(message,set_result,20)    
+
+    assert mq.list_tasks.empty?
+    assert_equal 1, mq.list_results.size
+    result = mq.take_result(1)
+    assert_equal set_result, result.payload
+  end
+  
+  def test_write_error
+    mq.write_message(@worker_message,10)  
+    message = mq.take_next_task(1)
+    mq.write_error(message,"something_bad_happened",10)
+    result = mq.take_result(1)          
+    assert_equal "something_bad_happened", result.payload
+  end
+
+  def test_write_complex_error
+    mq.write_message(@worker_message,10)  
+    message = mq.take_next_task(1)
+    error = {:error => "something_bad_happened"}
+    mq.write_error(message,error,10)
+    result = mq.take_result(1)          
+    assert_equal error, result.payload
+  end                      
+  
+  def test_write_worker_status
+    assert mq.write_worker_status({
+      :worker_id  => 5,
+      :hostname   => 'localhost',
+      :process_id => $$,
+      :name       => "waiting for master or tasks",
+      :iteration  => 0,
+      :processed  => 0,
+      :version    => 1,
+      :started_at => Time.now.to_i
+    })                 
+    ws = SkynetWorkerQueue.find_by_worker_id(5)
+    assert_equal ws.iteration, 0
+
+    assert mq.write_worker_status({
+      :worker_id  => 5,
+      :hostname   => 'localhost',
+      :process_id => $$,
+      :name       => "waiting for master or tasks",
+      :processed  => 0,
+      :version    => 1,
+      :started_at => Time.now.to_i
+    })                            
+    ws.reload
+    assert_equal ws.iteration, nil    
+  end
+  
+  def test_take_worker_status
+    assert mq.write_worker_status({
+      :worker_id  => 5,
+      :hostname   => 'localhost',
+      :process_id => $$,
+      :name       => "waiting for master or tasks",
+      :processed  => 0,
+      :version    => 1,
+      :started_at => Time.now.to_i
+    })        
+    
+    status = mq.take_worker_status({:hostname   => 'localhost',:process_id => $$ })
+    assert_equal $$, status.process_id        
+  end
+  
+  def test_read_all_worker_statuses
+    assert mq.write_worker_status({
+      :worker_id  => 5,
+      :hostname   => 'localhost',
+      :process_id => $$,
+      :name       => "waiting for master or tasks",
+      :processed  => 0,
+      :version    => 1,
+      :started_at => Time.now.to_i
+    })        
+    status = mq.read_all_worker_statuses.first
+
+    # status = mq.take_worker_status({:hostname   => 'localhost',:process_id => $$ })
+    assert_equal $$, status.process_id        
+  end
+
+  def test_worker_version
+    mq.set_worker_version(2)
+    assert_equal 2, mq.get_worker_version
+    mq.set_worker_version(10)
+    assert_equal 10, mq.get_worker_version
+    mq.set_worker_version(11)
+    mq.set_worker_version(12)
+    assert_equal 1, SkynetMessageQueue.count(:id, :conditions => "tasktype = 'version'")
+  end
+  
+  def test_version_active?
+    mq.set_worker_version(98)
+    @worker_message.version = mq.get_worker_version
+    mq.write_message(@worker_message,10)
+    message = SkynetMessageQueue.find(:first, :conditions => "tasktype = 'task'")
+    assert_equal message.version, 98
+    mq.set_worker_version(99)
+    assert_equal 99, mq.get_worker_version
+    assert_equal true, mq.version_active?(98)
+    assert_equal true, mq.version_active?(99)
+    message.destroy
+    assert_equal false, mq.version_active?(98)    
+    assert_equal true, mq.version_active?(99)    
+  end
+    
+
+  private
+
+  def mq
+		Skynet::MessageQueueAdapter::Mysql.new
+  end
+
+end