skynet 0.9.1 → 0.9.2

data/app_generators/skynet_install/templates/skynet_schema.sql

@@ -0,0 +1,56 @@
+CREATE TABLE skynet_message_queues (
+  id int(11) NOT NULL auto_increment,
+  queue_id int(11) default '0',
+  tran_id bigint(20) unsigned default NULL,
+  created_on datetime default NULL,
+  updated_on datetime default NULL,
+  tasktype varchar(255) default NULL,
+  task_id bigint(20) unsigned default NULL,
+  job_id bigint(20) unsigned default NULL,
+  raw_payload text,
+  payload_type varchar(255) default NULL,
+  name varchar(255) default NULL,
+  expiry int(11) default NULL,
+  expire_time decimal(16,4) default NULL,
+  iteration int(11) default NULL,
+  version int(11) default NULL,
+  timeout decimal(16,4) default NULL,
+  retry int(11) default '0',
+  PRIMARY KEY  (id),
+  UNIQUE KEY index_skynet_message_queues_on_tran_id (tran_id),
+  KEY index_skynet_message_queues_on_job_id (job_id),
+  KEY index_skynet_message_queues_on_task_id (task_id),
+  KEY index_skynet_mqueue_for_take (queue_id,tasktype,payload_type,expire_time)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+CREATE TABLE skynet_worker_queues (
+  id int(11) NOT NULL auto_increment,
+  queue_id int(11) default '0',
+  created_on datetime default NULL,
+  updated_on datetime default NULL,
+  tasktype varchar(255) default NULL,
+  tasksubtype varchar(255) default NULL,
+  worker_id bigint(20) unsigned default NULL,
+  hostname varchar(255) default NULL,
+  process_id int(11) default NULL,
+  job_id bigint(20) unsigned default NULL,
+  task_id bigint(20) unsigned default NULL,
+  iteration int(11) default NULL,
+  name varchar(255) default NULL,
+  map_or_reduce varchar(255) default NULL,
+  started_at decimal(16,4) default NULL,
+  version int(11) default NULL,
+  processed int(11) default NULL,
+  timeout decimal(16,4) default NULL,
+  PRIMARY KEY  (id),
+  UNIQUE KEY index_skynet_worker_queues_on_worker_id (worker_id),
+  KEY index_skynet_worker_queues_on_hostname_and_process_id (hostname,process_id)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+CREATE TABLE skynet_queue_temperature (
+  id int(11) NOT NULL auto_increment,
+  queue_id int(11) default '0',
+  updated_on datetime default NULL,
+  count int(11) default '0',
+  temperature decimal(6,4) default NULL,
+  type varchar(255) default NULL,
+  PRIMARY KEY  (id)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;

data/bin/skynet

@@ -1,8 +1,33 @@
 #!/usr/bin/env ruby
 
+# This is the main skynet starter script.
+# It can be used to start the skynet_tuplespace_server as well as all skynet workers.
+# It is important that this script has access to your code. If you want to run
+# skynet within your code you'll want to read about bin/skynet_install[link:files/bin/skynet_install.html]
+#
+# Usage: skynet [options]
+#     -w, --workers WORKERS            Number of workers to start.  The default is 4 and is stored in Skynet::CONFIG[:NUMBER_OF_WORKERS]
+#     -i, --increment-worker-version   Increment Worker Version
+#     -a, --add-workers WORKERS        Number of workers to add.
+#     -k, --remove-workers WORKERS     Number of workers to remove.
+#     -r, --required LIBRARY           Require the specified libraries
+#         --restart-all-workers        Restart All Workers                                      
+#         --restart-workers            Restart Workers                                      
+#
+# If you have chosen to use the TupleSpace message queue adapter this script will see if there is an available TS first
+# and start one if there is not.  You can also start the bin/skynet_tuplespace_server[link:files/bin/skynet_tuplespace_server.html] manually.  
+#
+# Running skynet starts a Skynet::Manager which in turn spawns the Skynet::Worker processes with the appropriate options.
+# You only need to run skynet once per machine.  If you want to add more workers, use the appropriate flags above to do so.
+# Only one manager should be running per machine.  The Skynet::Manager does not dole out tasks
+#
+# You should set all of your Skynet::CONFIG (or Skynet.configure()) options here (or in one of your environment files.)
+# See Skynet::Config for more information on configuration options.
+
+require 'rubygems'
 require File.expand_path(File.dirname(__FILE__)) + '/../lib/skynet.rb'
 
-Skynet::CONFIG[:WORKER_CHECK_DELAY]     = 4
+Skynet::CONFIG[:WORKER_CHECK_DELAY]     ||= 4
 Skynet::CONFIG[:LAUNCHER_PATH]          = File.expand_path(__FILE__)
 
 begin
@@ -16,5 +41,4 @@ rescue Skynet::ConnectionError
   end
 end
 
-
 Skynet.new

data/bin/skynet_install

@@ -1,3 +1,27 @@
+#!/usr/bin/env ruby
+
+# skynet_install is used to install skynet binaries into your application.
+# The value of using your own skynet binaries is that they can ensure your code
+# is available to all skynet workers.
+# This is also how you can run skynet from within rails. (using --rails)
+#
+# USAGE: skynet_install [--rails] directory (can be '.' for current)"
+# 
+#   Options:
+#       -v, --version                    Show the skynet_install version number and quit.
+#           --include-migration          Include mysql migration if you want to use mysql as your message queue
+#       -r, --rails                      Install into rails app
+#                                        Default: false
+#   General Options:
+#       -h, --help                       Show this help message and quit.
+#       -p, --pretend                    Run but do not make any changes.
+#       -f, --force                      Overwrite files that already exist.
+#       -s, --skip                       Skip files that already exist.
+#       -q, --quiet                      Suppress normal output.
+#       -t, --backtrace                  Debugging: show backtrace on errors.
+#       -c, --svn                        Modify files with subversion. (Note: svn must be in path)
+# 
+
 require 'rubygems'
 require 'rubigen'
 

data/bin/skynet_tuplespace_server

@@ -1,5 +1,16 @@
 #!/usr/bin/env ruby
 
+# SkynetTupleSpace server is one of the message queues you can use with Skynet.   Make sure you set:
+#   Skynet::CONFIG[:MESSAGE_QUEUE_ADAPTER] = "Skynet::MessageQueueAdapter::TupleSpace"
+#
+# Usage: skynet_tuplespace_server (start|stop|run) [options]
+#     -t, --ontop TRUE                 Dont Daemonize
+#     -p, --port PORT                  Port to listen on. default 7647
+#     -o, --log LOGFILE                Logfile to log to
+#     -l, --loglevel LOGLEVEL          Log level defaults to DEBUG
+#     -d, --piddir PIDDIR              Directory to put pidfile
+#     -u, --drburi Drb URI             What DRbURI to use
+
 require 'rubygems'
 require 'daemons'
 require 'pp'
@@ -36,6 +47,8 @@ OptionParser.new do |opt|
     else
       options[:drburi] = "druby://#{v}"
     end
+    options[:drburi] =~ /druby:\/\/.+?:(\d*)/
+    options[:port] = $1.to_i
   end
 
   opt.parse!(ARGV)

data/config/hoe.rb

@@ -61,6 +61,7 @@ hoe = Hoe.new(GEM_NAME, VERS) do |p|
   p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n")
   p.extra_deps = [
     ['daemons',">= 1"],
+    ['rubigen', ">=1.1.1"]
     ]
     # An array of rubygem dependencies [name, version], e.g. [ ['active_support', '>= 1.3.1'] ]
   

data/lib/skynet.rb

@@ -8,6 +8,7 @@ require 'drb'
 require 'skynet_guid_generator'
 require 'skynet_logger'
 require 'skynet_config'
+require 'timeout'    
 
 Skynet::CONFIG[:SKYNET_PATH]    ||= File.expand_path(File.dirname(__FILE__) +"/..")
 # Skynet::CONFIG[:LAUNCHER_PATH]  ||= File.expand_path(ENV['_'])
@@ -17,6 +18,7 @@ require 'skynet_message'
 require 'message_queue_adapters/message_queue_adapter'
 require 'message_queue_adapters/tuple_space'
 require "skynet_message_queue"
+require 'skynet_partitioners'
 require 'skynet_job'
 require 'skynet_worker'
 require 'skynet_task'
@@ -32,3 +34,4 @@ end
 require 'mapreduce_test'
 require 'skynet_launcher'
 require 'skynet_console'
+require 'mapreduce_helper'

data/lib/skynet/mapreduce_helper.rb

@@ -0,0 +1,74 @@
+module MapreduceHelper
+# You can include the MapreduceHelper into your class to give you standard self.map and self.reduce methods.
+# You need only implement self.map_each and self.reduce_each methods which accept a single item (istead of an arrad)
+#
+# Example Usage:
+# This example is a bit contrived.
+# 
+#   class MapReduceTest
+#     include MapreduceHelper
+#   
+#     def self.run
+#       job = Skynet::Job.new(
+#         :mappers          => 2, 
+#         :reducers         => 1,
+#         :map_reduce_class => self,
+#         :map_data         => ['http://www.geni.com'.'http://www.yahoo.com','http://www.cnet.com']
+#       )    
+#       results = job.run
+#     end
+#   
+#     def self.map_each(url)
+#       SomeUrlSlurper.gather_results(url)  # returns an array of urls of sites that link to the given url
+#     end
+#   
+#     def self.reduce(linked_from_url)
+#       SomeUrlSluper.find_text("mysite", linked_from_url)   # finds all the times "mysite" appears in the given url, which we know links to the url given in the map_data
+#     end
+#   end
+# 
+#   MapReduceTest.run
+
+  
+  def self.included(base)
+    base.extend MapreduceHelper
+  end               
+
+  # Takes an array of map_data, iterates over that array calling self.map_each(item) for each 
+  # item in that array.   Catches exceptions in each iteration and continues processing.
+  def map(map_data_array)
+    raise Skynet::Job::BadMapOrReduceError.new("#{self.class} has no self.map_each method.") unless self.respond_to?(:map_each)
+    if map_data_array.is_a?(Array)
+      results = []
+      map_data_array.each do |data|
+        begin
+          results << map_each(data)
+        rescue Exception => e
+          error "ERROR IN #{self} [#{e.class} #{e.message}] #{e.backtrace.join("\n")}"
+        end                
+      end
+      results
+    else
+      map_each(map_data_array)
+    end
+  end
+  
+  # Takes an array of post reduce_partitioned data, iterates over that array calling self.reduce_each(item) for each 
+  # item in that array.   Catches exceptions in each iteration and continues processing.
+  def reduce(reduce_partitioned_data_array)
+    raise Skynet::Job::BadMapOrReduceError.new("#{self.class} has no self.reduce_each method.") unless self.respond_to?(:reduce_each)
+    if reduce_partitioned_data_array.is_a?(Array)
+      results = []
+      reduce_partitioned_data_array.each do |data|
+        begin
+          results << reduce_each(data)
+        rescue Exception => e
+          error "ERROR IN #{self} [#{e.class} #{e.message}] #{e.backtrace.join("\n")}"
+        end        
+      end
+      results
+    else
+      reduce_each(reduce_partitioned_data_array)
+    end
+  end  
+end

data/lib/skynet/message_queue_adapters/mysql.rb

@@ -28,7 +28,6 @@ class Skynet
       SEARCH_FIELDS = [:tasktype, :task_id, :job_id, :payload_type, :expire_time, :iteration, :version] unless defined?(SEARCH_FIELDS)
       
       Skynet::CONFIG[:MYSQL_MESSAGE_QUEUE_TEMP_CHECK_DELAY] ||= 30
-
       
       @@db_set = false
       
@@ -36,21 +35,36 @@ class Skynet
         :mysql
       end
 
-      def initialize
-        if Skynet::CONFIG[:QUEUE_DATABASE] and not @@db_set
-          begin
-            SkynetMessageQueue.establish_connection Skynet::CONFIG[:QUEUE_DATABASE]
-            SkynetWorkerQueue.establish_connection Skynet::CONFIG[:QUEUE_DATABASE]
-          rescue ActiveRecord::AdapterNotSpecified => e
-            warn "#{Skynet::CONFIG[:QUEUE_DATABASE]} not defined as a database adaptor #{e.message}"
+      def initialize     
+        if Skynet::CONFIG[:MYSQL_MESSAGE_QUEUE_TABLE]
+          SkynetMessageQueue.table_name = Skynet::CONFIG[:MYSQL_MESSAGE_QUEUE_TABLE]
+        end
+        if not @@db_set
+          if Skynet::CONFIG[:MYSQL_QUEUE_DATABASE]
+            begin
+              SkynetMessageQueue.establish_connection Skynet::CONFIG[:MYSQL_QUEUE_DATABASE]
+              SkynetWorkerQueue.establish_connection Skynet::CONFIG[:MYSQL_QUEUE_DATABASE]
+            rescue ActiveRecord::AdapterNotSpecified => e
+              error "#{Skynet::CONFIG[:MYSQL_QUEUE_DATABASE]} not defined as a database adaptor #{e.message}"
+            end
+          elsif not ActiveRecord::Base.connected?
+              db_options = {
+                :adapter  => Skynet::CONFIG[:MYSQL_ADAPTER],
+                :host     => Skynet::CONFIG[:MYSQL_HOST],
+                :username => Skynet::CONFIG[:MYSQL_USERNAME],
+                :password => Skynet::CONFIG[:MYSQL_PASSWORD],
+                :database => Skynet::CONFIG[:MYSQL_DATABASE]
+              }
+              ActiveRecord::Base.establish_connection(db_options)
           end
         end
         @@db_set = true
+        
+      end
+      
+      def message_queue_table
+        Skynet::CONFIG[:MYSQL_MESSAGE_QUEUE_TABLE] || SkynetMessageQueue.table_name
       end
-
-      # def initialize
-      #   SkynetMessageQueue.connection.execute("set session TRANSACTION ISOLATION LEVEL READ UNCOMMITTED")
-      # end
       
       def self.debug_class_desc
         "MYSQLMQ"
@@ -61,12 +75,11 @@ class Skynet
       end
 
       def template_to_conditions(template,fields=Skynet::Message.fields)
-        fields = fields.invert
         conditions = []             
         values = []            
         
-        fields.keys.each do |field|
-          value = template[fields[field]]
+        fields.each_with_index do |field,ii|
+          value = template[ii]
           next unless value
           if value.is_a?(Range)              
             conditions << "#{field} BETWEEN #{value.first} AND #{value.last}"
@@ -81,8 +94,9 @@ class Skynet
       end
             
       def message_to_hash(message,timeout=nil,fields=Skynet::Message.fields)
+        timeout ||= message.expiry        
         hash = {}
-        fields.values.each do |field|
+        fields.each do |field|
           next if field == :drburi
           # next unless message.send(field)
           if message.send(field).is_a?(Symbol)
@@ -99,57 +113,110 @@ class Skynet
         end
         hash
       end
+            
+      def write_fallback_message(message_row, message)
+        tran_id  = get_unique_id(1)
+        ftm = message.fallback_task_message            
+        update_sql = %{
+          update #{message_queue_table} 
+          SET iteration = #{ftm.iteration }, 
+          expire_time = #{ftm.expire_time}, 
+          updated_on = '#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}',
+          tran_id = #{tran_id}                   
+          WHERE id = #{message_row.id} AND iteration = #{message.iteration}
+          AND tran_id #{(message_row.tran_id ? " =#{message_row.tran_id}" : ' IS NULL')}
+        }            
+        rows = update(update_sql) || 0
+        message_row.tran_id = tran_id if rows == 1
+        rows        
+      end
+      
+      def take_next_task(curver,timeout=0.5,payload_type=nil,queue_id=0)
+        timeout = Skynet::CONFIG[:MYSQL_NEXT_TASK_TIMEOUT] if timeout < 1
+        debug "TASK NEXT TASK!!!!!!! timeout: #{timeout} queue_id:#{queue_id}"     
 
-      def take_next_task(curver,timeout=0,payload_type=nil)
-        timeout = Skynet::CONFIG[:NEXT_TASK_TIMEOUT] if timeout < 1
-        debug "TASK NEXT TASK!!!!!!! timeout: #{timeout}"     
-        message = nil
-        start = Time.now    
-        rows = nil
+        start    = Time.now    
+        template = Skynet::Message.next_task_template(curver, payload_type)        
+        message  = nil
+        
         loop do
-          # debug "start #{Time.now} timeout #{start + timeout}"
-          message_row = take(Skynet::Message.next_task_template(curver,payload_type),start,timeout)
-          next unless message_row
+          rows     = 0
+          message  = nil
+          template = Skynet::Message.next_task_template(curver, payload_type)
+          begin              
+            message_row = find_next_message(template, payload_type)
+            if message_row
+              message = Skynet::Message.new(message_row.attributes)
+              rows = write_fallback_message(message_row, message)
 
-          begin
-            message = Skynet::Message.new(message_row.attributes)
+              if rows < 1
+                old_temp = temperature(payload_type)
+                set_temperature(payload_type, template_to_conditions(template), queue_id)
+                debug "MISSCOLLISION PTYPE #{payload_type} OLDTEMP: #{old_temp} NEWTEMP: #{temperature(payload_type)}"
+              else
+                break
+              end
+            else # no messages on queue with this temp
+              old_temp = temperature(payload_type)                                                              
+              if old_temp > 1
+                set_temperature(payload_type, template_to_conditions(template), queue_id)
+              end
+              debug "MISS PTYPE #{payload_type} OLDTEMP: #{old_temp} NEWTEMP: #{temperature(payload_type)}"
+            end
           rescue Skynet::Message::BadMessage => e
-            message = nil
             message_row.destroy
             next
+          rescue ActiveRecord::StatementInvalid => e
+            if e.message =~ /Deadlock/                                 
+              old_temp = temperature(payload_type)
+              set_temperature(payload_type, template_to_conditions(template), queue_id)
+              debug "COLLISION PTYPE #{payload_type} OLDTEMP: #{old_temp} NEWTEMP: #{temperature(payload_type)}"
+            else
+              raise e
+            end
+          end
+          if Time.now.to_f > start.to_f + timeout
+            debug "MISSTIMEOUT PTYPE #{payload_type} #{temperature(payload_type)}"
+            raise Skynet::RequestExpiredError.new
+          else    
+            sleepy = rand(timeout * 0.5 )
+            debug "EMPTY QUEUE #{temperature(payload_type)} SLEEPING: #{timeout} / #{sleepy}"
+            sleep sleepy
           end
-          ftm = message.fallback_task_message        
-          rows = update("update skynet_message_queues set iteration = #{ftm.iteration }, expire_time = #{ftm.expire_time} where iteration = #{message.iteration} and id = #{message_row.id}")
-            
-          # message = nil if rows == 0
-          return message if message
         end
-      end
 
-      
+        return message
+      end
       
       def write_message(message,timeout=nil)
+        timeout ||= message.expiry
         SkynetMessageQueue.create(message_to_hash(message, timeout))
       end
       
       def write_result(message,result=[],timeout=nil)
+        timeout ||= message.expiry
         result_message = message.result_message(result)
         result_message.expire_time = nil
-        update_message(result_message,timeout)
+        update_message_with_result(result_message,timeout)
       end
       
-      def update_message(message,timeout=nil)
+      def update_message_with_result(message,timeout=nil)
+        timeout ||= message.expiry
         timeout_sql = (timeout ? ", timeout = #{timeout}, expire_time = #{Time.now.to_f + timeout}" : '')
-        rows = 0
-        rows = update(%{
-          update skynet_message_queues 
+        rows = 0      
+        raw_payload_sql = " raw_payload = "
+        raw_payload_sql << (message.raw_payload ? "'#{::Mysql.escape_string(message.raw_payload)}'" : 'NULL')
+        update_sql = %{
+          update #{message_queue_table} 
           set tasktype = "#{message.tasktype}", 
-          raw_payload = '#{message.raw_payload}',
+          #{raw_payload_sql},
           payload_type = "#{message.payload_type}",
+          updated_on = "#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}",
           tran_id = NULL
           #{timeout_sql}
           where task_id = #{message.task_id}
-          })                  
+          }                     
+        rows = update(update_sql)                  
          
         raise Skynet::RequestExpiredError.new() if rows == 0         
       end
@@ -173,6 +240,7 @@ class Skynet
           # sleep_time ||= timeout
 
           message_row = SkynetMessageQueue.find(:first,:conditions => conditions)
+          
           break if message_row
 
           if Time.now.to_f > start.to_f + timeout
@@ -203,12 +271,11 @@ class Skynet
 
       def write_error(message,error='',timeout=nil)
         message.expire_time = nil
-        update_message(message.error_message(error),timeout)
+        update_message_with_result(message.error_message(error),timeout)
       end
 
-      def write_worker_status(task, timeout=nil) 
+      def write_worker_status(task, timeout=nil)
         message = Skynet::WorkerStatusMessage.new(task)
-        worker_fields = Skynet::WorkerStatusMessage.fields.reject {|k,f| f == :process_id or f == :hostname or f == :tasksubtype or f == :tasktype}
         update_hash = message_to_hash(message, timeout, Skynet::WorkerStatusMessage.fields)
         update_hash.each do |k,v|
           if not v
@@ -226,7 +293,7 @@ class Skynet
             rows = update(insert_sql)
           rescue ActiveRecord::StatementInvalid => e
             if e.message =~ /Duplicate/
-              error "DUPLICATE WORKER #{e.message}"
+              error "DUPLICATE WORKER #{e.message} #{e.backtrace.join("\n")}"
             else
               raise e
             end
@@ -256,40 +323,53 @@ class Skynet
       end
 
 
-      def clear_worker_status(hostname=nil)
-        if hostname
-          SkynetWorkerQueue.connection.execute("delete from skynet_worker_queues where hostname = '#{hostname}'")
+      def clear_worker_status(hostname=nil)    
+        sql = "delete from skynet_worker_queues "
+        if hostname                             
+          sql << "where hostname = '#{hostname}'"
+        end
+        SkynetWorkerQueue.connection.execute(sql)        
+      end
+
+      def version_active?(curver=nil, queue_id=0)
+        return true unless curver
+        message_row = find_next_message(Skynet::Message.next_task_template(curver, nil, queue_id), :any, 1)
+        if message_row or curver.to_i == get_worker_version.to_i
+          true
         else
-          SkynetWorkerQueue.destroy_all
+          false
         end
       end
 
       def set_worker_version(ver=nil)                      
         ver ||= 1
-        # SkynetWorkerQueue.transaction do
-          # SkynetWorkerQueue.connection.execute("delete from skynet_worker_queues where tasktype = 'workerversion'")
-        SkynetWorkerQueue.connection.insert("replace skynet_worker_queues (worker_id, tasktype, version) values (0, 'workerversion',#{ver})")          
-        # end
+        SkynetMessageQueue.connection.insert("replace #{message_queue_table} (tran_id, task_id, tasktype, version) values (0, 0, 'version',#{ver})")
         ver
       end
 
       def get_worker_version
-        # ver = SkynetWorkerQueue.connection.select_value("select min(version) from skynet_message_queues where tasktype = 'task'")
-        ver = SkynetWorkerQueue.connection.select_value("select version from skynet_worker_queues where tasktype = 'workerversion'")
+        ver = SkynetMessageQueue.connection.select_value("select version from #{message_queue_table} where tran_id = 0 and tasktype = 'version'")
         if not ver
-          set_worker_version(1)
+          begin
+            SkynetMessageQueue.connection.insert("insert into #{message_queue_table} (tran_id, task_id, tasktype, version) values (0, 0, 'version', 1)")
+          rescue ActiveRecord::StatementInvalid => e
+            if e.message =~ /Duplicate/
+              return get_worker_version
+            else
+              raise e
+            end
+          end
           ver = 1
         end
         ver.to_i
       end
       
-
       def clear_outstanding_tasks
-        SkynetMessageQueue.destroy_all
+        SkynetMessageQueue.connection.execute("delete from #{message_queue_table} where tasktype = 'task'")
       end       
 
       def delete_expired_messages
-        SkynetMessageQueue.connection.delete("delete from skynet_message_queues where expire_time BETWEEN 1 AND '#{Time.now.to_f}'")
+        SkynetMessageQueue.connection.execute("delete from #{message_queue_table} where (expire_time BETWEEN 1 AND '#{Time.now.to_f}') and iteration = -1")
       end
 
 # select hostname, iteration, count(id) as number_of_workers, count(iteration) as iteration, sum(processed) as processed, max(started_at) as most_recent_task_time from skynet_worker_queues where tasksubtype = 'worker' group by hostname, iteration;      
@@ -301,27 +381,28 @@ class Skynet
 
       def stats
         stats = {
-          :servers            => {}, 
-          :results            => 0, 
-          :taken_tasks        => 0, 
-          :untaken_tasks      => 0,
-          :taken_master_tasks => 0,
-          :taken_task_tasks   => 0, 
+          :servers              => {}, 
+          :results              => 0, 
+          :taken_tasks          => 0, 
+          :untaken_tasks        => 0,
+          :taken_master_tasks   => 0,
+          :taken_task_tasks     => 0, 
           :untaken_master_tasks => 0,
           :untaken_task_tasks   => 0, 
-          :processed          => 0, 
-          :number_of_workers  => 0,
-          :active_workers     => 0,
-          :idle_workers       => 0,
-          :hosts              => 0,
-          :masters            => 0,
-          :taskworkers        => 0,
-          :time               => Time.now.to_f
+          :failed_tasks         => 0,
+          :processed            => 0, 
+          :number_of_workers    => 0,
+          :active_workers       => 0,
+          :idle_workers         => 0,
+          :hosts                => 0,
+          :masters              => 0,
+          :taskworkers          => 0,
+          :time                 => Time.now.to_f
          }
 
         stat_rows = SkynetWorkerQueue.connection.select_all(%{
           SELECT tasktype, payload_type, iteration, count(id) as number_of_tasks
-          FROM skynet_message_queues
+          FROM #{message_queue_table}
           GROUP BY tasktype, payload_type, iteration          
          })       
          # pp stat_rows
@@ -338,9 +419,11 @@ class Skynet
              if row["iteration"].to_i > 0
                stats["taken_#{type_of_tasks}".to_sym] += row["number_of_tasks"].to_i 
                stats[:taken_tasks] += row["number_of_tasks"].to_i
-             else
+             elsif row["iteration"].to_i == 0
                stats["untaken_#{type_of_tasks}".to_sym] += row["number_of_tasks"].to_i 
                stats[:untaken_tasks] += row["number_of_tasks"].to_i
+             else
+               stats[:failed_tasks] += row["number_of_tasks"].to_i 
              end
            end
          end
@@ -415,13 +498,15 @@ class Skynet
       def update(sql)                                            
         rows = 0
         3.times do
-          begin
-            rows = SkynetMessageQueue.connection.update(sql)
+          begin    
+            SkynetMessageQueue.transaction do
+              rows = SkynetMessageQueue.connection.update(sql)
+            end
             return rows
           rescue ActiveRecord::StatementInvalid => e
             if e.message =~ /Deadlock/ or e.message =~ /Transaction/
               error "#{self.class} update had collision #{e.message}"            
-              sleep 0.1
+              sleep 0.2
               next
             else
               raise e
@@ -432,83 +517,24 @@ class Skynet
       end
     
       Skynet::CONFIG[:MYSQL_TEMPERATURE_CHANGE_SLEEP] ||= 40
-
-      @@temperature ||= {}
-      @@temperature[:task] ||= 1
-      @@temperature[:master] ||= 1
-      @@temperature[:any] ||= 1
       
-      def take(template,start=Time.now,timeout=1,sleep_time=nil)      
-        conditions = template_to_conditions(template)
-        sleep_time ||= timeout
-        transaction_id = get_unique_id(1)
-        times_tried = 0
-        payload_type = template[Skynet::Message.fields.invert[:payload_type]]
-        payload_type ||= :any                                             
-        payload_type = payload_type.to_sym
-
-        10.times do         
-          begin              
-            ## TEPERATURE
-            temperature_sql = (temperature(payload_type) > 1 ? " AND id % #{temperature(payload_type).ceil} = #{rand(temperature(payload_type)).to_i} " : '')
-
-            ###  Mqke sure we get the old ones.  If we order by on ever select its VERY expensive.
-            order_by = (payload_type != :master and rand(100) < 5) ? "ORDER BY payload_type desc, created_on desc" : '' 
-            
-            sql = <<-SQL 
-              SELECT *
-              FROM skynet_message_queues 
-              WHERE #{conditions} #{temperature_sql}
-              #{order_by}
-              LIMIT 1
-            SQL
-            
-            message_row = SkynetMessageQueue.find_by_sql(sql).first
-            if message_row
-              update_conditions = "ID = #{message_row.id} and tran_id "
-              if message_row.tran_id
-                update_conditions << "= #{message_row.tran_id}"
-              else
-                update_conditions << 'IS NULL'
-              end
-              rows = SkynetMessageQueue.connection.update(
-                "UPDATE skynet_message_queues set tran_id = #{transaction_id} WHERE #{update_conditions}"
-              )
-              if rows < 1
-                old_temp = temperature(payload_type)
-                set_temperature(payload_type,conditions)
-                info "MISSCOLLISION PTYPE #{payload_type} OLDTEMP: #{old_temp} NEWTEMP: #{temperature(payload_type)}"
-                next 
-              end
-              return message_row
-            else
-              old_temp = temperature(payload_type)                                                              
-              set_temperature(payload_type,conditions)
-              info "MISS PTYPE #{payload_type} OLDTEMP: #{old_temp} NEWTEMP: #{temperature(payload_type)}"
-              break if temperature(payload_type) == 1 and old_temp == 1
-              next
-            end
-          rescue ActiveRecord::StatementInvalid => e
-            if e.message =~ /Deadlock/                                 
-              old_temp = temperature(payload_type)
-              set_temperature(payload_type,conditions)
-              info "COLLISION PTYPE #{payload_type} OLDTEMP: #{old_temp} NEWTEMP: #{temperature(payload_type)}"
-              next
-            else
-              raise e
-            end
-          end
-        end
+      def find_next_message(template, payload_type, temperature=nil)
+        conditions = template_to_conditions(template)              
+        temperature ||= temperature(payload_type)                
+        temperature_sql = (temperature > 1 ? " AND id % #{temperature.ceil} = #{rand(temperature).to_i} " : '')
 
-        if Time.now.to_f > start.to_f + timeout
-          debug "MISSTIMEOUT PTYPE #{payload_type} #{temperature(payload_type)}"
-          raise Skynet::RequestExpiredError.new
-        else    
-          sleepy = rand(sleep_time * 0.5 )
-          debug "EMPTY QUEUE #{temperature(payload_type)} SLEEPING: #{sleep_time} / #{sleepy}"
-          sleep sleepy
-          return false 
-        end
+        ###  Mqke sure we get the old ones.  If we order by on ever select its VERY expensive.
+        order_by = (payload_type != :master and rand(100) < 5) ? "ORDER BY payload_type desc, created_on desc" : '' 
+        
+        sql = <<-SQL 
+          SELECT *
+          FROM #{message_queue_table} 
+          WHERE #{conditions} #{temperature_sql}
+          #{order_by}
+          LIMIT 1
+        SQL
+        
+        SkynetMessageQueue.find_by_sql(sql).first
       end
 
       # Skynet::CONFIG[:temperature_growth_rate] ||= 2
@@ -537,37 +563,64 @@ class Skynet
       #   values[set][:total_score] += score
       # end
         
+      @@temperature ||= {}
+      @@temperature[:task] ||= 1
+      @@temperature[:master] ||= 1
+      @@temperature[:any] ||= 1
+
       def temperature(payload_type)
+        payload_type ||= :any                                             
+        payload_type   = payload_type.to_sym
         @@temperature[payload_type.to_sym]
       end
+      
+      Skynet::CONFIG[:MYSQL_QUEUE_TEMP_POW] ||= 0.6
                
-## try SQRT *2
-## try POW 0.6 or .75
-      def set_temperature(payload_type,conditions)                                            
-        temp_q_conditions = "type = '#{payload_type}' AND updated_on < '#{(Time.now - 5).strftime('%Y-%m-%d %H:%M:%S')}'"
+      def set_temperature(payload_type, conditions, queue_id=0)                                            
+        payload_type ||= :any                                             
+        payload_type   = payload_type.to_sym
+        
+        temp_q_conditions = "queue_id = #{queue_id} AND type = '#{payload_type}' AND updated_on < '#{(Time.now - 5).strftime('%Y-%m-%d %H:%M:%S')}'"
 #        "POW(#{(rand(40) + 40) * 0.01})"
 # its almost like the temperature table needs to store the POW and adjust that to be adaptive.  Like some % of the time it
 # uses the one in the table, and some % it tries a new one and scores it.
-        temperature = SkynetWorkerQueue.connection.select_value(%{select (
-          CASE WHEN (@t:=FLOOR(
-          POW(@c:=(SELECT count(*) FROM skynet_message_queues WHERE #{conditions}
-        ),0.6))) < 1 THEN 1 ELSE @t END) from skynet_queue_temperature WHERE #{temp_q_conditions} 
-        })
-        if temperature
-          update("UPDATE skynet_queue_temperature SET temperature = #{temperature} WHERE #{temp_q_conditions}")
-          @@temperature[payload_type.to_sym] = temperature.to_f
-        else
-          sleepy = rand Skynet::CONFIG[:MYSQL_TEMPERATURE_CHANGE_SLEEP]
-          sleep sleepy          
-          @@temperature[payload_type.to_sym] = SkynetWorkerQueue.connection.select_value("select temperature from skynet_queue_temperature WHERE type = '#{payload_type}'").to_f
+        begin
+          temperature = SkynetMessageQueue.connection.select_value(%{select (
+            CASE WHEN (@t:=FLOOR(
+            POW(@c:=(SELECT count(*) FROM #{message_queue_table} WHERE #{conditions}
+          ),#{Skynet::CONFIG[:MYSQL_QUEUE_TEMP_POW]}))) < 1 THEN 1 ELSE @t END) from skynet_queue_temperature WHERE #{temp_q_conditions} 
+          })
+          if temperature
+            rows = update("UPDATE skynet_queue_temperature SET temperature = #{temperature} WHERE #{temp_q_conditions}")
+            @@temperature[payload_type.to_sym] = temperature.to_f
+          else
+            sleepy = rand Skynet::CONFIG[:MYSQL_TEMPERATURE_CHANGE_SLEEP]
+            sleep sleepy          
+            @@temperature[payload_type.to_sym] = get_temperature(payload_type, queue_id)
+          end
+        rescue ActiveRecord::StatementInvalid => e
+          if e.message =~ /away/
+            ActiveRecord::Base.connection.reconnect!
+            SkynetMessageQueue.connection.reconnect!
+          end
         end
-        # update("UPDATE skynet_queue_temperature SET type = '#{payload_type}', temperature = CASE WHEN @t:=FLOOR(SQRT(select count(*) from skynet_message_queues WHERE #{conditions})) < 1 THEN 1 ELSE @t END")
-        # tasks = SkynetMessageQueue.connection.select_value("select count(*) from skynet_message_queues WHERE #{conditions}").to_i
+        # update("UPDATE skynet_queue_temperature SET type = '#{payload_type}', temperature = CASE WHEN @t:=FLOOR(SQRT(select count(*) from #{message_queue_table} WHERE #{conditions})) < 1 THEN 1 ELSE @t END")
+        # tasks = SkynetMessageQueue.connection.select_value("select count(*) from #{message_queue_table} WHERE #{conditions}").to_i
         # sleep 4 if payload_type == :tasks and tasks < 100
         # @@temperature[payload_type.to_sym] = tasks ** 0.5
         # @@temperature[payload_type.to_sym] *= multiplier
         @@temperature[payload_type.to_sym] = 1 if @@temperature[payload_type.to_sym] < 1
       end
+      
+      def get_temperature(payload_type, queue_id=0)
+        payload_type ||= :any                                             
+        payload_type   = payload_type.to_sym
+        value = SkynetMessageQueue.connection.select_value("select temperature from skynet_queue_temperature WHERE type = '#{payload_type}'").to_f
+        if not value
+          SkynetMessageQueue.connection.execute("insert into skynet_queue_temperature (queue_id,type,temperature) values (#{queue_id},'#{payload_type}',#{@@temperature[payload_type.to_sym]})")
+        end
+        value
+      end
     end
   end
 end