timocratic-skynet 0.9.4

data/lib/skynet/skynet_partitioners.rb

@@ -0,0 +1,96 @@
+class Skynet
+                                       # Collection of partitioning utilities
+  class Partitioners
+    include SkynetDebugger
+
+    # Split one block of data into partitions
+    #
+    class SimplePartitionData < Partitioners
+
+      def self.reduce_partition(data, partitions)    
+        partitioned_data = Array.new
+
+        # If data size is significantly greater than the number of desired
+        # partitions, we can divide the data roughly but the last partition
+        # may be smaller than the others.
+        #      
+        return data if (not data) or data.empty?
+    
+        if partitions >= data.length
+          data.each do |datum|
+           partitioned_data << [datum]
+          end
+        elsif (data.length >= partitions * 2)
+          # Use quicker but less "fair" method
+          size = data.length / partitions
+
+          if (data.length % partitions != 0)
+            size += 1 # Last slice of leftovers
+          end
+
+          (0..partitions - 1).each do |i|
+            partitioned_data[i] = data[i * size, size]
+          end
+        else
+          # Slower method, but partitions evenly
+          partitions = (data.size < partitions ? data.size : partitions)
+          (0..partitions - 1).each { |i| partitioned_data[i] = Array.new }
+  
+          data.each_with_index do |datum, i|
+            partitioned_data[i % partitions] << datum
+          end
+        end
+  
+        partitioned_data
+      end
+    end
+
+
+    class RecombineAndSplit < Partitioners
+      # Tries to be smart about what kind of data its getting, whether array of arrays or array of arrays of arrays.
+      #
+      def self.reduce_partition(post_map_data,new_partitions)
+        return post_map_data unless post_map_data.is_a?(Array) and (not post_map_data.empty?) and post_map_data.first.is_a?(Array)
+        ### Why did I do this?  It breaks badly.
+        # if not post_map_data.first.first.is_a?(Array)
+        #   partitioned_data = post_map_data.flatten
+        # else
+          partitioned_data = post_map_data.inject(Array.new) do |data,part| 
+            data += part
+          end    
+        # end    
+        partitioned_data = Skynet::Partitioners::SimplePartitionData.reduce_partition(partitioned_data, new_partitions)
+        debug "POST PARTITIONED DATA_SIZE", partitioned_data.size
+        debug "POST PARTITIONED DATA", partitioned_data
+        partitioned_data
+      end
+    end
+    
+    class ArrayDataSplitByFirstEntry < Partitioners
+      # Smarter partitioner for array data, generates simple sum of array[0]
+      # and ensures that all arrays sharing that key go into the same partition.
+      #
+      def self.reduce_partition(post_map_data, new_partitions)
+        partitions = []
+        (0..new_partitions - 1).each { |i| partitions[i] = Array.new }
+        cnt = 0
+        post_map_data.each do |partition|
+          partition.each do |array|            
+            next unless array.is_a?(Array) and array.size >= 2
+            if array[0].kind_of?(Fixnum)
+              key = array[0] % new_partitions
+            elsif array[0].kind_of?(String)
+              key = array[0].sum % new_partitions
+            else                 
+              cnt += 1
+              key = cnt % new_partitions
+            end
+            partitions[key] << array
+          end
+        end
+        partitions
+      end
+    end
+
+  end
+end

data/lib/skynet/skynet_ruby_extensions.rb

@@ -0,0 +1,53 @@
+module Enumerable    
+  def mapreduce(klass=nil,options={},&block)               
+    data = []
+    if self.is_a?(Hash)
+      self.each {|k,v| data << {k => v}}
+    else
+      data = self
+    end
+    jobopts = {
+      :mappers                => 20000,
+      :map_data               => data,
+      :name                   => "#{klass} Enumerable MASTER",
+      :map_name               => "#{klass} Enumerable MAP",
+      :reduce_name            => "#{klass} Enumerable REDUCE",
+      :map_timeout            => 3600,
+      :reduce_timeout         => 3600,
+      :master_timeout         => 3600,
+      :master_result_timeout  => 3600
+    }                                
+
+    jobopts[:map_reduce_class] = klass.to_s if klass
+
+    options.each { |k,v| jobopts[k] = v }
+    if block_given?
+      jobopts[:map] = block
+    end                               
+    
+    if block_given? or not jobopts[:async]
+      job = Skynet::Job.new(jobopts.merge(:local_master => true))
+    else
+      job = Skynet::AsyncJob.new(jobopts)
+    end
+    job.run    
+  end  
+end
+
+class String
+  ### THIS IS TAKEN DIRECTLY FROM ActiveSupport::Inflector
+  # Constantize tries to find a declared constant with the name specified
+  # in the string. It raises a NameError when the name is not in CamelCase
+  # or is not initialized.
+  #
+  # Examples
+  #   "Module".constantize #=> Module
+  #   "Class".constantize #=> Class
+  def constantize
+    unless /\A(?:::)?([A-Z]\w*(?:::[A-Z]\w*)*)\z/ =~ self
+      raise NameError, "#{camel_cased_word.inspect} is not a valid constant name!"
+    end
+
+    Object.module_eval("::#{$1}", __FILE__, __LINE__)
+  end
+end

data/lib/skynet/skynet_task.rb

@@ -0,0 +1,118 @@
+class Skynet
+  class Task
+    include SkynetDebugger
+
+    class ConstructorError < StandardError; end
+    class TimeoutError     < StandardError; end
+    
+    attr_reader :data, :process, :result, :map_or_reduce, :marshalable
+    attr_accessor :name, :result_timeout, :retry
+    
+    @@log = nil
+    
+    def self.debug_class_desc
+      "TASK"
+    end
+    
+    def self.master_task(job)
+      options = {
+        :async        => false,
+        :local_master => true,
+        :map_name     => job.map_name || job.name,
+        :reduce_name  => job.reduce_name || job.name,          
+      }
+      Skynet::Job::FIELDS.each do |field|
+        next if options.has_key?(field)
+        options[field] = job.send(field) if job.send(field)
+      end
+      
+      master_job = Skynet::Job.new(options)
+      
+      self.new(
+        :task_id        => master_job.task_id, 
+        :data           => nil, 
+        :process        => master_job.to_h, 
+        :map_or_reduce  => :master,
+        :name           => master_job.name,
+        :result_timeout => master_job.master_timeout,
+        :retry          => master_job.master_retry || Skynet::CONFIG[:DEFAULT_MASTER_RETRY]
+      )
+    end
+    
+    def initialize(opts = {})
+      unless opts[:task_id] and opts[:process] and opts[:map_or_reduce]
+        raise ConstructorError.new("Must provide task_id, process and map_or_reduce")      
+      end
+      @marshalable    = true
+      @task_id        = opts[:task_id].to_i
+      @data           = opts[:data]
+      self.process    = opts[:process]
+      @name           = opts[:name]
+      @map_or_reduce  = opts[:map_or_reduce]
+      @result_timeout = opts[:result_timeout]
+      @retry          = opts[:retry]
+    end
+    
+    def process=(process)
+      if process.is_a?(Proc)
+        @marshalable = false
+      end
+      @process = process
+    end  
+    
+    def can_marshal?
+      @marshalable
+    end
+    
+    def task_or_master
+      if @map_or_reduce == :master
+        @map_or_reduce 
+      else
+        :task
+      end
+    end
+
+    def task_id
+      @task_id.to_i
+    end
+    
+    def run(iteration=nil)
+      info "running task #{name} TIMEOUT: #{result_timeout} task_id:#{task_id} MorR:#{map_or_reduce} PROCESS CLASS: #{@process.class}"
+      begin
+        Timeout::timeout(@result_timeout) do
+          if @process.class == Proc
+            debug " - #{@map_or_reduce} using Proc"
+            @process.call @data
+          elsif @map_or_reduce == :master
+            debug " - as master"
+            job = Skynet::Job.new(@process)
+            job.run
+          elsif @process.class == String
+            debug " - #{@map_or_reduce} using class #{@process}"
+            @process.constantize.send(@map_or_reduce,@data)
+          end
+        end
+      rescue Timeout::Error => e
+        # ==========
+        # = XXX NEWSFEED HACK
+        # = I'm printing the data hash, but that hash has all this shit added to it after runing through newsfeed.
+        # = It's actually nice to be able to see what was added, but sometimes its too much data.
+        # = Though the handy part will be adding instrumentation to the event_hash and seeing it onyl during a timeout.
+        # ==========
+        
+        if @data.is_a?(Array) and @data.first.is_a?(Hash)          
+          @data.each {|h|h.delete(:event_object)}                      
+        end
+        raise TimeoutError.new("TASK TIMED OUT! #{name} IT:[#{iteration}] timeout:#{@result_timeout} #{e.inspect} DATA: #{@data.inspect} #{e.backtrace.join("\n")}")
+
+      # ==========
+      # = XXX This rescue block is probably not necessary.  Just for debugging for now. =
+      # ==========
+      rescue Exception => e
+        error "Error running task #{e.inspect} TASK:", self, e.backtrace.join("\n")
+        raise e
+      end
+    end
+    
+  end  ## END class Task
+end

data/lib/skynet/skynet_tuplespace_server.rb

@@ -0,0 +1,83 @@
+#!/usr/bin/env ruby -w
+
+require 'rinda/ring'
+require 'rinda/tuplespace'
+require 'rubygems'
+require 'logger'
+require 'optparse'
+require 'pp'
+
+class Rinda::TupleSpaceProxy
+  def take(tuple, sec=nil, &block)
+    port = []
+    port.push @ts.move(nil, tuple, sec, &block)
+    port[0]
+  end
+end
+
+class Rinda::Tuple
+
+  require 'ostruct'
+
+  def init_with_ary(ary)
+    if ary.instance_of?(DRb::DRbUnknown)
+      begin
+        Marshal.load(ary.buf)
+      rescue Exception => e
+        raise Rinda::RindaError.new("DRb couldn't marshall tuple of type #{ary.name}, it was turned into a DRb::DRbUnknown object.\nMarshal exception #{e.inspect}\nOriginal object:\n\t#{ary.buf}.\n\nStacktrace:\n")
+      end
+    else
+      @tuple = Array.new(ary.size)
+      @tuple.size.times do |i|
+        @tuple[i] = ary[i]
+      end
+    end
+  end
+end
+
+class Skynet
+  class Task
+  end
+  class Message
+    class Payload
+    end
+  end
+
+  class AsyncJob
+  end
+
+  class Job
+  end
+
+  class TuplespaceServer
+
+    def self.start(options)
+      options[:port] ||= 7647
+      log = Logger.new(options[:logfile])
+      log.level = Object.module_eval("#{"Logger::" + options[:loglevel].upcase}", __FILE__, __LINE__) if options[:loglevel]
+      log.info "STARTING TUPLESPACE SERVER ON PORT: #{options[:port]} Logging to #{options[:logfile]}"
+
+      # Create a TupleSpace to hold named services, and start running
+      ts = Rinda::TupleSpace.new
+      begin
+        if options[:use_ringserver] and options[:port]
+          DRb.start_service
+          tuple = [:name,:TupleSpace, ts, 'Tuple Space']
+          renewer = Rinda::SimpleRenewer.new
+          ring_ts = Rinda::TupleSpace.new
+          ring_ts.write(tuple, renewer)
+
+          server = Rinda::RingServer.new(ring_ts, options[:port])
+        end
+        if options[:drburi]
+          DRb.start_service(options[:drburi], ts)          
+        end
+        DRb.thread.join
+      rescue SystemExit, Interrupt
+      rescue Exception, RuntimeError => e
+        log.fatal "Couldn't start Skynet Server #{e.inspect}"
+      end
+
+    end
+  end
+end

data/lib/skynet/skynet_worker.rb

@@ -0,0 +1,451 @@
+class Skynet
+  class Worker
+
+    include SkynetDebugger
+    include Skynet::GuidGenerator
+
+    RETRY_TIME = 2
+    Skynet::CONFIG[:WORKER_VERSION_CHECK_DELAY] ||= 30
+
+    Skynet::CONFIG[:WORKER_MAX_MEMORY] ||= 500
+
+    MEMORY_CHECK_DELAY = 30
+    MANAGER_PING_INTERVAL = 60
+
+    attr_accessor :message,:task, :mq, :processed
+    attr_reader :worker_id, :worker_info, :worker_type, :queue_id
+
+    class Error             < StandardError; end
+    class RespawnWorker     < Skynet::Error; end
+    class ConnectionFailure < Skynet::Error; end
+    class NoManagerError    < Skynet::Error; end
+
+    def self.debug_class_desc
+      "WORKER-#{$$}"
+    end
+
+    def initialize(worker_type, options = {})
+      @worker_id    = get_unique_id(1).to_i
+      @worker_type  = worker_type.to_sym
+      @queue_id     = options[:queue_id] || 0
+      @processed    = 0
+      @in_process   = false
+      @mq           = Skynet::MessageQueue.new
+
+      debug "THIS WORKER TAKES #{worker_type}"
+
+      @worker_info = {
+        :tasktype     => worker_type,
+        :hostname     => hostname,
+        :process_id   => process_id,
+        :worker_type  => payload_type,
+        :worker_id    => worker_id,
+        :version      => mq.get_worker_version,
+      }
+      @worker_info.merge!(options)
+    end
+
+    def process_id
+      $$
+    end
+
+    def hostname
+      @machine_name ||= Socket.gethostname
+    end
+
+    def version
+      @curver
+    end
+
+    def new_version_respawn?
+       if !@verchecktime
+        @verchecktime = Time.now
+        begin
+          @curver = mq.get_worker_version
+          debug "FINDING INITIAL VER #{@curver}"
+        rescue  Skynet::RequestExpiredError => e
+          warn "NO INITIAL VER IN MQ using 1"
+          @curver = 1
+        end
+      else
+        if Time.now < (@verchecktime + Skynet::CONFIG[:WORKER_VERSION_CHECK_DELAY])
+          return false
+        else
+          @verchecktime = Time.now
+          begin
+            newver = mq.get_worker_version
+            # debug "CURVER #{@curver} NEWVER: #{newver}"
+            if newver != @curver and not mq.version_active?(@curver, queue_id)
+              info "RESTARTING WORKER ON PID #{$$}"
+              return true
+            end
+          rescue Skynet::RequestExpiredError => e
+            warn "NO CURRENT WORKER REV IN MQ still using 1"
+            mq.set_worker_version(1)
+            return false
+          end
+        end
+      end
+      return false
+    end
+
+    def notify_worker_started
+      write_worker_status(
+        @worker_info.merge({
+          :name       => "waiting for #{@worker_type}",
+          :processed  => 0,
+          :started_at => Time.now.to_i
+        })
+      )
+    end
+
+    def notify_task_begun(task)
+      task[:processed] = @processed
+      task[:started_at] = Time.now.to_i
+      @in_process = true      
+      write_worker_status(@worker_info.merge(task))
+    end
+
+    def notify_task_complete
+      @processed += 1
+      @in_process = false
+
+      write_worker_status(
+        @worker_info.merge({
+          :task_id       => 0,
+          :job_id        => 0,
+          :name          => "waiting for #{@worker_type}",
+          :processed     => @processed,
+          :map_or_reduce => nil,
+          :started_at    => Time.now.to_i
+        })
+      )
+    end
+
+    def notify_worker_stop
+      info "Worker #{process_id} stopping..."
+      write_worker_status(
+      @worker_info.merge({
+        :task_id       => 0,
+        :job_id        => 0,
+        :name          => "waiting for #{@worker_type}",
+        :processed     => @processed,
+        :process_id    => nil,
+        :map_or_reduce => nil,
+        :started_at    => Time.now.to_i
+        })
+      )
+    end           
+    
+    def manager_send(method,*args)
+      begin
+        manager.send(method,*args)
+      rescue DRb::DRbConnError, Errno::ECONNREFUSED  => e
+        error "Worker could not connect to manager to call #{method} on manager #{e.inspect}"
+      rescue Exception  => e
+        error "Worker could not connect call #{method} on manager #{e.inspect} args:", args
+      end
+    end
+
+    def write_worker_status(status)
+      manager_send(:worker_notify,status)
+    end                              
+    
+    def manager
+      Skynet::Manager.get
+    end
+    
+    def payload_type
+      return nil if worker_type == :any
+      return worker_type
+    end
+
+    def interrupt
+      if @die
+        exit
+      else
+        @die = true
+        if not @in_process
+          notify_worker_stop
+          exit
+        end
+      end
+    end
+
+    def start
+      exceptions = 0
+      conerror   = 0
+      @curver    = nil
+
+      # setup signal handlers for manager
+      Signal.trap("HUP")  do
+        @respawn = true
+        raise Skynet::Worker::RespawnWorker.new if not @in_process
+      end
+      Signal.trap("TERM") { interrupt       }
+      Signal.trap("INT")  { interrupt       }
+
+      raise Skynet::Worker::RespawnWorker.new if new_version_respawn?
+
+      printlog "STARTING WORKER @ VER:#{@curver} type:#{@worker_type} QUEUE_ID:#{queue_id}"
+
+      notify_worker_started
+
+      message = nil
+      task    = nil
+
+      loop do
+        message = nil
+        begin
+          if Skynet::CONFIG[:WORKER_MAX_PROCESSED] and Skynet::CONFIG[:WORKER_MAX_PROCESSED] > 0 and @processed >= Skynet::CONFIG[:WORKER_MAX_PROCESSED]
+            raise Skynet::Worker::RespawnWorker.new("WORKER OVER MAX MEM AT: #{get_memory_size} MAX: #{Skynet::CONFIG[:WORKER_MAX_MEMORY]}")
+          end
+          if @die
+            exit
+          elsif @respawn
+            raise Skynet::Worker::RespawnWorker.new()
+          end
+
+          if local_mem = max_memory_reached?
+            raise Skynet::Worker::RespawnWorker.new("WORKER OVER MAX MEM AT: #{local_mem} MAX: #{Skynet::CONFIG[:WORKER_MAX_MEMORY]}")
+          end
+
+          if conerror > 0
+            @mq = Skynet::MessageQueue.new
+            warn "WORKER RECONNECTED AFTER #{conerror} tries"
+            conerror = 0
+          end
+
+          # debug "1 START LOOPSSS at VER #{@curver}"
+          #
+          # debug "LOOK FOR WORK USING TEMPLATE", Skynet::Message.task_template(@curver)
+          # message = Skynet::Message.new(mq.take(Skynet::Message.task_template(@curver),0.00001))
+          message = mq.take_next_task(@curver, 0.00001, payload_type, queue_id)
+
+          next unless message.respond_to?(:payload)
+
+          task = message.payload
+          error "BAD MESSAGE", task unless task.respond_to?(:map_or_reduce)
+
+          info "STEP 2 GOT MESSAGE #{message.name} type:#{task.map_or_reduce}, jobid: #{message.job_id}, taskid:#{message.task_id} it: #{message.iteration}"
+          debug "STEP 2.1 message=", message.to_a
+          # info "STEP 3 GOT TASK taskid: #{task.task_id}"
+          # debug "STEP 3.1 task=", task
+          next unless task
+          # maybe instead of putting a time in the future, it puts the start time and an offset in seconds
+
+          # task.debug "taking task #{task.task_id} name:#{task.name}..."
+
+          info "STEP 4 RUNNING TASK #{message.name} jobid: #{message.job_id} taskid: #{task.task_id}"
+          notify_task_begun({
+            :job_id        => message.job_id,
+            :task_id       => message.task_id,
+            :iteration     => message.iteration,
+            :name          => message.name,
+            :map_or_reduce => task.map_or_reduce
+          })
+          result = task.run(message.iteration)
+
+          info "STEP 5 GOT RESULT FROM RUN TASK #{message.name} jobid: #{message.job_id} taskid: #{task.task_id}"
+          debug "STEP 5.1 RESULT DATA:", result
+
+          result_message = mq.write_result(message,result,task.result_timeout)
+          info "STEP 6 WROTE RESULT MESSAGE #{message.name} jobid: #{message.job_id} taskid: #{task.task_id}"
+          # debug "STEP 6.1 RESULT_MESSAGE:", result_message
+          notify_task_complete
+
+        rescue Skynet::Task::TimeoutError => e
+          error "Task timed out while executing #{e.inspect} #{e.backtrace.join("\n")}"
+          @in_process = false
+          next
+
+        rescue Skynet::Worker::RespawnWorker => e
+          info "Respawning and taking worker status #{e.message}"
+          notify_worker_stop
+          raise e
+
+        rescue Skynet::RequestExpiredError => e
+          if new_version_respawn?
+            notify_worker_stop
+            manager_send(:restart_worker,$$)
+          end
+          sleep 1
+          next
+
+        rescue Skynet::ConnectionError, DRb::DRbConnError => e
+          conerror += 1
+          retry_time = conerror > 6 ? RETRY_TIME * 3 : RETRY_TIME
+          error "#{e.message}, RETRY #{conerror} in #{retry_time} seconds !!"
+          @mq = nil
+          sleep retry_time
+          if conerror > 20
+            fatal "TOO MANY RECONNECTION EXCEPTIONS #{e.message}"
+            notify_worker_stop
+            raise e
+          end
+          next
+
+        rescue NoManagerError => e
+          fatal e.message
+          break
+        rescue Interrupt, SystemExit => e
+          info "Exiting..."
+          notify_worker_stop
+          break
+        rescue Exception => e
+          error "skynet_worker.rb:#{__LINE__} #{e.inspect} #{e.backtrace.join("\n")}"
+          exceptions += 1
+          break if exceptions > 1000
+          #mq.take(@next_worker_message.task_template,0.0005) if message
+          if message
+            mq.write_error(message,"#{e.inspect} #{e.backtrace.join("\n")}",(task.respond_to?(:result_timeout) ? task.result_timeout : 200))
+          else
+            # what do we do here
+            # mq.write_error(message,"ERROR in WORKER [#{$$}] #{e.inspect} #{e.backtrace.join("\n")}")
+          end
+          # mq.write_error("ERROR in WORKER [#{$$}] #{e.inspect} #{e.backtrace.join("\n")}")
+          @in_process = false
+          next
+        end
+      end
+    end
+
+    @@ok_to_mem_check = false
+    @@lastmem = nil
+    @@memct = 0
+
+    def max_memory_reached?
+      return false unless ok_to_mem_check?
+       if !@memchecktime
+        @memchecktime = Time.now
+        return false
+      elsif Time.now > (@memchecktime + MEMORY_CHECK_DELAY)
+        @memchecktime = Time.now
+        local_mem = get_memory_size.to_i
+        return local_mem if local_mem > Skynet::CONFIG[:WORKER_MAX_MEMORY]
+      else
+        false
+      end
+    end
+
+    def find_pid_size(file, format=:notpretty)
+      begin
+        open(file).each { |line|
+          if line.index('VmSize')
+            temp = line[7..-5].strip.to_f/1000
+            return BigDecimal(temp.to_s).truncate(5).to_s('F') if format == :pretty
+            return temp
+          end
+        }
+      rescue Exception => e
+        warn "ERROR #{e.inspect}"
+        '0'
+      end
+    end
+
+    def get_memory_size
+      find_pid_size("/proc/self/status")
+    end
+
+    def ok_to_mem_check?
+      return true if @@ok_to_mem_check == true
+      return false if @@ok_to_mem_check == :notok
+      if File.exists?('/proc/self/status')
+      @@lastmem ||= get_memory_size.to_i
+      return @@ok_to_mem_check = true
+      else
+        @@ok_to_mem_check = :notok
+        return false
+      end
+    end
+
+    def self.start(options={})
+      options[:worker_type]    ||= :any
+      options[:required_libs]  ||= []
+
+      OptionParser.new do |opt|
+        opt.banner = "Usage: worker [options]"
+        opt.on('-r', '--required LIBRARY', 'Include the specified libraries') do |v|
+          options[:required_libs] << v
+        end
+        opt.on('--worker_type=WORKERTYPE', "master, task or any") do |v|
+          if ["any","master","task"].include?(v)
+            options[:worker_type] = v
+          else
+            raise Skynet::Error.new("#{v} is not a valid worker_type")
+          end
+        end
+        opt.on('--config=CONFIG_FILE', 'Where to find the skynet.rb config file') do |v|
+          options[:config_file] = File.expand_path(v)
+        end
+        opt.on('--queue=QUEUE_NAME', 'Which queue should these workers use (default "default").') do |v|
+          options[:queue] = v
+        end
+        opt.on('--queue_id=queue_id', 'Which queue should these workers use (default 0).') do |v|
+          options[:queue_id] = v.to_i
+        end
+        opt.parse!(ARGV)
+      end
+
+      if options[:queue]
+        if options[:queue_id]
+          raise Skynet::Error.new("You may either provide a queue_id or a queue, but not both.")
+        end
+        options[:queue_id] = config.queue_id_by_name(options[:queue])
+      end
+
+      options[:required_libs].each do |adlib|
+        begin
+          require adlib
+        rescue MissingSourceFile => e
+          error "The included lib #{adlib} was not found: #{e.inspect}"
+          exit
+        end
+      end
+
+      options[:config_file] ||= Skynet::CONFIG[:CONFIG_FILE]
+      if options[:config_file]
+        begin
+          require options[:config_file]
+        rescue MissingSourceFile => e
+          error "The config file at #{options[:config_file]} was not found: #{e.inspect}"
+          exit
+        end
+      else
+        error "Config file missing. Please add a config/skynet_config.rb before starting."        
+        exit
+      end
+
+      debug "WORKER STARTING WORKER_TYPE?:#{options[:worker_type]}. QUEUE: #{Skynet::Config.new.queue_name_by_id(options[:queue_id])}"
+
+      begin
+        worker = Skynet::Worker.new(options[:worker_type], options)
+        worker.start
+      rescue Skynet::Worker::NoManagerError => e
+        fatal e.message
+        exit
+      rescue Skynet::Worker::RespawnWorker => e
+        warn "WORKER #{$$} SCRIPT CAUGHT RESPAWN.  RESTARTING #{e.message}"
+        cmd = "ruby #{Skynet::CONFIG[:LAUNCHER_PATH]} --worker_type=#{options[:worker_type]} --queue_id=#{options[:queue_id]} "
+        cmd << "--config=#{options[:config_file]} "
+        cmd << "-r #{options[:required_libs].join(' -r ')}" if options[:required_libs] and not options[:required_libs].empty?
+        pid = Skynet.fork_and_exec(cmd)
+        exit
+      rescue SystemExit
+        info "WORKER #{$$} EXITING GRACEFULLY"
+      rescue Exception => e
+        fatal "WORKER #{$$} DYING #{e.class} #{e.message} #{e.backtrace}"
+        report = ExceptionReport.new(e)
+        report.save
+      end
+    end
+  end
+end
+
+class ExceptionReport
+  def initialize(*args)
+  end
+
+  def save
+  end
+end