skynet 0.9.2 → 0.9.3

data/lib/skynet/skynet_console.rb

@@ -30,7 +30,10 @@ class Skynet
       end
       
       IRB.setup(Skynet::CONFIG[:LAUNCHER_PATH])
-      IRB.conf[:PROMPT_MODE]  = :SIMPLE
+      IRB.conf[:PROMPT][:SKYNET]  = IRB.conf[:PROMPT][:SIMPLE].dup
+      IRB.conf[:PROMPT][:SKYNET][:PROMPT_I] = "skynet>>"
+      
+      IRB.conf[:PROMPT_MODE]  = :SKYNET
       irb = IRB::Irb.new()
       IRB.conf[:MAIN_CONTEXT] = irb.context      
       irb.context.workspace.main.extend Skynet::ConsoleHelper

data/lib/skynet/skynet_console_helper.rb

@@ -1,6 +1,10 @@
 module Skynet::ConsoleHelper
 # All of these commands can be run at the 'skynet console'.
 
+    def log
+      Skynet::Logger.get
+    end
+
     def mq
       @mq ||= Skynet::MessageQueue.new
     end                       
@@ -22,7 +26,7 @@ module Skynet::ConsoleHelper
     end
 
     def manager
-      @manager ||= DRbObject.new(nil,Skynet::CONFIG[:SKYNET_LOCAL_MANAGER_URL])
+      @manager ||= Skynet::Manager.get
     end
 
     def add_lib(lib)

data/lib/skynet/skynet_debugger.rb

@@ -1,3 +1,36 @@
+# ==SkynetDebugger
+# The SkynetDebugger is a module you can include in any of your classes that will give you easy access
+# to the Skynet::Logger.  including SkynetDebugger gives you a number of logging methods.  Each logging method
+# lets you pass a message as well as an optional number of objects which will be pretty_printed after your message.
+# Log lines print with their log level, PID, time, class, and message.  eg.
+#
+#   [WARN] #78002 2008-04-11 14:17:15.363167 <WORKER-78002> Exiting... 
+#
+# You can set the log_level and log_file with  (See Skynet::Config)
+#  Skynet::CONFIG[:SKYNET_LOG_FILE]
+#  Skynet::CONFIG[:SKYNET_LOG_LEVEL]
+#
+# Possible log levels include
+#  Logger::DEBUG 
+#  Logger::INFO
+#  Logger::WARN
+#  Logger::ERROR 
+#  Logger::FATAL 
+#
+# ==Methods
+#  log - returns the Skynet::Logger
+#
+#  debug(msg,*objects_to_inspect)
+#
+#  info(msg,*objects_to_inspect)
+#
+#  warn(msg,*objects_to_inspect)
+#
+#  error(msg,*objects_to_inspect)
+#
+#  fatal(msg,*objects_to_inspect)
+# 
+#  printlog(msg,*objects_to_inspect)  #printlog will ALWAYS print to the log as log level [LOG] regardless of the LOG_LEVEL
 module SkynetDebugger
   
   def self.included(base)
@@ -12,22 +45,18 @@ module SkynetDebugger
     self.class.args_pp(*args)
   end
 
-
   def debug(*args)
     self.class.debug(*args)
   end
 
-
   def info(*args)
     self.class.info(*args)
   end
 
-
   def warn(*args)
     self.class.warn(*args)
   end
 
-
   def error(*args)
     self.class.error(*args)
   end
@@ -35,11 +64,23 @@ module SkynetDebugger
   def fatal(*args)
     self.class.fatal(*args)
   end          
+
+  def printlog(*args)
+    self.class.printlog(*args)
+  end          
   
   def debug_header
     self.class.debug_header
   end
   
+  def stderr(*args)
+    self.class.stderr
+  end
+
+  def stdout(*args)
+    self.class.stdout
+  end
+  
   module ClassMethods
     
     def debug_class_desc
@@ -51,6 +92,7 @@ module SkynetDebugger
       "##{$$} #{t.strftime("%Y-%m-%d %H:%M:%S")}.#{t.usec} <#{debug_class_desc}>"
     end
     
+    # log
     def log
       Skynet::Logger.get
     end
@@ -78,6 +120,18 @@ module SkynetDebugger
     def fatal(msg, *args)
       log.fatal "[FATAL] #{debug_header} #{msg} #{args_pp(*args)}"
     end
+
+    def printlog(msg, *args)
+      log.unknown "[LOG] #{debug_header} #{msg} #{args_pp(*args)}"
+    end
+
+    def stderr(msg, *args)
+      $stderr.puts "#{debug_header} #{msg} #{args_pp(*args)}"
+    end
+
+    def stdout(msg, *args)
+      $stdout.puts "#{debug_header} #{msg} #{args_pp(*args)}"
+    end
   
   end
 

data/lib/skynet/skynet_job.rb

@@ -9,8 +9,10 @@ class Skynet
   # There are also many global configuration options which can be controlled through Skynet::CONFIG
   #
   # Example Usage:
+  # Create a file called mapreduce_test.rb with the following.
   # 
-  #   class MapReduceTest
+  #   class MapreduceTest
+  #     include SkynetDebugger   ## This gives you logging methods such as log, error, info, fatal
   #   
   #     def self.run
   #       job = Skynet::Job.new(
@@ -41,17 +43,28 @@ class Skynet
   #     end
   #   end
   # 
-  #   MapReduceTest.run
   #
-  # You might notice that self.map and self.reduce both accept Arrays.   If you do not want to deal with
-  # getting arrays of map_data or reduce_data, you can include MapreduceHelper into your class and then
-  # implement self.map_each and self.reduce_each methods.  The included self.map and self.reduce methods
-  # will handle iterating over the map_data and reduce_data, passing each element to your map_each and
-  # reduce_each methods respectively.  They will also handle error handling within that loop to make sure
-  # even if a single map or reduce fails, processing will continue.  If you do not want processing to
-  # continue if a map fails, do not use the MapreduceHelper mixin.
+  # You need to make sure Skynet is running with your class loaded.  That's is how Skynet works.
+  # Since there is no easy way to actually pass code around the network, each skynet worker must
+  # already have your code loaded.  If you have skynet started, stop it and then start it with the -r flag
+  # to tell it where to find your class it should require.
+  #    $ skynet -r mapreduce_test.rb
+  # Then go into the skynet console to test running your map reduce task.
+  #    $ skynet console -r mapreduce_test.rb
+  #    skynet>> MapreduceTest.run     # returns {2=>2, 3=>1}
   #
-  # There are many other options to control various defaults and timeouts.
+  # In the example above, you might notice that self.map and self.reduce both accept Arrays.   
+  # If you do not want to deal with getting arrays of map_data or reduce_data, you can include MapreduceHelper
+  # into your class and then implement self.map_each and self.reduce_each methods.  
+  # The included self.map and self.reduce methods will handle iterating over the map_data and reduce_data, 
+  # passing each element to your map_each and reduce_each methods respectively.  They will also handle error 
+  # handling within that loop to make sure even if a single map or reduce fails, processing will continue.  
+  # If you do not want processing to continue if a map fails, do not use the MapreduceHelper mixin.
+  #
+  # Since Skynet must have your code, you will probably want to install skynet into the application
+  # that skynet needs access to in order to run your jobs.  See bin/skynet_install[link:files/bin/skynet_install.html] for more info.
+  #
+  # See new for the many other options to control various Skynet::Job settings.
   class Job
     include SkynetDebugger
     include Skynet::GuidGenerator		
@@ -70,7 +83,7 @@ class Skynet
               :map, :map_partitioner, :reduce, :reduce_partition, :map_reduce_class,
               :master_retry, :map_retry, :reduce_retry,
               :keep_map_tasks, :keep_reduce_tasks,
-              :local_master, :async
+              :local_master, :async, :data_debug
             ]                                                       
 
     FIELDS.each do |method| 
@@ -83,7 +96,7 @@ class Skynet
       end
     end        
     
-    attr_accessor :use_local_queue
+    attr_accessor :use_local_queue, :data_debug
     
     Skynet::CONFIG[:JOB_DEFAULTS] = {
       :queue_id              => 0,
@@ -220,7 +233,7 @@ class Skynet
     #   If a number is provided, the master will run the reduce_tasks locally if there are 
     #   LESS THAN OR EQUAL TO the number provided.
     #   You may also set Skynet::CONFIG[:DEFAILT_REDUCVE_MAP_TASKS] DEFAULT 1
-    def initialize(options = {})                            
+    def initialize(options = {})
       FIELDS.each do |field|
         if options.has_key?(field)
           self.send("#{field}=".to_sym,options[field])
@@ -314,7 +327,10 @@ class Skynet
       self.use_local_queue = map_local?
       if map_tasks        
         number_of_tasks = 0
-        map_tasks.each do |task|
+        size = map_tasks.size - 1
+        printlog "MESSAGES TO MAP ENQUEUE #{size}" if data_debug?
+        map_tasks.each_with_index do |task,ii|
+          printlog "#{size - ii} MAP TASKS LEFT TO ENQUEUE" if data_debug?
           number_of_tasks += 1
           enqueue_messages(tasks_to_messages(task))
         end            
@@ -332,8 +348,9 @@ class Skynet
     end
 
     def partition_data(post_map_data)
-      debug "RUN REDUCE 3.1 BEFORE PARTITION #{display_info} reducers: #{reducers}"
+      info "RUN REDUCE 3.1 BEFORE PARTITION #{display_info} reducers: #{reducers}"
       debug "RUN REDUCE 3.1 : #{reducers} #{name}, job_id:#{job_id}", post_map_data  
+      printlog "RUN REDUCE 3.1 : #{reducers} #{name}, job_id:#{job_id}", post_map_data if data_debug?
       return unless post_map_data
       partitioned_data = nil
       if not @reduce_partition
@@ -353,19 +370,23 @@ class Skynet
         partitioned_data = @reduce_partition.call(post_map_data, reducers)
       end
       partitioned_data.compact! if partitioned_data
-      debug "RUN REDUCE 3.2 AFTER PARTITION #{display_info} reducers: #{reducers}"
+      info "RUN REDUCE 3.2 AFTER PARTITION #{display_info} reducers: #{reducers}"
       debug "RUN REDUCE 3.2 AFTER PARTITION  #{display_info} data:", partitioned_data if partitioned_data
+      printlog "RUN REDUCE 3.2 AFTER PARTITION  #{display_info} data:", partitioned_data if data_debug?
       partitioned_data
     end
 
     def reduce_enqueue(partitioned_data)    
       return partitioned_data unless @reduce and reducers and reducers > 0
       debug "RUN REDUCE 3.3 CREATED REDUCE TASKS #{display_info}", partitioned_data
+      size = partitioned_data.size
+      printlog "REDUCE MESSAGES TO ENQUEUE #{size}" if data_debug?
       
       reduce_tasks = self.reduce_tasks(partitioned_data)
       self.use_local_queue = reduce_local?(reduce_tasks)
       number_of_tasks = 0
-      reduce_tasks.each do |task|
+      reduce_tasks.each_with_index do |task,ii|
+        printlog "#{size - ii} REDUCE TASKS LEFT TO ENQUEUE" if data_debug?
         number_of_tasks += 1
         enqueue_messages(tasks_to_messages(task))
       end            
@@ -374,20 +395,23 @@ class Skynet
     
     def reduce_results(number_of_tasks)
       results = gather_results(number_of_tasks, reduce_timeout, reduce_name)
-      if results.is_a?(Array) and results.first.is_a?(Hash)
-        hash_results = Hash.new
-        results.each {|h| hash_results.merge!(h) if h.class == Hash}
-        results = hash_results
-      elsif results.is_a?(Array) and results.first.is_a?(Array)
-        results = results.compact      
+      printlog "REDUCE RESULTS", results if data_debug?
+      if results.is_a?(Array) and results.first.is_a?(Array)
+        final = []
+        results.each do |result|
+          final += result
+        end
+        results = final
       end
       debug "RUN REDUCE 3.4 AFTER REDUCE #{display_info} results size: #{results ? results.size : ''}"
       debug "RUN REDUCE 3.4 AFTER REDUCE #{display_info} results:", results if results
+      printlog "POST REDUCE RESULTS", results if data_debug?
       return results
     end
 
     def enqueue_messages(messages)
-      messages.each do |message|
+      size = messages.size
+      messages.each_with_index do |message,ii|
         timeout = message.expiry || 5
         debug "RUN TASKS SUBMITTING #{message.name} job_id: #{job_id} #{message.payload.is_a?(Skynet::Task) ? 'task' + message.payload.task_id.to_s : ''}"        
         debug "RUN TASKS WORKER MESSAGE #{message.name} job_id: #{job_id}", message.to_a
@@ -395,6 +419,14 @@ class Skynet
       end
     end
 
+    # Given a job_id, returns the results from the message queue.  Used to retrieve results of asyncronous jobs.
+    def self.results_by_job_id(job_id,timeout=2)
+      result_message = mq.take_result(job_id,timeout)
+      result     = result_message.payload
+      return nil unless result
+      return result
+    end
+
     def gather_results(number_of_tasks, timeout=nil, description=nil)
       debug "GATHER RESULTS job_id: #{job_id} - NOT AN ASYNC JOB"
       results = {}
@@ -415,6 +447,7 @@ class Skynet
             debug "RESULT returned TASKID: #{result_message.task_id} #{results[result_message.task_id].inspect}"
           end          
           debug "RESULT collected: #{(results.keys + errors.keys).size}, remaining: #{(number_of_tasks - (results.keys + errors.keys).uniq.size)}"
+          printlog "RESULT collected: #{(results.keys + errors.keys).size}, remaining: #{(number_of_tasks - (results.keys + errors.keys).uniq.size)}" if data_debug?
           break if (number_of_tasks - (results.keys + errors.keys).uniq.size) <= 0
         end
       rescue Skynet::RequestExpiredError => e
@@ -582,6 +615,10 @@ class Skynet
 
     def single?
       @single
+    end   
+    
+    def data_debug?
+      @data_debug || Skynet::CONFIG[:SKYNET_JOB_DEBUG_DATA_LEVEL]
     end
 
     def reset!

data/lib/skynet/skynet_launcher.rb

@@ -1,14 +1,40 @@
 # FIXME: should be a module
 class Skynet
   include SkynetDebugger
-  def self.new(options={})            
+  def self.start(options={})
+    begin
+      mq = Skynet::MessageQueue.new
+    rescue Skynet::ConnectionError
+      if Skynet::MessageQueue.adapter == :tuplespace
+        ts_port = Skynet::CONFIG[:TS_SERVER_HOSTS].first.split(':').last
+        # puts "trying to make ts skynet_tuplespace_server --port=#{ts_port} --logfile=#{Skynet.config.logfile_location} --piddir=#{Skynet.config.skynet_pid_dir} --use_ringserver=#{Skynet.config.ts_use_ringserver} --drburi=#{Skynet.config.ts_drburi} start"
+        cmd = "skynet_tuplespace_server  --port=#{ts_port} --logfile=#{Skynet.config.logfile_location} --piddir=#{Skynet.config.skynet_pid_dir} --use_ringserver=#{Skynet.config.ts_use_ringserver} --drburi=#{Skynet.config.ts_drburi} start"
+        pid = fork do
+          exec(cmd)
+        end
+        sleep Skynet::CONFIG[:TS_SERVER_START_DELAY]
+      end
+    end
+
+    options[:script_path] = Skynet::CONFIG[:LAUNCHER_PATH]
+    
     if ARGV.detect {|a| a == 'console' }
       ARGV.delete('console')
       Skynet::Console.start
     elsif options[:worker_type] or ARGV.detect {|a| a =~ /worker_type/ }
       Skynet::Worker.start(options)
     else
-      Skynet::Manager.start(options)
+      if ARGV.include?('stop')
+        Skynet::Manager.stop(options)
+      else
+        options["daemonize"] = true if ARGV.include?('start')      
+        Skynet::Manager.start(options)
+      end
     end
   end
-end
+
+  def self.new(options={})
+    warn("Skynet.new is deprecated, please use Skynet.start instead")
+    start(options)
+  end
+end

data/lib/skynet/skynet_logger.rb

@@ -16,7 +16,7 @@ class Skynet
     
     def self.get
       if not @@log
-        @@log = self.new(Skynet::CONFIG[:SKYNET_LOG_FILE])
+        @@log = self.new(Skynet::Config.new.logfile_location)
         @@log.level = Skynet::CONFIG[:SKYNET_LOG_LEVEL]
       end  
       @@log
@@ -25,7 +25,13 @@ class Skynet
     def self.log=(log)
       @@log = log
     end
+
+    def printlog(*args)
+      self.class.get.unknown(*args)
+    end
+
   end
+  
     
   # This module can be mixed in to add logging methods to your class.
   module Loggable
@@ -48,5 +54,9 @@ class Skynet
     def fatal
       log = Skynet::Logger.get
     end
+    
+    def unknown
+      log = Skynet::Logger.get
+    end
   end
 end

data/lib/skynet/skynet_manager.rb

@@ -1,64 +1,87 @@
-class Skynet
-  begin
-    require 'fastthread'
-  rescue LoadError
-    # puts 'fastthread not installed, using thread instead'
-    require 'thread'
-  end
+require 'yaml'
 
+class Skynet
   class Manager
-    
+
     class Error < StandardError
     end
-    
+
     include SkynetDebugger
 
     Skynet::CONFIG[:PERCENTAGE_OF_TASK_ONLY_WORKERS]    ||= 0.7
     Skynet::CONFIG[:PERCENTAGE_OF_MASTER_ONLY_WORKERS]  ||= 0.2
-                
+
     def self.debug_class_desc
       "MANAGER"
-    end                  
-        
+    end
+
     attr_accessor :required_libs, :queue_id
-    attr_reader   :config
-    
-    def initialize(options)      
+    attr_reader   :config, :worker_queue, :wqts
+
+    def initialize(options)
       raise Error.new("You must provide a script path to Skynet::Manager.new.") unless options[:script_path]
-      @script_path          = options[:script_path]
+      @script_path          = options[:script_path] || Skynet::CONFIG[:LAUNCHER_PATH]
       # info "Skynet Launcher Path: [#{@script_path}]"
       @workers_requested    = options[:workers]  || 4
-      @required_libs        = options[:adlibs]   || []
+      @required_libs        = options[:required_libs]   || []
       @queue_id             = options[:queue_id] || 0
       @number_of_workers    = 0
       @workers_by_type      = {:master => [], :task => [], :any => []}
       @signaled_workers     = []
-      @workers_running      = {}
+      @worker_queue         = {}
       @workers_restarting   = 0
-      @all_workers_started  = false                      
-      @config               = Skynet::Config.new      
+      @all_workers_started  = false
+      @config               = Skynet::Config.new
       @mutex                = Mutex.new
-    end   
+      @wqts                 = Queue.new                      
+    end
+
+    def worker_notify(item)
+      @wqts.push(item)
+    end
+
+    def start_worker_queue_thread
+      Thread.new do
+        last_save_time = Time.now
+        loop do
+          task = @wqts.pop
+          begin
+            status = Skynet::WorkerStatusMessage.new(task)
+            status.started_at = status.started_at.to_i
+            @mutex.synchronize do          
+              @worker_queue[status.worker_id] = status
+            end
+            if last_save_time < Time.now - 60
+              save_worker_queue_to_file
+              last_save_time = Time.now
+            end
+          rescue Exception => e
+            error "Error in worker queue thread #{e.inspect} #{e.backtrace.join("\n")}"
+          end
+        end
+      end      
+    end  
     
     def start_workers
+      load_worker_queue_from_file
+      start_worker_queue_thread
+
       setup_signals
-      
+
       starting = workers_to_start(@workers_requested)
       warn "Starting #{starting} workers.  QUEUE: #{config.queue_name_by_id(queue_id)} #{@workers_requested - starting} already running."
       add_worker(starting)
     end
-    
+
     ### maybe workers_to_start should be a method
     def workers_to_start(workers_to_start)
-      pids = worker_queue_pids
-      if not pids.empty?      
-        pids.each do |worker_pid|
-          if worker_alive?(worker_pid)          
-            @workers_running[worker_pid] = Time.now
+      if not worker_pids.empty?
+        worker_pids.each do |worker_pid|
+          if worker_alive?(worker_pid)
             @number_of_workers  += 1
             workers_to_start    -= 1
           else
-            take_worker_status(worker_pid)
+            mark_worker_as_stopped(worker_pid)
           end
           return 0 if workers_to_start < 1
         end
@@ -67,242 +90,214 @@ class Skynet
     end
 
     def check_started_workers
-      workers = []                              
       begin
         100.times do |ii|
-          workers = worker_queue
-          warn "Checking started workers, #{workers.size} out of #{@number_of_workers} after the #{(ii+1)}th try..."
-          break if workers.size >= @number_of_workers        
-          sleep (@number_of_workers - workers.size)
-        end                                          
+          warn "Checking started workers, #{active_workers.size} out of #{@number_of_workers} after the #{(ii+1)}th try..."
+          break if active_workers.size >= @number_of_workers
+          sleep (@number_of_workers - active_workers.size)
+        end
       rescue Exception => e
         fatal "Something bad happened #{e.inspect} #{e.backtrace.join("\n")}"
       end
 
       @all_workers_started = true
 
-      warn "FINISHED STARTING ALL #{workers.size} WORKERS"
-      if workers.size > @number_of_workers
-        warn "EXPECTED #{@number_of_workers}" 
-        @number_of_workers = workers.size
+      printlog "FINISHED STARTING ALL #{active_workers.size} WORKERS"
+      if active_workers.size > @number_of_workers
+        warn "EXPECTED #{@number_of_workers}"
+        @number_of_workers = active_workers.size
       end
     end
-    
+
 # the main application loop
     def run
-      loop do         
+      loop do
         next unless @all_workers_started
         begin
           check_workers
           sleep Skynet::CONFIG[:WORKER_CHECK_DELAY]
         rescue SystemExit, Interrupt => e
-          fatal "Manager Exiting!"
+          printlog "Manager Exiting!"
           exit
         rescue Exception => e
           fatal "Something bad happened #{e.inspect} #{e.backtrace.join("\n")}"
         end
       end
     end
-    
+
     def check_workers
-      worker_queue = self.worker_queue
-      q_pids = worker_queue_pids(worker_queue) || []          
-      info "Checking on #{@number_of_workers} workers..." unless @shutdown
-      check_running_pids(worker_queue)
-      check_number_of_workers(worker_queue)
-      true          
-    end
-
-    def check_running_pids(worker_queue)
-      # There are workers running that are not in the queue.  When does this happen?
-      q_pids = worker_queue_pids(worker_queue) || []
-      if @workers_running.keys.size > q_pids.size
-         (@workers_running.keys - q_pids).each do |wpid|
-           error "Missing worker #{wpid} from worker queue. Removing and/or killing."
-           Process.kill("TERM",wpid) if worker_alive?(wpid)
-           @workers_running.delete(wpid)
-           q_pids.delete(wpid)          
-         end
-      end
-      
-      q_pids.each do |wpid|
+      debug "Checking on #{@number_of_workers} workers..." unless @shutdown
+      check_running_pids
+      check_number_of_workers
+      true
+    end
+
+    def check_running_pids
+      worker_pids.each do |wpid|
         if not worker_alive?(wpid)
-          error "Worker #{wpid} was in queue and but was not running.  Removing from queue."
-          take_worker_status(wpid)
-          @workers_running.delete(wpid)
+          if @shutdown
+            info "Worker #{wpid} shut down gracefully.  Removing from queue."
+          else
+            error "Worker #{wpid} was in queue and but was not running.  Removing from queue."
+          end
+          mark_worker_as_stopped(wpid)
           @number_of_workers -= 1
-          q_pids.delete(wpid)
         end
       end
-      q_pids
-    end                          
-
-    def check_number_of_workers(worker_queue=self.worker_queue)
-      q_pids = worker_queue_pids(worker_queue) || []
-      if @shutdown         
-        worker_shutdown(worker_queue)
-        if q_pids.size < 1
+      worker_pids
+    end
+
+    def check_number_of_workers
+      if @shutdown
+        worker_shutdown
+        if worker_pids.size < 1
           exit
-        end        
+        end
       elsif @workers_restarting > 0
-        if @workers_requested - q_pids.size != 0
-          restarting = @workers_requested - q_pids.size  
-          warn "RESTART MODE: Expected #{@number_of_workers} workers.  #{q_pids.size} running. #{restarting} are still restarting"          
-        else                    
-          warn "RESTART MODE: Expected #{@number_of_workers} workers.  #{q_pids.size} running."
-        end
-        @workers_restarting = @workers_requested - q_pids.size
-        
-      elsif q_pids.size != @number_of_workers
-        starting = 0        
-        if q_pids.size.to_f / @workers_requested.to_f < 0.85
-          starting = @workers_requested - q_pids.size  
-          error "Expected #{@number_of_workers} workers.  #{q_pids.size} running. Starting #{starting}"          
-          @number_of_workers = q_pids.size
-          add_worker(starting)          
-        else          
-          
-          error "Expected #{@number_of_workers} workers.  #{q_pids.size} running."
-          @number_of_workers = q_pids.size
+        if @workers_requested - worker_pids.size != 0
+          restarting = @workers_requested - worker_pids.size
+          warn "RESTART MODE: Expected #{@number_of_workers} workers.  #{worker_pids.size} running. #{restarting} are still restarting"
+        else
+          warn "RESTART MODE: Expected #{@number_of_workers} workers.  #{worker_pids.size} running."
+        end
+        @workers_restarting = @workers_requested - worker_pids.size
+
+      elsif worker_pids.size != @number_of_workers
+        starting = 0
+        if worker_pids.size.to_f / @workers_requested.to_f < 0.85
+          starting = @workers_requested - worker_pids.size
+          error "Expected #{@number_of_workers} workers.  #{worker_pids.size} running. Starting #{starting}"
+          @number_of_workers = worker_pids.size
+          add_worker(starting)
+        else
+
+          error "Expected #{@number_of_workers} workers.  #{worker_pids.size} running."
+          @number_of_workers = worker_pids.size
         end
       end
     end
-                        
-    def worker_shutdown(worker_queue)
-      q_pids = worker_queue_pids(worker_queue) || []
+
+    def worker_shutdown
       if not @masters_dead
-        workers_to_kill = worker_queue.select do |w| 
-          w.map_or_reduce == "master" and @workers_running.include?(w.process_id)
-        end                           
-        warn "Shutting down masters.  #{q_pids.size} workers still running." if q_pids.size > 0
+        workers_to_kill = active_workers.select do |w|
+          w.map_or_reduce == "master" and active_workers.detect{|status| status.process_id == w.process_id and worker_alive?(w.process_id)}
+        end
+        warn "Shutting down masters.  #{worker_pids.size} workers still running." if worker_pids.size > 0
 
         worker_pids_to_kill = workers_to_kill.collect { |w| w.process_id }
         if worker_pids_to_kill and not worker_pids_to_kill.empty?
-          warn "FOUND MORE RUNNING MASTERS WE HAVEN'T KILLED:", worker_pids_to_kill                                                    
-          remove_worker(worker_pids_to_kill)                                        
+          warn "FOUND MORE RUNNING MASTERS WE HAVEN'T KILLED:", worker_pids_to_kill
+          remove_worker(worker_pids_to_kill)
         end
 
-        if not worker_queue.detect { |w| w.map_or_reduce == "master" }
-          signal_workers("INT")
+        if not active_workers.detect { |w| w.map_or_reduce == "master" }
+          signal_workers("TERM")
           @masters_dead = true
-          sleep 1
-          return check_number_of_workers()
         else
-          sleep 4
-          return check_number_of_workers()
+          return check_number_of_workers
         end
-      else
-        warn "Shutting down.  #{q_pids.size} workers still running." if q_pids.size > 0
       end
-      if q_pids.size < 1
+      if worker_pids.size < 1
         info "No more workers running."
-      end        
-    end      
-        
-    def take_worker_status(worker_process_id)
-      begin
-        mq.take_worker_status({
-          :hostname   => hostname,
-          :process_id => worker_process_id
-        },0.00001)
-      rescue Skynet::QueueTimeout => e
-        error "Couldnt take worker status for #{hostname} #{worker_process_id}"
+      else
+        warn "Shutting down.  #{worker_pids.size} workers still running." if worker_pids.size > 0
       end
-    end 
-         
+    end
+
     def worker_alive?(worker_pid)
-      begin                   
-        IO.popen("ps -o pid,command -p #{worker_pid}", "r") do |ps|
-          return ps.detect {|line| line =~ /worker_type/}
-        end
-      rescue Errno::ENOENT => e
-        return false
-      end
-      false
-    end    
-    
+      Skynet.process_alive?(worker_pid)
+    end
 
     def add_workers(*args)
       add_worker(*args)
     end
-    
+
     def add_worker(workers=1)
       num_task_only_workers = (workers * Skynet::CONFIG[:PERCENTAGE_OF_TASK_ONLY_WORKERS]).to_i
       num_master_only_workers = (workers * Skynet::CONFIG[:PERCENTAGE_OF_MASTER_ONLY_WORKERS]).to_i
       warn "Adding #{workers} WORKERS. Task Workers: #{num_task_only_workers}, Master Workers: #{num_master_only_workers} Master & Task Workers: #{workers - num_task_only_workers - num_master_only_workers}"
-      
+
       @all_workers_started = false
       worker_types = {:task => 0, :master => 0, :any => 0}
       (1..workers).collect do |ii|
         worker_type = :any
-        if (ii <= num_master_only_workers) 
-          worker_type = :master                          
+        if (ii <= num_master_only_workers)
+          worker_type = :master
           worker_types[:master] += 1
         elsif (ii > num_master_only_workers and ii <= num_master_only_workers + num_task_only_workers)
           worker_type = :task
           worker_types[:task] += 1
         else
           worker_types[:any] += 1
-        end                          
+        end
         cmd = "#{@script_path} --worker_type=#{worker_type}"
+        cmd << " --config='#{Skynet::CONFIG[:CONFIG_FILE]}'" if Skynet::CONFIG[:CONFIG_FILE]
         cmd << " --queue_id=#{queue_id}"
         cmd << " -r #{required_libs.join(' -r ')}" if required_libs and not required_libs.empty?
-        wpid = self.fork_and_exec(cmd)
+        wpid = Skynet.fork_and_exec(cmd)
+        Skynet.close_console
         @workers_by_type[worker_type] ||= []
         @workers_by_type[worker_type] << wpid
         warn "Adding Worker ##{ii} PID: #{wpid} QUEUE: #{queue_id}, WORKER_TYPE?:#{worker_type}"
         @mutex.synchronize do
           @number_of_workers += 1
         end
-        @workers_running[wpid] = Time.now
         sleep 0.01
         wpid
-      end                  
-      info "DISTRO", worker_types
-      check_started_workers 
+      end
+      info "Worker Distribution", worker_types
+      check_started_workers
     end
-                    
+
     def remove_workers(workers=1)
-      pids = worker_queue_pids[0...workers]
+      pids = worker_pids[0...workers]
       remove_worker(pids)
     end
 
     def remove_worker(pids = nil)
       pids = [pids] unless pids.kind_of?(Array)
       info "Removing workers #{pids.join(",")} from worker queue.  They will die gracefully when they finish what they're doing."
-      wq = worker_queue
       pids.collect do |wpid|
-        @workers_running.delete(wpid)
+        Process.kill("INT",wpid)
+        mark_worker_as_stopped(wpid)
         @number_of_workers -= 1
-        @workers_running.delete(wpid)      
         warn "REMOVING WORKER #{wpid}"
         @signaled_workers << wpid
-        Process.kill("INT",wpid)      
-      end                       
+      end
       pids
     end
 
-    def signal_workers(signal,worker_type=nil)
-      worker_queue.each do |worker|
-        next if worker_type and not @workers_by_type[worker_type].include?(worker.process_id)
-        warn "SHUTTING DOWN #{worker.process_id} MR: #{worker.map_or_reduce}"
-        @workers_running.delete(worker.process_id)
-        Process.kill(signal,worker.process_id)
-        @signaled_workers << worker.process_id
-      end
-    end 
-    
-    def restart_all_workers
-      hostnames = {}
-      mq.read_all_worker_statuses.each do |status|
-        hostnames[status.hostname] = true
+    def mark_worker_as_stopped(wpid)
+      worker = @worker_queue.values.detect {|status| status.process_id == wpid}
+      if worker and not worker_alive?(wpid)
+        @worker_queue.delete_if {|worker_id, status| status.process_id == wpid }
+        worker_pids.delete(worker.process_id)
+        worker.started_at = Time.now.to_f
+        worker.process_id = nil
       end
-      hostnames.keys.each do |remote_hostname|
-        manager = DRbObject.new(nil,"druby://#{remote_hostname}:40000")
-        manager.restart_workers
+    end
+
+    def signal_workers(signal,worker_type=[])
+      worker_types = [worker_type].flatten
+      active_workers.each do |worker|
+        worker_types.each do |worker_type|
+          if worker_type == :idle
+            next if worker_type and worker.task_id
+          else
+            next if worker_type and not @workers_by_type[worker_type].include?(worker.process_id)
+          end
+        end
+        warn "SHUTTING DOWN #{worker.process_id} MR: #{worker.map_or_reduce} SIG: #{signal}"
+        begin
+          Process.kill(signal,worker.process_id)
+        rescue Errno::ESRCH
+          warn "Tried to kill a process that didn't exist #{worker.process_id}"
+        end
+        # mark_worker_as_stopped(worker.process_id)
+        @signaled_workers << worker.process_id
       end
     end
-    
+
     def hard_restart_workers
       @all_workers_started = false
       signal_workers("TERM")
@@ -319,34 +314,33 @@ class Skynet
     def restart_worker(wpid)
       info "RESTARTING WORKER #{wpid}"
       @mutex.synchronize do
-        @workers_running.delete(wpid)
+        Process.kill("HUP",wpid)
+        mark_worker_as_stopped(wpid)
         @workers_restarting += 1
       end
-      Process.kill("HUP",wpid)
       sleep Skynet::CONFIG[:WORKER_CHECK_DELAY]
     end
 
     def restart_workers
       @all_workers_started = false
       signal_workers("HUP")
-      @workers_running = {}
       sleep @number_of_workers
       check_started_workers
     end
 
     def setup_signals
-      Signal.trap("HUP")  do 
+      Signal.trap("HUP")  do
         restart_workers
       end
       Signal.trap("TERM") do
-        if @term          
+        if @term
           terminate
         else
           @term=true
           shutdown
         end
       end
-       
+
       Signal.trap("INT") do
         if @shutdown
           terminate
@@ -359,116 +353,210 @@ class Skynet
     def shutdown
       info(:shutdown)
       @shutdown = true
-      signal_workers("INT",:master)
-      signal_workers("INT",:any)
+      signal_workers("TERM",[:idle,:master,:any])
     end
 
-    def terminate             
-      info(:terminate)                  
-      signal_workers("TERM")
+    def terminate
+      info(:terminate)
+      signal_workers("KILL")
+      sleep 1
       exit
     end
 
-    def fork_and_exec(command)
-      pid = fork do                                                                  
-        exec("/bin/sh -c \"#{command}\"")
-        exit
+    def save_worker_queue_to_file
+      debug "Writing worker queue to file #{Skynet.config.manager_statfile_location}"
+      File.open(Skynet.config.manager_statfile_location,"w") do |f|
+        f.write(YAML.dump(@worker_queue))
       end
-      Process.detach(pid) if (pid != 0)
-      pid
     end
 
-    def mq
-      @mq ||= Skynet::MessageQueue.new
-    end    
+    def load_worker_queue_from_file
+      if File.exists?(Skynet.config.manager_statfile_location)
+        File.open(Skynet.config.manager_statfile_location,"r") do |f|
+          begin
+            @worker_queue = YAML.load(f.read)
+            raise Error.new("Bad Manager File returned type #{@worker_queue.class}") unless @worker_queue.is_a?(Hash)
+          rescue Exception => e
+            error "Error loading manager stats file: #{f}", e
+            @worker_queue = {}
+            save_worker_queue_to_file
+          end
+        end
+      end
+    end
 
-    def worker_queue
-      mq.read_all_worker_statuses(hostname)
+    def prune_inactive_worker_stats
+      @worker_queue.delete_if{|worker_id, worker| !worker.process_id.is_a?(Fixnum) }
+      stats
+    end
+
+    def self.stats_for_hosts(manager_hosts=nil)
+      manager_hosts ||= Skynet::CONFIG[:MANAGER_HOSTS] || ["localhost"]
+      stats = {
+        :servers           => {},
+        :processed         => 0,
+        :number_of_workers => 0,
+        :active_workers    => 0,
+        :idle_workers      => 0,
+        :hosts             => 0,
+        :masters           => 0,
+        :taskworkers       => 0,
+        :time              => Time.now.to_f
+      }
+      servers = {}
+      manager_hosts.each do |manager_host|
+        begin
+          manager = DRbObject.new(nil,"druby://#{manager_host}:#{Skynet::CONFIG[:SKYNET_LOCAL_MANAGER_PORT]}")
+          manager_stats = manager.stats
+          servers[manager_host] = manager_stats
+          manager_stats.each do |key,value|
+            next unless value.is_a?(Fixnum)
+            stats[key] ||= 0
+            stats[key] += value
+          end
+        rescue DRb::DRbConnError, Errno::ECONNREFUSED  => e
+          warn "Couldn't get stats from manager at druby://#{manager_host}:#{Skynet::CONFIG[:SKYNET_LOCAL_MANAGER_PORT]}"
+        end
+      end
+      stats[:servers] = servers
+      stats[:hosts]   = manager_hosts
+      stats
+    end
+
+    def stats
+      started_times   = @worker_queue.values.collect{|worker| worker.started_at }.sort
+      active_started_times   = active_workers.collect{|worker|worker.started_at }.sort
+      stats = {
+        :hostname                    => hostname,
+        :earliest_update             => started_times.first,
+        :latest_update               => started_times.last,
+        :active_earliest_update      => active_started_times.first,
+        :active_latest_update        => active_started_times.last,
+        :processed                   => 0,
+        :processed_by_active_workers => 0,
+        :number_of_workers           => 0,
+        :idle_workers                => 0,
+        :shutdown_workers            => 0,
+      }
+      @worker_queue.values.collect{|worker|stats[:processed] += worker.processed}
+      active_workers.collect{|worker|stats[:processed_by_active_workers] += worker.processed}
+      currently_active_workers, idle_workers = active_workers.partition{|worker| worker.map_or_reduce }
+      stats[:number_of_workers]             = active_workers.size
+      stats[:active_workers]                = currently_active_workers.size
+      stats[:idle_workers]                  = idle_workers.size
+      stats[:shutdown_workers]              = inactive_workers.size
+      stats[:masters]                       = active_workers.select{|worker|worker.tasktype.to_s == "master"}.size
+      stats[:master_or_task_workers]        = active_workers.select{|worker|worker.tasktype.to_s == "any"}.size
+      stats[:taskworkers]                   = active_workers.select{|worker|worker.tasktype.to_s == "task"}.size
+      stats[:active_masters]                = currently_active_workers.select{|worker|worker.tasktype.to_s == "master"}.size
+      stats[:active_master_or_task_workers] = currently_active_workers.select{|worker|worker.tasktype.to_s == "any"}.size
+      stats[:active_taskworkers]            = currently_active_workers.select{|worker|worker.tasktype.to_s == "task"}.size
+      stats[:idle_masters]                  = idle_workers.select{|worker|worker.tasktype.to_s == "master"}.size
+      stats[:idle_master_or_task_workers]   = idle_workers.select{|worker|worker.tasktype.to_s == "any"}.size
+      stats[:idle_taskworkers]              = idle_workers.select{|worker|worker.tasktype.to_s == "task"}.size
+      stats
+    end
+
+    def active_workers
+      @worker_queue.values.select{|status| status.process_id.is_a?(Fixnum) }
+    end
+
+    def inactive_workers
+      @worker_queue.values.select{|status| !status.process_id.is_a?(Fixnum) }
     end
-    
-    def worker_queue_pids(worker_queue=self.worker_queue)
-      worker_queue.collect {|w| w.process_id}
-    end        
 
     def worker_pids
-      worker_queue_pids
-    end           
-    
+      active_workers.collect {|w| w.process_id}
+    end
+
     def parent_pid
       $$
     end
 
     def hostname
       @machine_name ||= Socket.gethostname
-    end           
+    end
 
     def ping
       true
     end
 
+    def self.local_manager_uri
+      "druby://localhost:#{Skynet::CONFIG[:SKYNET_LOCAL_MANAGER_PORT]}"
+    end
+    
+    def self.get
+      DRbObject.new(nil,local_manager_uri)
+    end
+
     def self.start(options={})
       options[:add_workers]    ||= nil
       options[:remove_workers] ||= nil
       options[:use_rails]      ||= false
       options[:required_libs]  ||= []
-      options[:workers]        ||= Skynet::CONFIG[:NUMBER_OF_WORKERS] || 4
-      options[:pid_file]       ||= Skynet::CONFIG[:SKYNET_PIDS_FILE]
-      options[:script_path]    ||= Skynet::CONFIG[:LAUNCHER_PATH]
-
+      
       config = Skynet::Config.new
 
       OptionParser.new do |opt|
-        opt.banner = %{Usage: 
+        opt.banner = %{Usage:
         > skynet [options]
 
+        OR to daemonize
+
+        > skynet [options] start
+        > skynet stop
+
         You can also run:
         > skynet console [options]
         }
-        opt.on('', '--restart-all-workers', 'Restart All Workers') do |v| 
+        opt.on('--restart-all-workers', 'Restart All Workers') do |v|
           puts "Restarting ALL workers on ALL machines."
           begin
-            manager = DRbObject.new(nil, Skynet::CONFIG[:SKYNET_LOCAL_MANAGER_URL])
+            manager = self.get
             manager.restart_all_workers
             exit
           rescue DRb::DRbConnError => e
-            puts "No manager running at #{Skynet::CONFIG[:SKYNET_LOCAL_MANAGER_URL]}  ERROR: #{e.inspect}"
+            puts "No manager running at #{local_manager_uri}  ERROR: #{e.inspect}"
             exit
           end
         end
-        opt.on('', '--restart-workers', 'Restart Workers') do |v| 
+        opt.on('--restart-workers', 'Restart Workers') do |v|
           puts "Restarting workers on this machine."
           begin
-            manager = DRbObject.new(nil, Skynet::CONFIG[:SKYNET_LOCAL_MANAGER_URL])
+            manager = self.get
             manager.restart_workers
             exit
           rescue DRb::DRbConnError => e
-            puts "No manager running at #{Skynet::CONFIG[:SKYNET_LOCAL_MANAGER_URL]}  ERROR: #{e.inspect}"
+            puts "No manager running at #{local_manager_uri}  ERROR: #{e.inspect}"
             exit
           end
         end
-        opt.on('-i', '--increment-worker-version', 'Increment Worker Version') do |v| 
+        opt.on('--increment-worker-version', 'Increment Worker Version') do |v|
           ver = Skynet::MessageQueue.new.increment_worker_version
           puts "Incrementing Worker Version to #{ver}"
           exit
         end
-        opt.on('-a', '--add-workers WORKERS', 'Number of workers to add.') do |v| 
+        opt.on('--add-workers=WORKERS', 'Number of workers to add.') do |v|
           options[:add_workers] = v.to_i
         end
-        opt.on('-k', '--remove-workers WORKERS', 'Number of workers to remove.') do |v| 
+        opt.on('--remove-workers=WORKERS', 'Number of workers to remove.') do |v|
           options[:remove_workers] = v.to_i
         end
-        opt.on('-w', '--workers WORKERS', 'Number of workers to start.') do |v| 
+        opt.on('--workers=WORKERS', 'Number of workers to start.') do |v|
           options[:workers] = v.to_i
-        end               
+        end
         opt.on('-r', '--required LIBRARY', 'Require the specified libraries') do |v|
           options[:required_libs] << File.expand_path(v)
         end
-        opt.on('-q', '--queue QUEUE_NAME', 'Which queue should these workers use (default "default").') do |v| 
+        opt.on('--config=CONFIG_FILE', 'Where to find the skynet.rb config file') do |v|
+          options[:config_file] = File.expand_path(v)
+        end
+        opt.on('--queue=QUEUE_NAME', 'Which queue should these workers use (default "default").') do |v|
           options[:queue] = v
-        end               
-        opt.on('-i', '--queue_id queue_id', 'Which queue should these workers use (default 0).') do |v| 
+        end
+        opt.on('--queue_id=queue_id', 'Which queue should these workers use (default 0).') do |v|
           options[:queue_id] = v.to_i
-        end               
+        end
         opt.parse!(ARGV)
       end
       if options[:queue]
@@ -477,9 +565,9 @@ class Skynet
         end
         options[:queue_id] = config.queue_id_by_name(options[:queue])
       else
-        options[:queue_id] ||= 0        
+        options[:queue_id] ||= 0
       end
-      
+
       options[:required_libs].each do |adlib|
         begin
           require adlib
@@ -487,12 +575,28 @@ class Skynet
           error "The included lib #{adlib} was not found: #{e.inspect}"
           exit
         end
-      end        
+      end
+
+      options[:config_file] ||= Skynet::CONFIG[:CONFIG_FILE]
+      if options[:config_file]
+        begin
+          require options[:config_file]
+        rescue MissingSourceFile => e
+          error "The config file at #{options[:config_file]} was not found: #{e.inspect}"
+          exit
+        end
+      elsif Skynet::CONFIG[:SYSTEM_RUNNER]
+        error "Config file missing. Please add a config/skynet_config.rb before starting."        
+      end
+
+      options[:workers]        ||= Skynet::CONFIG[:NUMBER_OF_WORKERS] || 4
+      options[:pid_file]       ||= Skynet::Config.pidfile_location
+      options[:script_path]    ||= Skynet::CONFIG[:LAUNCHER_PATH]
 
       # Handle add or remove workers
       if options[:add_workers] or options[:remove_workers]
         begin
-          manager = DRbObject.new(nil, Skynet::CONFIG[:SKYNET_LOCAL_MANAGER_URL])    
+          manager = self.get
           if options[:add_workers]
             pids = manager.add_worker(options[:add_workers])
             warn "ADDING #{options[:add_workers]} workers PIDS: #{pids.inspect}"
@@ -501,7 +605,7 @@ class Skynet
             warn "REMOVING #{options[:remove_workers]} workers PIDS: #{pids.inspect}"
           end
         rescue DRb::DRbConnError => e
-          warn "Couldnt add or remove workers. There are probably no workers running. At least I couldn't find a skynet_manager around at #{Skynet::CONFIG[:SKYNET_LOCAL_MANAGER_URL]} #{e.inspect}"
+          warn "Couldnt add or remove workers. There are probably no workers running. At least I couldn't find a skynet_manager around at #{local_manager_uri} #{e.inspect}"
         rescue Exception => e
           warn "Couldnt add or remove workers #{e.inspect} #{e.backtrace.join("\n")}"
         end
@@ -519,25 +623,84 @@ class Skynet
 
         debug "CONTINUING TO START : There IS an available MessageQueue", options
 
-        # create main pid file
-        File.open(options[:pid_file], 'w') do |file|
-          file.puts($$)
-        end
+        begin
+          if oldpid = read_pid_file
+            errmsg = nil
+            if Skynet.process_alive?(oldpid)
+              errmsg = "Another Skynet Manager is running at pid: #{oldpid}"
+              warn errmsg
+              stderr errmsg
+              exit
+            else
+              errmsg =  "Deleting stale pidfile #{Skynet::Config.pidfile_location}"
+              warn errmsg
+              stderr errmsg
+              File.unlink(Skynet::Config.pidfile_location) if File.exist?(Skynet::Config.pidfile_location)
+            end
+          end
 
-        begin                                                                                                                   
-          info "STARTING THE MANAGER!!!!!!!!!!!"
-          @manager = Skynet::Manager.new(options)
-          DRb.start_service(Skynet::CONFIG[:SKYNET_LOCAL_MANAGER_URL], @manager)
-          info "WORKER MANAGER URI: #{DRb.uri}"
-          @manager.start_workers
-          @manager.run
-          DRb.thread.join
+          printlog "STARTING THE MANAGER!!!!!!!!!!! port: #{Skynet::CONFIG[:SKYNET_LOCAL_MANAGER_PORT]}"
+          puts "Starting Skynet..."
+          printlog "Skynet Stopped"
+          if options["daemonize"]
+            Skynet.safefork do
+              sess_id = Process.setsid
+              write_pid_file
+              Skynet.close_console
+              run_manager(options)
+              exit!
+            end
+          else
+            write_pid_file
+            run_manager(options)
+          end
         rescue SystemExit, Interrupt
         rescue Exception => e
-          fatal("Error in Manager.  Manager Dying. #{e.inspect} #{e.backtrace}")        
+          fatal("Error in Manager.  Manager Dying. #{e.inspect} #{e.backtrace}")
         end
       end
     end
 
+    def self.run_manager(options)
+      @manager = Skynet::Manager.new(options)
+      @drb_manager = DRb.start_service("druby://:#{Skynet::CONFIG[:SKYNET_LOCAL_MANAGER_PORT]}", @manager)
+      @manager.start_workers
+      info "MANAGER STARTED ON PORT: #{Skynet::CONFIG[:SKYNET_LOCAL_MANAGER_PORT]}"
+      @manager.run
+    end
+
+    # stop the daemon, nicely at first, and then forcefully if necessary
+    def self.stop(options = {})
+      pid = read_pid_file
+      if not pid
+        puts "The Skynet Manager is not running. No PID found in #{Skynet::Config.pidfile_location}"
+        exit
+      end
+      $stdout.puts "Stopping Skynet"
+      printlog "Stopping Skynet"
+      Process.kill("TERM", pid)
+      180.times { Process.kill(0, pid); sleep(1) }
+      Process.kill("TERM", pid)
+      180.times { Process.kill(0, pid); sleep(1) }
+      $stdout.puts("using kill -9 #{pid}")
+      Process.kill("KILL", pid)
+    rescue Errno::ESRCH => e
+      printlog "Skynet Stopped"
+    ensure
+      File.unlink(Skynet::Config.pidfile_location) if File.exist?(Skynet::Config.pidfile_location)
+    end
+
+    def self.read_pid_file
+      pidfile = Skynet::Config.pidfile_location
+      File.read(pidfile).to_i if File.exist?(pidfile)
+    end
+
+    def self.write_pid_file
+      pidfile = Skynet::Config.pidfile_location
+      info "Writing PIDFILE to #{pidfile}"
+      open(pidfile, "w") {|f| f << Process.pid << "\n"}
+      at_exit { File.unlink(pidfile) if read_pid_file == Process.pid }
+    end
+
   end
 end