timocratic-skynet 0.9.4

data/lib/skynet/skynet_config.rb

@@ -0,0 +1,232 @@
+class Skynet       
+
+  CONFIG = {
+    :ENABLE                               => true,
+    :SOLO                                 => false,
+    :APP_ROOT                             => nil,
+    :SKYNET_LOG_DIR                       => nil,
+    :SKYNET_PID_DIR                       => nil,
+    :SKYNET_PID_FILE                      => "skynet.pid",
+    :SKYNET_LOG_FILE                      => "skynet.log",
+    :SKYNET_MANAGER_STATS_FILE            => "skynet_manager_stats.txt",
+    :SKYNET_LOG_LEVEL                     => Logger::ERROR,
+    :SKYNET_LOCAL_MANAGER_PORT            => "40000",
+    :MESSAGE_QUEUE_ADAPTER                => ("Skynet::MessageQueueAdapter::TupleSpace" || "Skynet::MessageQueueAdapter::Mysql"),
+    :TS_USE_RINGSERVER                    => true,
+    :TS_DRBURI                            => "druby://localhost:47647",   # If you do not use RINGSERVER, you must specifiy the DRBURI
+    :TS_SERVER_HOSTS                      => ["localhost:7647"],
+    :TS_SERVER_START_DELAY                => 10,
+    # :MYSQL_QUEUE_DATABASE                 => "skynet_queue",
+    :MYSQL_TEMPERATURE_CHANGE_SLEEP       => 40,
+    :MYSQL_MESSAGE_QUEUE_TAPLE            => "skynet_message_queues",
+    :MYSQL_MESSAGE_QUEUE_TEMP_CHECK_DELAY => 40,
+    :MYSQL_NEXT_TASK_TIMEOUT              => 60,
+    :MYSQL_ADAPTER                        => "mysql",
+    :MYSQL_HOST                           => "localhost",
+    :MYSQL_DATABASE                       => nil, # 'skynet'
+    :MYSQL_USERNAME                       => nil,
+    :MYSQL_PASSWORD                       => "",
+    :NUMBER_OF_WORKERS                    => 4,
+    :WORKER_CHECK_DELAY                   => 5,
+    :WORKER_MAX_MEMORY                    => 500,
+    :WORKER_MAX_PROCESSED                 => 1000,
+    :WORKER_VERSION_CHECK_DELAY           => 30,
+    # :GUID_GENERATOR                     => nil,
+    :PERCENTAGE_OF_TASK_ONLY_WORKERS      => 0.7,
+    :PERCENTAGE_OF_MASTER_ONLY_WORKERS    => 0.2,
+    :MAX_RETRIES                          => 6,
+    :DEFAULT_MASTER_RETRY                 => 0,
+    :DEFAULT_MAP_RETRY                    => 3,
+    :DEFAULT_REDUCE_RETRY                 => 3,
+    :DEFAULT_KEEP_MAP_TASKS               => 1,
+    :DEFAULT_KEEP_REDUCE_TASKS            => 1,
+    :MESSAGE_QUEUES                       => ['first', 'second', 'third', 'fourth', 'fifth', 'sixth', 'seventh', 'eighth', 'nineth']
+  } unless defined?(CONFIG)
+                           
+  def self.silent
+    if block_given?
+      Skynet.configure(:SKYNET_LOG_LEVEL => 10) do
+        yield
+      end
+    else
+      raise Error.new("Please provide a block to Skynet.silent")
+    end
+  end
+  
+  def self.configure(config={})
+    old_config = CONFIG.dup
+    config.each {|k,v| CONFIG[k] = v}
+    Skynet::Logger.log = nil
+    if block_given?
+      ret = yield
+      CONFIG.keys.each do |key|
+        CONFIG.delete(key)
+      end
+      old_config.each {|k,v| CONFIG[k] = v}
+      ret
+    end
+  end
+  
+  def self.config
+    Skynet::Config.new
+  end
+
+  def self.solo(config = {}) 
+    raise Skynet::Error.new("You provide a code block to Skynet.solo") unless block_given?
+    result = nil
+    Skynet::Logger.log = nil
+    begin
+      config[:ENABLE]              = true
+      config[:SOLO]                = true
+      config[:SKYNET_LOG_FILE]   ||= STDOUT
+      config[:SKYNET_LOG_LEVEL]  ||= Logger::ERROR
+      configure(config) do
+        result = yield
+      end
+    rescue Exception => e
+      error "Something bad happened #{e.inspect} #{e.backtrace.join("\n")}"
+    end
+    return result
+  end
+  
+  # Skynet has many global configuration options.
+  # You can access specific options via Skynet::CONFIG[:OPTION] = ?
+  # You can set many options via                                          
+  #
+  #    Skynet.configure(:OPTION => option, :ANOTHEROPTION => anotheroption)
+  #
+  # If you want specific configuration to only apply to a block of code you can pass configure a block
+  #
+  #    Skynet.configure(:SOMEOPTION => 'value') do
+  #       run code here
+  #    end
+  #
+  # Config Options and current defaults:
+  #  Skynet.configure(
+  #   :ENABLE                               => true,
+  #   :SOLO                                 => false,
+  #   :SKYNET_PID_DIR                       => "/tmp",
+  #   :SKYNET_PID_FILE                      => "skynet.pid",
+  #   :SKYNET_LOG_DIR                       => ENV["HOME"],
+  #   :SKYNET_LOG_FILE                      => "skynet.log",
+  #   :SKYNET_LOG_LEVEL                     => Logger::ERROR,
+  #   :SKYNET_LOCAL_MANAGER_URL             => "druby://localhost:40000",
+  #   :MESSAGE_QUEUE_ADAPTER                => "Skynet::MessageQueueAdapter::TupleSpace",
+  #   :TS_DRBURI                            => "druby://localhost:47647"
+  #   :TS_USE_RINGSERVER                    => true,
+  #   :TS_SERVER_HOSTS                      => ["localhost:7647"],
+  #   :TS_SERVER_START_DELAY                => 10,
+  #   :MYSQL_QUEUE_DATABASE                 => "skynet_queue",
+  #   :MYSQL_TEMPERATURE_CHANGE_SLEEP       => 40,
+  #   :MYSQL_QUEUE_TEMP_POW                 => 0.6,
+  #   :MYSQL_MESSAGE_QUEUE_TAPLE            => "skynet_message_queues",
+  #   :MYSQL_MESSAGE_QUEUE_TEMP_CHECK_DELAY => 40,
+  #   :MYSQL_NEXT_TASK_TIMEOUT              => 60,
+  #   :WORKER_CHECK_DELAY                   => 40,
+  #   :GUID_GENERATOR                       => nil,
+  #   :NUMBER_OF_WORKERS                    => 4,
+  #   :PERCENTAGE_OF_TASK_ONLY_WORKERS      => 0.7,
+  #   :PERCENTAGE_OF_MASTER_ONLY_WORKERS    => 0.2,
+  #   :MAX_RETRIES                          => 6,
+  #   :DEFAULT_MASTER_RETRY                 => 0,
+  #   :DEFAULT_MAP_RETRY                    => 3,
+  #   :DEFAULT_REDUCE_RETRY                 => 3
+  #   :KEEP_MAP_TASKS                       => false,
+  #   :KEEP_REDUCE_TASKS                    => false
+  #  )
+  #
+  # ENABLE turns Skynet on or off.
+  #
+  # SOLO turns on SOLO mode where Skynet can run in the current process without workers.  
+  # Its almost a Skynet emulation mode
+  #
+  # MESSAGE_QUEUE_ADAPTER
+  # Skynet comes with 2 message queue adaptors 
+  #   Skynet::MessageQueueAdapter::TupleSpace 
+  #   Skynet::MessageQueueAdapter::Mysql
+  # The default is TupleSpace which is an in memory database.
+  # The mysql MQ takes running a migration that comes with skynet_install
+  #
+  # The following only apply to the TupleSpace adapter
+  #   :TS_DRBURI                            => "druby://localhost:47647"
+  #   :TS_USE_RINGSERVER                    => true,
+  #   :TS_SERVER_HOSTS                      => ["localhost:7647"],
+  #
+  # The following only apply to the Mysql adapter
+  #   :MYSQL_QUEUE_DATABASE                 => "skynet_queue",
+  #   :MYSQL_TEMPERATURE_CHANGE_SLEEP       => 40,
+  #   :MYSQL_QUEUE_TEMP_POW                 => 0.6,
+  #   :MYSQL_MESSAGE_QUEUE_TAPLE            => "skynet_message_queues",
+  #   :MYSQL_MESSAGE_QUEUE_TEMP_CHECK_DELAY => 40,
+  #   :MYSQL_NEXT_TASK_TIMEOUT              => 60,
+
+  class Config     
+    include Enumerable
+
+    def each
+      Skynet::CONFIG.each {|k,v| yield k,v}
+    end
+
+    def self.add_message_queue(queue_name)
+      self.message_queues << queue_name
+    end
+    
+    def self.queue_id_by_name(queue_name)
+      if Skynet::CONFIG[:MESSAGE_QUEUES].index(queue_name)
+        return Skynet::CONFIG[:MESSAGE_QUEUES].index(queue_name)
+      else     
+        raise Skynet::Error("#{queue_name} is not a valid queue")
+      end
+    end       
+    
+    def self.queue_name_by_id(queue_id)               
+      queue_id = queue_id.to_i
+      if Skynet::CONFIG[:MESSAGE_QUEUES][queue_id]
+        return Skynet::CONFIG[:MESSAGE_QUEUES][queue_id]
+      else     
+        raise Skynet::Error("#{queue_id} is not a valid queue_id")
+      end
+    end
+    
+    def self.logfile_location
+      if skynet_log_file.is_a?(String) and skynet_log_dir
+        skynet_log_dir.sub(/\/$/,'') + "/" + skynet_log_file.sub(/^\//,'')
+      else
+        skynet_log_file
+      end
+    end
+
+    def self.pidfile_location
+      if skynet_pid_dir and skynet_pid_file
+        skynet_pid_dir.sub(/\/$/,'') + "/" + skynet_pid_file.sub(/^\//,'')
+      else
+        skynet_pid_dir
+      end
+    end
+    
+    def manager_statfile_location
+      if skynet_log_dir.is_a?(String) and skynet_log_dir
+        skynet_log_dir.sub(/\/$/,'') + "/" + skynet_manager_stats_file.sub(/^\//,'')
+      else
+        skynet_log_dir
+      end
+    end    
+    def method_missing(name,*args)
+      if self.class.respond_to?(name)
+        self.class.send(name,*args)
+      else        
+        self.class.method_missing(name,*args)
+      end
+    end                               
+    
+    def self.method_missing(name, *args)
+      name = name.to_s.upcase.to_sym
+      if name.to_s =~ /^(.*)=$/
+        name = $1.to_sym
+        Skynet::CONFIG[name] = args.first
+      else
+        Skynet::CONFIG[name]
+      end
+    end
+  end
+end

data/lib/skynet/skynet_console.rb

@@ -0,0 +1,50 @@
+class Skynet
+  class Console    
+    def self.start(libs=[])
+      require 'rubygems'
+      require 'optparse'
+      require 'skynet'
+      require "skynet_console_helper"
+      require "irb/completion"
+      require "irb"
+
+      irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
+
+      options = { 
+        :irb           => irb, 
+        :required_libs => [] 
+      }
+      
+      OptionParser.new do |opt|
+        opt.banner = "Usage: skynet console [options]"
+        opt.on("--irb=[#{irb}]", 'Invoke a different irb.') { |v| options[:irb] = v }
+        opt.on('-r', '--required LIBRARY', 'Require the specified libraries. To include multiple libraries, include multiple -r options. ie. -r skynet -r fileutils') do |v|
+          options[:required_libs] << File.expand_path(v)
+        end
+        opt.parse!(ARGV)
+      end
+                         
+      options[:required_libs] + libs
+      options[:required_libs].uniq.each do |lib|
+        require lib
+      end
+      
+      IRB.setup(Skynet::CONFIG[:LAUNCHER_PATH])
+      IRB.conf[:PROMPT][:SKYNET]  = IRB.conf[:PROMPT][:SIMPLE].dup
+      IRB.conf[:PROMPT][:SKYNET][:PROMPT_I] = "skynet>>"
+      
+      IRB.conf[:PROMPT_MODE]  = :SKYNET
+      irb = IRB::Irb.new()
+      IRB.conf[:MAIN_CONTEXT] = irb.context      
+      irb.context.workspace.main.extend Skynet::ConsoleHelper
+
+      trap("SIGINT") do
+        irb.signal_handle
+      end
+
+      catch(:IRB_EXIT) do
+        irb.eval_input
+      end      
+    end
+  end
+end

data/lib/skynet/skynet_console_helper.rb

@@ -0,0 +1,66 @@
+module Skynet::ConsoleHelper
+# All of these commands can be run at the 'skynet console'.
+
+    def log
+      Skynet::Logger.get
+    end
+
+    def mq
+      @mq ||= Skynet::MessageQueue.new
+    end                       
+
+    def stats
+      mq.stats
+    end
+
+    def increment_worker_version
+      mq.increment_worker_version
+    end
+
+    def get_worker_version
+      mq.get_worker_version
+    end
+
+    def set_worker_version(*args)
+      mq.set_worker_version(*args)
+    end
+
+    def manager
+      @manager ||= Skynet::Manager.get
+    end
+
+    def add_lib(lib)
+      manager.required_libs << File.expand_path(lib)
+      manager.restart_workers
+    end                      
+
+    def restart_workers
+      manager.restart_workers
+    end
+
+    def add_workers(num)
+      manager.add_workers(num)
+    end
+
+    def remove_workers(num)
+      manager.remove_workers(num)
+    end
+
+    # ===============
+    # = Doesnt work =
+    # ===============
+    # def help
+    #   puts <<-HELP
+    #    mq
+    #    stats
+    #    increment_worker_version
+    #    get_worker_version
+    #    set_worker_version(version)
+    #    manager
+    #    add_lib(library_to_include)  -- forces a restart
+    #    restart_workers
+    #    add_workers(number_of_workers)
+    #    remove_workers(number_of_workers)
+    #   HELP
+    # end
+end

data/lib/skynet/skynet_debugger.rb

@@ -0,0 +1,138 @@
+# ==SkynetDebugger
+# The SkynetDebugger is a module you can include in any of your classes that will give you easy access
+# to the Skynet::Logger.  including SkynetDebugger gives you a number of logging methods.  Each logging method
+# lets you pass a message as well as an optional number of objects which will be pretty_printed after your message.
+# Log lines print with their log level, PID, time, class, and message.  eg.
+#
+#   [WARN] #78002 2008-04-11 14:17:15.363167 <WORKER-78002> Exiting... 
+#
+# You can set the log_level and log_file with  (See Skynet::Config)
+#  Skynet::CONFIG[:SKYNET_LOG_FILE]
+#  Skynet::CONFIG[:SKYNET_LOG_LEVEL]
+#
+# Possible log levels include
+#  Logger::DEBUG 
+#  Logger::INFO
+#  Logger::WARN
+#  Logger::ERROR 
+#  Logger::FATAL 
+#
+# ==Methods
+#  log - returns the Skynet::Logger
+#
+#  debug(msg,*objects_to_inspect)
+#
+#  info(msg,*objects_to_inspect)
+#
+#  warn(msg,*objects_to_inspect)
+#
+#  error(msg,*objects_to_inspect)
+#
+#  fatal(msg,*objects_to_inspect)
+# 
+#  printlog(msg,*objects_to_inspect)  #printlog will ALWAYS print to the log as log level [LOG] regardless of the LOG_LEVEL
+module SkynetDebugger
+  
+  def self.included(base)
+    base.extend ClassMethods
+  end
+
+  def log
+    self.class.log
+  end  
+
+  def args_pp(*args)
+    self.class.args_pp(*args)
+  end
+
+  def debug(*args)
+    self.class.debug(*args)
+  end
+
+  def info(*args)
+    self.class.info(*args)
+  end
+
+  def warn(*args)
+    self.class.warn(*args)
+  end
+
+  def error(*args)
+    self.class.error(*args)
+  end
+
+  def fatal(*args)
+    self.class.fatal(*args)
+  end          
+
+  def printlog(*args)
+    self.class.printlog(*args)
+  end          
+  
+  def debug_header
+    self.class.debug_header
+  end
+  
+  def stderr(*args)
+    self.class.stderr
+  end
+
+  def stdout(*args)
+    self.class.stdout
+  end
+  
+  module ClassMethods
+    
+    def debug_class_desc
+      self.to_s
+    end
+    
+    def debug_header
+      t = Time.now
+      "##{$$} #{t.strftime("%Y-%m-%d %H:%M:%S")}.#{t.usec} <#{debug_class_desc}>"
+    end
+    
+    # log
+    def log
+      Skynet::Logger.get
+    end
+
+    def args_pp(*args)
+      "#{args.length > 0 ? args.pretty_print_inspect : ''}"
+    end      
+
+    def debug(msg,*args)
+      log.debug "[DEBUG] #{debug_header} #{msg} #{args_pp(*args)}"
+    end
+
+    def info(msg, *args)
+      log.info "[INFO] #{debug_header} #{msg} #{args_pp(*args)}"
+    end
+
+    def warn(msg, *args)
+      log.warn "[WARN] #{debug_header} #{msg} #{args_pp(*args)}"
+    end
+
+    def error(msg, *args)
+      log.error "[ERROR] #{debug_header} #{msg} #{args_pp(*args)}"
+    end
+
+    def fatal(msg, *args)
+      log.fatal "[FATAL] #{debug_header} #{msg} #{args_pp(*args)}"
+    end
+
+    def printlog(msg, *args)
+      log.unknown "[LOG] #{debug_header} #{msg} #{args_pp(*args)}"
+    end
+
+    def stderr(msg, *args)
+      $stderr.puts "#{debug_header} #{msg} #{args_pp(*args)}"
+    end
+
+    def stdout(msg, *args)
+      $stdout.puts "#{debug_header} #{msg} #{args_pp(*args)}"
+    end
+  
+  end
+
+end