timocratic-skynet 0.9.4

data/Rakefile

@@ -0,0 +1,5 @@
+require 'config/requirements'
+require 'config/hoe' # setup Hoe + all gem configuration
+
+require 'newgem/tasks'
+Dir['tasks/**/*.rake'].each { |rake| load rake }

data/app_generators/skynet_install/USAGE

@@ -0,0 +1,5 @@
+Description:
+  
+  
+Usage:
+  

data/app_generators/skynet_install/skynet_install_generator.rb

@@ -0,0 +1,94 @@
+class SkynetInstallGenerator < RubiGen::Base
+  
+  DEFAULT_SHEBANG = File.join(Config::CONFIG['bindir'],
+                              Config::CONFIG['ruby_install_name'])
+  
+  default_options :in_rails => false
+  default_options :mysql    => false
+  
+  attr_reader :name
+  attr_reader :in_rails
+  attr_reader :mysql
+  
+  def initialize(runtime_args, runtime_options = {})
+    super
+    usage if args.empty?
+    @destination_root = File.expand_path(args.shift)
+    @name = base_name
+    extract_options
+  end
+
+  def manifest
+    record do |m|
+
+      # Ensure appropriate folder(s) exists
+      BASEDIRS.each { |path| m.directory path }
+      
+      # Create stubs
+      m.template  "skynet_config.rb", "config/skynet_config.rb", :collision => :ask, :chmod => 0655
+      if @in_rails
+        m.directory 'config/initializers'
+        m.template  "skynet_initializer.rb", "config/initializers/skynet.rb", :collision => :ask, :chmod => 0655
+      end
+      if @mysql
+        m.directory 'db/migrate'
+        m.template   "skynet_mysql_schema.sql", "db/skynet_mysql_schema.sql", :collision => :ask, :chmod => 0655
+        m.migration_template "migration.rb", "db/migrate", 
+          :collision => :ask, 
+          :assigns => {
+              :migration_name => "CreateSkynetTables"
+          },  :migration_file_name => "create_skynet_tables"
+      end
+    end
+  end
+
+  protected
+    def banner
+      <<-EOS
+Creates a ...
+
+USAGE: #{spec.name} [--rails] [--mysql] directory (can be '.' for current)"
+Installs: 
+  ./config/skynet_config.rb
+EOS
+    end
+
+    def add_options!(opts)
+      opts.separator ''
+      opts.separator 'Options:'
+      # For each option below, place the default
+      # at the top of the file next to "default_options"
+      opts.on("-v", "--version", "Show the #{File.basename($0)} version number and quit.")
+      opts.on("--mysql", 
+             "Include mysql migration if you want to use mysql as your message queue.  
+             Installs:
+             ./db/skynet_mysql_schema.sql
+             ./db/migrate/db/migrate/###_create_skynet_tables.rb
+             ") do |mysql|
+               options[:mysql] = true if mysql
+             end
+      opts.on("-r", "--rails",
+              "Install into rails app.
+              Installs:
+              ./config/initializers/skynet.rb
+              (If using rails 1, make sure to add require 'skynet' to your environment.rb)",
+              "Default: false") do |rails| 
+                options[:rails] = true if rails
+              end
+    end
+    
+    def extract_options
+      # for each option, extract it into a local variable (and create an "attr_reader :author" at the top)
+      # Templates can access these value via the attr_reader-generated methods, but not the
+      # raw instance variable value.
+      @in_rails = options[:rails]
+      @mysql    = options[:mysql]
+    end
+
+    # Installation skeleton.  Intermediate directories are automatically
+    # created so don't sweat their absence here.
+    BASEDIRS = %w(
+      config
+      log
+    )
+end

data/app_generators/skynet_install/templates/migration.rb

@@ -0,0 +1,43 @@
+class <%= migration_name %> < ActiveRecord::Migration
+  def self.up
+    create_table :skynet_message_queues do |t|
+      t.column  :id,            "bigint unsigned primary key"
+      t.column  :queue_id,      :integer, :default => 0
+      t.column  :tran_id,       "bigint unsigned"
+      t.column  :created_on,    :timestamp
+      t.column  :updated_on,    :timestamp
+      t.column  :tasktype,      :string
+      t.column  :task_id,       'bigint unsigned'
+      t.column  :job_id,        'bigint unsigned'
+      t.column  :raw_payload,   "longtext"
+      t.column  :payload_type,  :string
+      t.column  :name,          :string
+      t.column  :expiry,        :integer
+      t.column  :expire_time,   "decimal(16,4)"
+      t.column  :iteration,     :integer
+      t.column  :version,       :integer
+      t.column  :timeout,       "decimal(16,4)"
+      t.column  :retry,         :integer, :default => 0
+    end                                   
+    create_table :skynet_queue_temperature do |t|
+      t.column  :id,            "bigint unsigned primary key"
+      t.column  :queue_id,      :integer, :default => 0      
+      t.column  :updated_on,    :timestamp
+      t.column  :count,         :integer, :default => 0
+      t.column  :temperature,   "decimal(6,4) default 1"
+      t.column  :type,          :string
+    end
+    add_index :skynet_message_queues, :job_id
+    add_index :skynet_message_queues, :task_id
+    add_index :skynet_message_queues, :tran_id, :unique => true
+    add_index :skynet_message_queues, [:queue_id,:tasktype,:payload_type,:expire_time], :name => "index_skynet_mqueue_for_take"
+    execute "insert into skynet_queue_temperature (queue_id,type) values (0,'master')"
+    execute "insert into skynet_queue_temperature (queue_id,type) values (0,'any')"
+    execute "insert into skynet_queue_temperature (queue_id,type) values (0,'task')"
+  end
+
+  def self.down
+    drop_table :skynet_queue_temperature
+    drop_table :skynet_message_queues
+  end
+end

data/app_generators/skynet_install/templates/skynet_config.rb

@@ -0,0 +1,50 @@
+# Skynet Configuration File.  Should be in APP_ROOT/config/skynet_config.rb  
+# Start skynet from within your app root with 
+# > skynet start
+
+require 'rubygems'
+require 'skynet'         
+
+<% if in_rails -%>
+# Load your rails app
+if not defined?(RAILS_GEM_VERSION)
+  require File.expand_path(File.dirname(__FILE__)) + '/../config/environment'
+end
+<% end -%>
+
+Skynet::CONFIG[:SKYNET_LOG_LEVEL] = Logger::ERROR
+<% if in_rails -%>
+Skynet::CONFIG[:APP_ROOT]         = RAILS_ROOT
+Skynet::CONFIG[:SKYNET_LOG_DIR]   = File.expand_path(Skynet::CONFIG[:APP_ROOT]+ "/log")
+Skynet::CONFIG[:SKYNET_PID_DIR]   = File.expand_path(Skynet::CONFIG[:APP_ROOT] + "/log")
+Skynet::CONFIG[:SKYNET_LOG_FILE]  = "skynet_#{RAILS_ENV}.log"
+Skynet::CONFIG[:SKYNET_PID_FILE]  = "skynet_#{RAILS_ENV}.pid"
+<% else -%>
+Skynet::CONFIG[:SKYNET_LOG_DIR]   = File.expand_path(File.dirname(__FILE__) + "/../log") 
+Skynet::CONFIG[:SKYNET_PID_DIR]   = File.expand_path(File.dirname(__FILE__) + "/../log") 
+<% end -%>
+
+<% if mysql -%>
+<% if not in_rails -%>
+# Use the mysql message queue adapter
+Skynet::CONFIG[:MESSAGE_QUEUE_ADAPTER] = "Skynet::MessageQueueAdapter::Mysql"
+Skynet::CONFIG[:MYSQL_HOST]            = "localhost"
+Skynet::CONFIG[:MYSQL_USERNAME]        = "root"
+Skynet::CONFIG[:MYSQL_PASSWORD]        = ""
+Skynet::CONFIG[:MYSQL_DATABASE]        = "skynet"
+<% else %>
+# Use the mysql message queue adapter
+Skynet::CONFIG[:MESSAGE_QUEUE_ADAPTER] = "Skynet::MessageQueueAdapter::Mysql"
+<% end -%>
+<% end -%>
+
+# ==================================================================
+# = Require any other libraries you want skynet to know about here =
+# ==================================================================
+
+
+# ===========================================
+# = Set your own configuration options here =
+# ===========================================
+# You can also configure skynet with
+# Skynet.configure(:SOME_CONFIG_OPTION => true, :SOME_OTHER_CONFIG => 3)

data/app_generators/skynet_install/templates/skynet_initializer.rb

@@ -0,0 +1 @@
+require RAILS_ROOT+'/config/skynet_config.rb'

data/app_generators/skynet_install/templates/skynet_mysql_schema.sql

@@ -0,0 +1,33 @@
+CREATE TABLE skynet_message_queues (
+  id int(11) NOT NULL auto_increment,
+  queue_id int(11) default '0',
+  tran_id bigint(20) unsigned default NULL,
+  created_on datetime default NULL,
+  updated_on datetime default NULL,
+  tasktype varchar(255) default NULL,
+  task_id bigint(20) unsigned default NULL,
+  job_id bigint(20) unsigned default NULL,
+  raw_payload longtext,
+  payload_type varchar(255) default NULL,
+  name varchar(255) default NULL,
+  expiry int(11) default NULL,
+  expire_time decimal(16,4) default NULL,
+  iteration int(11) default NULL,
+  version int(11) default NULL,
+  timeout decimal(16,4) default NULL,
+  retry int(11) default '0',
+  PRIMARY KEY  (id),
+  UNIQUE KEY index_skynet_message_queues_on_tran_id (tran_id),
+  KEY index_skynet_message_queues_on_job_id (job_id),
+  KEY index_skynet_message_queues_on_task_id (task_id),
+  KEY index_skynet_mqueue_for_take (queue_id,tasktype,payload_type,expire_time)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+CREATE TABLE skynet_queue_temperature (
+  id int(11) NOT NULL auto_increment,
+  queue_id int(11) default '0',
+  updated_on datetime default NULL,
+  count int(11) default '0',
+  temperature decimal(6,4) default NULL,
+  type varchar(255) default NULL,
+  PRIMARY KEY  (id)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;

data/bin/skynet

@@ -0,0 +1,71 @@
+#!/usr/bin/env ruby
+
+# This is the main skynet starter script.
+# It can be used to start the skynet_tuplespace_server as well as all skynet workers.
+# It is important that this script has access to your code. If you want to run
+# skynet within your code you'll want to read about bin/skynet_install[link:files/bin/skynet_install.html]
+#
+# Usage: skynet [options]
+#     -w, --workers WORKERS            Number of workers to start.  The default is 4 and is stored in Skynet::CONFIG[:NUMBER_OF_WORKERS]
+#     -i, --increment-worker-version   Increment Worker Version
+#     -a, --add-workers WORKERS        Number of workers to add.
+#     -k, --remove-workers WORKERS     Number of workers to remove.
+#     -r, --required LIBRARY           Require the specified libraries
+#         --restart-all-workers        Restart All Workers                                      
+#         --restart-workers            Restart Workers                                      
+#
+# If you have chosen to use the TupleSpace message queue adapter this script will see if there is an available TS first
+# and start one if there is not.  You can also start the bin/skynet_tuplespace_server[link:files/bin/skynet_tuplespace_server.html] manually.  
+#
+# Running skynet starts a Skynet::Manager which in turn spawns the Skynet::Worker processes with the appropriate options.
+# You only need to run skynet once per machine.  If you want to add more workers, use the appropriate flags above to do so.
+# Only one manager should be running per machine.  The Skynet::Manager does not dole out tasks
+#
+# You should set all of your Skynet::CONFIG (or Skynet.configure()) options here (or in one of your environment files.)
+# See Skynet::Config for more information on configuration options.
+
+require 'rubygems'
+require File.expand_path(File.dirname(__FILE__)) + '/../lib/skynet.rb'
+
+
+if config_arg = ARGV.detect {|i| i =~ /--help/ }
+  Skynet.start
+  exit
+end
+
+conf_file = nil
+if config_arg = ARGV.detect {|i| i =~ /--config/i }
+  conf_file = config_arg.split('=').last
+  ARGV.delete_if {|i| i =~ /--config/ }  
+else
+  ['config/skynet_config.rb', 'skynet_config.rb', "../config/skynet_config.rb" , "#{ENV['HOME']}/.skynet/config.rb"].each do |path|
+    if File.exists?(path)
+      conf_file = path
+      break
+    end
+  end
+end
+
+if conf_file
+  Skynet::CONFIG[:CONFIG_FILE]      = File.expand_path(conf_file)
+  Skynet::CONFIG[:APP_ROOT]         = File.dirname(Skynet::CONFIG[:CONFIG_FILE])
+  Skynet::CONFIG[:SKYNET_LOG_DIR]   = File.expand_path(Skynet::CONFIG[:APP_ROOT]+ "/../log")
+  Skynet::CONFIG[:SKYNET_PID_DIR]   = File.expand_path(Skynet::CONFIG[:APP_ROOT] + "/../log")
+  begin
+    require conf_file
+  rescue MissingSourceFile => e
+    puts "Specified config file #{conf_file} is missing."
+    exit
+  end
+  puts "using config file: #{File.expand_path(conf_file)}" unless conf_file == 'config/skynet_config.rb'
+else
+  puts "No configuration file found.\nTry running skynet_install to create a config/skynet_config.rb or starting skynet with --config=CONFIG_FILE_LOCATION.  Exiting."
+  exit
+end
+
+Skynet::CONFIG[:WORKER_CHECK_DELAY]     ||= 4
+Skynet::CONFIG[:LAUNCHER_PATH]          = File.expand_path(__FILE__)
+
+Skynet::CONFIG[:SYSTEM_RUNNER] = true
+
+Skynet.start

data/bin/skynet_install

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+
+# skynet_install is used to install skynet libraries into your application.
+# Specifically, it installs a config/skynet_config.rb file
+# Once you have a config/skynet_config.rb you can run skynet start from within your root_app_dir/
+# This is also how you can run skynet from within rails. (using --rails)
+#
+# USAGE: skynet_install [--rails] directory/ (can be '.' for current)"
+# 
+#   Options:
+#       -v, --version                    Show the skynet_install version number and quit.
+#           --mysql                      Include mysql migration if you want to use mysql as your message queue
+#       -r, --rails                      Install into rails app
+#                                        Default: false
+#   General Options:
+#       -h, --help                       Show this help message and quit.
+#       -p, --pretend                    Run but do not make any changes.
+#       -f, --force                      Overwrite files that already exist.
+#       -s, --skip                       Skip files that already exist.
+#       -q, --quiet                      Suppress normal output.
+#       -t, --backtrace                  Debugging: show backtrace on errors.
+#       -c, --svn                        Modify files with subversion. (Note: svn must be in path)
+# 
+
+require 'rubygems'
+require 'rubigen'
+
+if %w(-v --version).include? ARGV.first
+  require 'skynet_install/version'
+  puts "#{File.basename($0)} #{SkynetInstall::VERSION::STRING}"
+  exit(0)
+end
+
+require 'rubigen/scripts/generate'
+RubiGen::Base.use_application_sources! :skynet_install
+RubiGen::Scripts::Generate.new.run(ARGV, :generator => 'skynet_install')

data/bin/skynet_tuplespace_server

@@ -0,0 +1,74 @@
+#!/usr/bin/env ruby
+
+# SkynetTupleSpace server is one of the message queues you can use with Skynet.   Make sure you set:
+#   Skynet::CONFIG[:MESSAGE_QUEUE_ADAPTER] = "Skynet::MessageQueueAdapter::TupleSpace"
+#
+# Usage: skynet_tuplespace_server (start|stop|run) [options]
+#     -t, --ontop=TRUE                 Dont Daemonize
+#     -p, --port=PORT                  Port to listen on. default 7647
+#     -o, --logfile=LOGFILE            Logfile to log to
+#     -l, --loglevel=LOGLEVEL          Log level defaults to DEBUG
+#     -d, --piddir=PIDDIR              Directory to put pidfile
+#     -u, --drburi=druby://DRB_URI     What DRbURI to use
+
+require 'rubygems'
+require 'daemons'
+require 'pp'
+
+require File.expand_path(File.dirname(__FILE__)) + '/../lib/skynet.rb'
+
+options = { 
+  :port           => 7647,
+  :logfile        => Skynet::Config.pidfile_location,
+  :loglevel       => "ERROR",
+  :piddir         => Skynet::CONFIG[:SKYNET_PID_DIR],
+  :use_ringserver => Skynet::CONFIG[:TS_USE_RINGSERVER],
+}
+
+OptionParser.new do |opt|
+  opt.banner = "Usage: skynet_tuplespace_server (start|stop|run) [options]"  
+  opt.on('-t', '--ontop TRUE', 'Dont Daemonize') do |v| 
+    options[:ontop] = true if v.upcase == "TRUE" or v == "1"
+  end
+  opt.on('-p', '--port=PORT', 'Port to listen on. default 7647') do |v| 
+    options[:port] = v.to_i
+  end
+  opt.on('-o', '--logfile=LOGFILE', 'Logfile to log to') do |v|
+    options[:logfile] = File.expand_path(v)
+  end
+  opt.on('-l', '--loglevel=LOGLEVEL', 'Log level defaults to DEBUG') do |v|
+    options[:loglevel] = v
+  end
+  opt.on('-d', '--piddir=PIDDIR', 'Directory to put pidfile') do |v|
+    options[:piddir] = File.expand_path(v)
+  end
+  opt.on('--use_ringserver=TRUE', 'Use ringserver?') do |v|
+    if v.downcase == "true" or v == "1"
+      options[:use_ringserver] = true
+    else
+      options[:use_ringserver] = false
+    end      
+  end
+  opt.on('-u', '--drburi=druby://DRB_URI', 'What DRbURI to use') do |v|
+    if v =~ %r{druby://}
+      options[:drburi] = v
+    else
+      options[:drburi] = "druby://#{v}"
+    end
+    options[:drburi] =~ /druby:\/\/.+?:(\d*)/
+    options[:port] ||= $1.to_i
+  end
+
+  opt.parse!(ARGV)
+end
+
+Daemons.run_proc("skynet_tuplespace_server#{options[:port]}",
+{
+  :dir_mode  => :normal,
+  :dir       => options[:piddir],
+  :backtrace => true,
+  # :monitor => true,
+  :ontop     => options[:ontop] || false
+}) do
+  server = Skynet::TuplespaceServer.start(options)
+end

data/config/hoe.rb

@@ -0,0 +1,75 @@
+require 'skynet/version'
+
+AUTHOR = 'Adam Pisoni'  # can also be an array of Authors
+EMAIL = "apisoni@geni.com"
+DESCRIPTION = "Skynet - A Ruby Map/Reduce Framework"
+GEM_NAME = 'skynet' # what ppl will type to install your gem
+RUBYFORGE_PROJECT = 'skynet' # The unix name for your project
+HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org"
+DOWNLOAD_PATH = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}"
+
+@config_file = "~/.rubyforge/user-config.yml"
+@config = nil
+RUBYFORGE_USERNAME = "unknown"
+def rubyforge_username
+  unless @config
+    begin
+      @config = YAML.load(File.read(File.expand_path(@config_file)))
+    rescue
+      puts <<-EOS
+ERROR: No rubyforge config file found: #{@config_file}
+Run 'rubyforge setup' to prepare your env for access to Rubyforge
+ - See http://newgem.rubyforge.org/rubyforge.html for more details
+      EOS
+      exit
+    end
+  end
+  RUBYFORGE_USERNAME.replace @config["username"]
+end
+
+
+REV = nil 
+# UNCOMMENT IF REQUIRED: 
+# REV = `svn info`.each {|line| if line =~ /^Revision:/ then k,v = line.split(': '); break v.chomp; else next; end} rescue nil
+VERS = Skynet::VERSION::STRING + (REV ? ".#{REV}" : "")
+RDOC_OPTS = ['--quiet', '--title', 'skynet documentation',
+    "--opname", "index.html",
+    "--line-numbers", 
+    "--main", "README",
+    "--inline-source"]
+
+class Hoe
+  def extra_deps 
+    @extra_deps.reject! { |x| Array(x).first == 'hoe' } 
+    @extra_deps
+  end 
+end
+
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+$hoe = Hoe.new(GEM_NAME, VERS) do |p|
+  p.author = AUTHOR 
+  p.description = DESCRIPTION
+  p.email = EMAIL
+  p.summary = DESCRIPTION
+  p.url = HOMEPATH
+  p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
+  p.test_globs = ["test/**/test_*.rb"]
+  p.clean_globs |= ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store']  #An array of file patterns to delete on clean.
+  
+  # == Optional
+  p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n")
+  p.extra_deps = [
+    ['daemons',">= 1"],
+    ['rubigen', ">=1.1.1"]
+    ]
+    # An array of rubygem dependencies [name, version], e.g. [ ['active_support', '>= 1.3.1'] ]
+  
+  #p.spec_extras = {}    # A hash of extra values to set in the gemspec.
+  
+end
+
+CHANGES = $hoe.paragraphs_of('History.txt', 0..1).join("\\n\\n")
+PATH    = (RUBYFORGE_PROJECT == GEM_NAME) ? RUBYFORGE_PROJECT : "#{RUBYFORGE_PROJECT}/#{GEM_NAME}"
+$hoe.remote_rdoc_dir = File.join(PATH.gsub(/^#{RUBYFORGE_PROJECT}\/?/,''), 'rdoc')
+$hoe.rsync_args = '-av --delete --ignore-errors'