brendan-skynet 0.9.3.1

data/app_generators/skynet_install/templates/skynet_config.rb

@@ -0,0 +1,54 @@
+# Skynet Configuration File.  Should be in APP_ROOT/config/skynet_config.rb  
+# Start skynet from within your app root with 
+# > skynet start
+
+require 'rubygems'
+require 'skynet'         
+
+<% if in_rails -%>
+# Load your rails app
+require File.expand_path(File.dirname(__FILE__)) + '/../config/environment' if not defined?(RAILS_GEM_VERSION)
+<% end -%>
+
+Skynet::CONFIG[:SKYNET_LOG_LEVEL] = Logger::ERROR
+<% if in_rails -%>
+  
+Skynet::CONFIG[:APP_ROOT]         = RAILS_ROOT
+Skynet::CONFIG[:SKYNET_LOG_DIR]   = File.expand_path(Skynet::CONFIG[:APP_ROOT]+ "/log")
+Skynet::CONFIG[:SKYNET_PID_DIR]   = File.expand_path(Skynet::CONFIG[:APP_ROOT] + "/log")
+Skynet::CONFIG[:SKYNET_LOG_FILE]  = "skynet_#{RAILS_ENV}.log"
+Skynet::CONFIG[:SKYNET_PID_FILE]  = "skynet_#{RAILS_ENV}.pid"
+<% else -%>
+Skynet::CONFIG[:SKYNET_LOG_DIR]   = File.expand_path(File.dirname(__FILE__) + "/../log") 
+Skynet::CONFIG[:SKYNET_PID_DIR]   = File.expand_path(File.dirname(__FILE__) + "/../log") 
+<% end -%>
+
+<% if mysql -%>
+<% if not in_rails -%>
+# Use the mysql message queue adapter
+Skynet::CONFIG[:MESSAGE_QUEUE_ADAPTER] = "Skynet::MessageQueueAdapter::Mysql"
+Skynet::CONFIG[:MYSQL_HOST]            = "localhost"
+Skynet::CONFIG[:MYSQL_USERNAME]        = "root"
+Skynet::CONFIG[:MYSQL_PASSWORD]        = ""
+Skynet::CONFIG[:MYSQL_DATABASE]        = "skynet"
+<% else %>
+# Use the mysql message queue adapter
+Skynet::CONFIG[:MESSAGE_QUEUE_ADAPTER] = "Skynet::MessageQueueAdapter::Mysql"
+<% end -%>
+<% end -%>
+
+# Load ActiveRecord extensions, if using ActiveRecord
+if defined?(ActiveRecord::Base)
+  require 'skynet_active_record_extensions'
+  require 'message_queue_adapters/mysql' 
+end
+
+# ==================================================================
+# = Require any other libraries you want skynet to know about here =
+# ==================================================================
+
+# ===========================================
+# = Set your own configuration options here =
+# ===========================================
+# You can also configure skynet with
+# Skynet.configure(:SOME_CONFIG_OPTION => true, :SOME_OTHER_CONFIG => 3)

data/app_generators/skynet_install/templates/skynet_initializer.rb

@@ -0,0 +1 @@
+require RAILS_ROOT+'/config/skynet_config.rb'

data/app_generators/skynet_install/templates/skynet_mysql_schema.sql

@@ -0,0 +1,33 @@
+CREATE TABLE skynet_message_queues (
+  id int(11) NOT NULL auto_increment,
+  queue_id int(11) default '0',
+  tran_id bigint(20) unsigned default NULL,
+  created_on datetime default NULL,
+  updated_on datetime default NULL,
+  tasktype varchar(255) default NULL,
+  task_id bigint(20) unsigned default NULL,
+  job_id bigint(20) unsigned default NULL,
+  raw_payload longtext,
+  payload_type varchar(255) default NULL,
+  name varchar(255) default NULL,
+  expiry int(11) default NULL,
+  expire_time decimal(16,4) default NULL,
+  iteration int(11) default NULL,
+  version int(11) default NULL,
+  timeout decimal(16,4) default NULL,
+  retry int(11) default '0',
+  PRIMARY KEY  (id),
+  UNIQUE KEY index_skynet_message_queues_on_tran_id (tran_id),
+  KEY index_skynet_message_queues_on_job_id (job_id),
+  KEY index_skynet_message_queues_on_task_id (task_id),
+  KEY index_skynet_mqueue_for_take (queue_id,tasktype,payload_type,expire_time)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+CREATE TABLE skynet_queue_temperature (
+  id int(11) NOT NULL auto_increment,
+  queue_id int(11) default '0',
+  updated_on datetime default NULL,
+  count int(11) default '0',
+  temperature decimal(6,4) default NULL,
+  type varchar(255) default NULL,
+  PRIMARY KEY  (id)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;

data/bin/skynet

@@ -0,0 +1,71 @@
+#!/usr/bin/env ruby
+
+# This is the main skynet starter script.
+# It can be used to start the skynet_tuplespace_server as well as all skynet workers.
+# It is important that this script has access to your code. If you want to run
+# skynet within your code you'll want to read about bin/skynet_install[link:files/bin/skynet_install.html]
+#
+# Usage: skynet [options]
+#     -w, --workers WORKERS            Number of workers to start.  The default is 4 and is stored in Skynet::CONFIG[:NUMBER_OF_WORKERS]
+#     -i, --increment-worker-version   Increment Worker Version
+#     -a, --add-workers WORKERS        Number of workers to add.
+#     -k, --remove-workers WORKERS     Number of workers to remove.
+#     -r, --required LIBRARY           Require the specified libraries
+#         --restart-all-workers        Restart All Workers                                      
+#         --restart-workers            Restart Workers                                      
+#
+# If you have chosen to use the TupleSpace message queue adapter this script will see if there is an available TS first
+# and start one if there is not.  You can also start the bin/skynet_tuplespace_server[link:files/bin/skynet_tuplespace_server.html] manually.  
+#
+# Running skynet starts a Skynet::Manager which in turn spawns the Skynet::Worker processes with the appropriate options.
+# You only need to run skynet once per machine.  If you want to add more workers, use the appropriate flags above to do so.
+# Only one manager should be running per machine.  The Skynet::Manager does not dole out tasks
+#
+# You should set all of your Skynet::CONFIG (or Skynet.configure()) options here (or in one of your environment files.)
+# See Skynet::Config for more information on configuration options.
+
+require 'rubygems'
+require File.expand_path(File.dirname(__FILE__)) + '/../lib/skynet.rb'
+
+
+if config_arg = ARGV.detect {|i| i =~ /--help/ }
+  Skynet.start
+  exit
+end
+
+conf_file = nil
+if config_arg = ARGV.detect {|i| i =~ /--config/i }
+  conf_file = config_arg.split('=').last
+  ARGV.delete_if {|i| i =~ /--config/ }  
+else
+  ['config/skynet_config.rb', 'skynet_config.rb', "../config/skynet_config.rb" , "#{ENV['HOME']}/.skynet/config.rb"].each do |path|
+    if File.exists?(path)
+      conf_file = path
+      break
+    end
+  end
+end
+
+if conf_file
+  Skynet::CONFIG[:CONFIG_FILE]      = File.expand_path(conf_file)
+  Skynet::CONFIG[:APP_ROOT]         = File.dirname(Skynet::CONFIG[:CONFIG_FILE])
+  Skynet::CONFIG[:SKYNET_LOG_DIR]   = File.expand_path(Skynet::CONFIG[:APP_ROOT]+ "/../log")
+  Skynet::CONFIG[:SKYNET_PID_DIR]   = File.expand_path(Skynet::CONFIG[:APP_ROOT] + "/../log")
+  begin
+    require conf_file
+  rescue MissingSourceFile => e
+    puts "Specified config file #{conf_file} is missing."
+    exit
+  end
+  puts "using config file: #{File.expand_path(conf_file)}" unless conf_file == 'config/skynet_config.rb'
+else
+  puts "No configuration file found.\nTry running skynet_install to create a config/skynet_config.rb or starting skynet with --config=CONFIG_FILE_LOCATION.  Exiting."
+  exit
+end
+
+Skynet::CONFIG[:WORKER_CHECK_DELAY]     ||= 4
+Skynet::CONFIG[:LAUNCHER_PATH]          = File.expand_path(__FILE__)
+
+Skynet::CONFIG[:SYSTEM_RUNNER] = true
+
+Skynet.start

data/bin/skynet_install

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+
+# skynet_install is used to install skynet libraries into your application.
+# Specifically, it installs a config/skynet_config.rb file
+# Once you have a config/skynet_config.rb you can run skynet start from within your root_app_dir/
+# This is also how you can run skynet from within rails. (using --rails)
+#
+# USAGE: skynet_install [--rails] directory/ (can be '.' for current)"
+# 
+#   Options:
+#       -v, --version                    Show the skynet_install version number and quit.
+#           --mysql                      Include mysql migration if you want to use mysql as your message queue
+#       -r, --rails                      Install into rails app
+#                                        Default: false
+#   General Options:
+#       -h, --help                       Show this help message and quit.
+#       -p, --pretend                    Run but do not make any changes.
+#       -f, --force                      Overwrite files that already exist.
+#       -s, --skip                       Skip files that already exist.
+#       -q, --quiet                      Suppress normal output.
+#       -t, --backtrace                  Debugging: show backtrace on errors.
+#       -c, --svn                        Modify files with subversion. (Note: svn must be in path)
+# 
+
+require 'rubygems'
+require 'rubigen'
+
+if %w(-v --version).include? ARGV.first
+  require 'skynet_install/version'
+  puts "#{File.basename($0)} #{SkynetInstall::VERSION::STRING}"
+  exit(0)
+end
+
+require 'rubigen/scripts/generate'
+RubiGen::Base.use_application_sources! :skynet_install
+RubiGen::Scripts::Generate.new.run(ARGV, :generator => 'skynet_install')

data/bin/skynet_tuplespace_server

@@ -0,0 +1,74 @@
+#!/usr/bin/env ruby
+
+# SkynetTupleSpace server is one of the message queues you can use with Skynet.   Make sure you set:
+#   Skynet::CONFIG[:MESSAGE_QUEUE_ADAPTER] = "Skynet::MessageQueueAdapter::TupleSpace"
+#
+# Usage: skynet_tuplespace_server (start|stop|run) [options]
+#     -t, --ontop=TRUE                 Dont Daemonize
+#     -p, --port=PORT                  Port to listen on. default 7647
+#     -o, --logfile=LOGFILE            Logfile to log to
+#     -l, --loglevel=LOGLEVEL          Log level defaults to DEBUG
+#     -d, --piddir=PIDDIR              Directory to put pidfile
+#     -u, --drburi=druby://DRB_URI     What DRbURI to use
+
+require 'rubygems'
+require 'daemons'
+require 'pp'
+
+require File.expand_path(File.dirname(__FILE__)) + '/../lib/skynet.rb'
+
+options = { 
+  :port           => 7647,
+  :logfile        => Skynet::Config.pidfile_location,
+  :loglevel       => "ERROR",
+  :piddir         => Skynet::CONFIG[:SKYNET_PID_DIR],
+  :use_ringserver => Skynet::CONFIG[:TS_USE_RINGSERVER],
+}
+
+OptionParser.new do |opt|
+  opt.banner = "Usage: skynet_tuplespace_server (start|stop|run) [options]"  
+  opt.on('-t', '--ontop TRUE', 'Dont Daemonize') do |v| 
+    options[:ontop] = true if v.upcase == "TRUE" or v == "1"
+  end
+  opt.on('-p', '--port=PORT', 'Port to listen on. default 7647') do |v| 
+    options[:port] = v.to_i
+  end
+  opt.on('-o', '--logfile=LOGFILE', 'Logfile to log to') do |v|
+    options[:logfile] = File.expand_path(v)
+  end
+  opt.on('-l', '--loglevel=LOGLEVEL', 'Log level defaults to DEBUG') do |v|
+    options[:loglevel] = v
+  end
+  opt.on('-d', '--piddir=PIDDIR', 'Directory to put pidfile') do |v|
+    options[:piddir] = File.expand_path(v)
+  end
+  opt.on('--use_ringserver=TRUE', 'Use ringserver?') do |v|
+    if v.downcase == "true" or v == "1"
+      options[:use_ringserver] = true
+    else
+      options[:use_ringserver] = false
+    end      
+  end
+  opt.on('-u', '--drburi=druby://DRB_URI', 'What DRbURI to use') do |v|
+    if v =~ %r{druby://}
+      options[:drburi] = v
+    else
+      options[:drburi] = "druby://#{v}"
+    end
+    options[:drburi] =~ /druby:\/\/.+?:(\d*)/
+    options[:port] ||= $1.to_i
+  end
+
+  opt.parse!(ARGV)
+end
+
+Daemons.run_proc("skynet_tuplespace_server#{options[:port]}",
+{
+  :dir_mode  => :normal,
+  :dir       => options[:piddir],
+  :backtrace => true,
+  # :monitor => true,
+  :ontop     => options[:ontop] || false
+}) do
+  server = Skynet::TuplespaceServer.start(options)
+end

data/config/hoe.rb

@@ -0,0 +1,75 @@
+require 'skynet/version'
+
+AUTHOR = 'Adam Pisoni'  # can also be an array of Authors
+EMAIL = "apisoni@geni.com"
+DESCRIPTION = "Skynet - A Ruby Map/Reduce Framework"
+GEM_NAME = 'skynet' # what ppl will type to install your gem
+RUBYFORGE_PROJECT = 'skynet' # The unix name for your project
+HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org"
+DOWNLOAD_PATH = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}"
+
+@config_file = "~/.rubyforge/user-config.yml"
+@config = nil
+RUBYFORGE_USERNAME = "unknown"
+def rubyforge_username
+  unless @config
+    begin
+      @config = YAML.load(File.read(File.expand_path(@config_file)))
+    rescue
+      puts <<-EOS
+ERROR: No rubyforge config file found: #{@config_file}
+Run 'rubyforge setup' to prepare your env for access to Rubyforge
+ - See http://newgem.rubyforge.org/rubyforge.html for more details
+      EOS
+      exit
+    end
+  end
+  RUBYFORGE_USERNAME.replace @config["username"]
+end
+
+
+REV = nil 
+# UNCOMMENT IF REQUIRED: 
+# REV = `svn info`.each {|line| if line =~ /^Revision:/ then k,v = line.split(': '); break v.chomp; else next; end} rescue nil
+VERS = Skynet::VERSION::STRING + (REV ? ".#{REV}" : "")
+RDOC_OPTS = ['--quiet', '--title', 'skynet documentation',
+    "--opname", "index.html",
+    "--line-numbers", 
+    "--main", "README",
+    "--inline-source"]
+
+class Hoe
+  def extra_deps 
+    @extra_deps.reject! { |x| Array(x).first == 'hoe' } 
+    @extra_deps
+  end 
+end
+
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+hoe = Hoe.new(GEM_NAME, VERS) do |p|
+  p.author = AUTHOR 
+  p.description = DESCRIPTION
+  p.email = EMAIL
+  p.summary = DESCRIPTION
+  p.url = HOMEPATH
+  p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
+  p.test_globs = ["test/**/test_*.rb"]
+  p.clean_globs |= ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store']  #An array of file patterns to delete on clean.
+  
+  # == Optional
+  p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n")
+  p.extra_deps = [
+    ['daemons',">= 1"],
+    ['rubigen', ">=1.1.1"]
+    ]
+    # An array of rubygem dependencies [name, version], e.g. [ ['active_support', '>= 1.3.1'] ]
+  
+  #p.spec_extras = {}    # A hash of extra values to set in the gemspec.
+  
+end
+
+CHANGES = hoe.paragraphs_of('History.txt', 0..1).join("\\n\\n")
+PATH    = (RUBYFORGE_PROJECT == GEM_NAME) ? RUBYFORGE_PROJECT : "#{RUBYFORGE_PROJECT}/#{GEM_NAME}"
+hoe.remote_rdoc_dir = File.join(PATH.gsub(/^#{RUBYFORGE_PROJECT}\/?/,''), 'rdoc')
+hoe.rsync_args = '-av --delete --ignore-errors'

data/config/requirements.rb

@@ -0,0 +1,17 @@
+require 'fileutils'
+include FileUtils
+
+require 'rubygems'
+%w[rake hoe newgem rubigen].each do |req_gem|
+  begin
+    require req_gem
+  rescue LoadError
+    puts "This Rakefile requires the '#{req_gem}' RubyGem."
+    puts "Installation: gem install #{req_gem} -y"
+    exit
+  end
+end
+
+$:.unshift(File.join(File.dirname(__FILE__), %w[.. lib]))
+
+require 'skynet'

data/examples/dgrep/README

@@ -0,0 +1,70 @@
+
+EXAMPLE 1: Distributed Grep
+------------------------------
+You have a list of files and want to see how many times each word shows up in those files.
+All of the map_reduce code is in lib/mapreduce_test.rb
+
+To 
+1. Run Skynet
+ > ./script/skynet start
+
+2. Watch the logs in another terminal
+ > tail -f log/skynet.log
+ 
+3. Run dgrep providing it with some words and a directory
+ > ./script/dgrep -r love hate ./data
+
+I've put all of shakespears poetry in the data directory.  You can download all of Shakespeare's work
+to add to the data at http://www.it.usyd.edu.au/~matty/Shakespeare/shakespeare.tar.gz
+
+Thats it!!
+
+EXAMPLE 2: Distributed Grep with distributed reducing
+------------------------------
+Same example as above, but this time we want to aggregate our results across multiple partitioners. This would
+be useful if you had a large reduce data set, or complex calculations in order to reduce.   All the code is already
+in dgrep.pl.  Specifically, notice I've included a self.reduce_partition method (which just forwards to a pre-packaged
+partitioner).  This partitioner not only breaks up the data returned from the mappers, but also makes sure each reducer 
+gets all the same keys.
+
+1. Run dgrep passing --reducers=2  This tells Skynet to try and use up to 2 reducers (instead of the default of 1)
+> ./script/dgrep -r --reducers=2 data file ./data
+
+This sample application was created by creating in the following way
+
+1. Create new app
+ > mkdir simple_example
+ > mkdir data lib
+
+2. Install Skynet
+ > skynet_install
+
+3. Add Data Files to /data
+These are any text files named testfileN.txt and so on
+
+4. Create dgrep.rb and drep script (found in lib/ and script/)
+
+------------------------------------
+
+EXAMPLE 1: Most Common Words
+------------------------------
+You have a list of files and want the 10 most common 5 or more letter words in all of those fules.
+
+1. Run Skynet
+ > ./script/skynet start
+
+2. Watch the logs in another terminal
+ > tail -f log/skynet.log
+ 
+3. Run dgrep providing it with the '-mcw' (most common words) command, some words and a directory
+ > ./script/dgrep -r -mcw ./data
+
+To see what's going on check out lib/most_common_words.rb
+------------------------------------
+
+If you'd like get a peak at what Skynet is doing, edit ./config/skynet_config.rb and set
+
+Skynet::CONFIG[:SKYNET_LOG_LEVEL] = Logger::INFO # or Logger::INFO
+
+and tail ./log/skynet.log
+

data/examples/dgrep/config/skynet_config.rb

@@ -0,0 +1,26 @@
+# Skynet Configuration File.  Should be in APP_ROOT/config/skynet_config.rb  
+# Start skynet from within your app root with 
+# $ skynet start
+
+require 'rubygems'
+require 'skynet'         
+
+Skynet::CONFIG[:SKYNET_LOG_LEVEL] = Logger::ERROR
+Skynet::CONFIG[:SKYNET_LOG_DIR]   = File.expand_path(File.dirname(__FILE__) + "/../log") 
+Skynet::CONFIG[:SKYNET_PID_DIR]   = File.expand_path(File.dirname(__FILE__) + "/../log") 
+
+# ==================================================================
+# = Require any other libraries you want skynet to know about here =
+# ==================================================================
+
+# require all ruby files in lib/
+Dir["#{File.dirname(__FILE__)}" + "/../lib/**/*.rb"].each do |file|
+  require file
+end
+
+# ===========================================
+# = Set your own configuration options here =
+# ===========================================
+# You can also configure skynet with
+# Skynet.configure(:SOME_CONFIG_OPTION => true, :SOME_OTHER_CONFIG => 3)
+

data/examples/dgrep/data/shakespeare/README

@@ -0,0 +1,2 @@
+Download the complete text of shakespeare at
+http://www.it.usyd.edu.au/~matty/Shakespeare/shakespeare.tar.gz