skynet 0.9.2 → 0.9.3

data/examples/rails_mysql_example/public/robots.txt

@@ -0,0 +1,5 @@
+# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
+#
+# To ban all spiders from the entire site uncomment the next two lines:
+# User-Agent: *
+# Disallow: /

data/examples/rails_mysql_example/script/about

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../config/boot'
+require 'commands/about'

data/examples/rails_mysql_example/script/console

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../config/boot'
+require 'commands/console'

data/examples/rails_mysql_example/script/destroy

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../config/boot'
+require 'commands/destroy'

data/examples/rails_mysql_example/script/generate

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../config/boot'
+require 'commands/generate'

data/examples/rails_mysql_example/script/performance/benchmarker

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../../config/boot'
+require 'commands/performance/benchmarker'

data/examples/rails_mysql_example/script/performance/profiler

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../../config/boot'
+require 'commands/performance/profiler'

data/examples/rails_mysql_example/script/performance/request

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../../config/boot'
+require 'commands/performance/request'

data/examples/rails_mysql_example/script/plugin

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../config/boot'
+require 'commands/plugin'

data/examples/rails_mysql_example/script/process/inspector

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../../config/boot'
+require 'commands/process/inspector'

data/examples/rails_mysql_example/script/process/reaper

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../../config/boot'
+require 'commands/process/reaper'

data/examples/rails_mysql_example/script/process/spawner

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../../config/boot'
+require 'commands/process/spawner'

data/examples/rails_mysql_example/script/runner

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../config/boot'
+require 'commands/runner'

data/examples/rails_mysql_example/script/server

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../config/boot'
+require 'commands/server'

data/examples/rails_mysql_example/test/fixtures/user_favorites.yml

@@ -0,0 +1,9 @@
+# Read about fixtures at http://ar.rubyonrails.org/classes/Fixtures.html
+
+one:
+  user_id: 1
+  favorite: MyString
+
+two:
+  user_id: 1
+  favorite: MyString

data/examples/rails_mysql_example/test/fixtures/users.yml

@@ -0,0 +1,11 @@
+# Read about fixtures at http://ar.rubyonrails.org/classes/Fixtures.html
+
+one:
+  name: MyString
+  password: MyString
+  favorites: MyString
+
+two:
+  name: MyString
+  password: MyString
+  favorites: MyString

data/examples/rails_mysql_example/test/test_helper.rb

@@ -0,0 +1,38 @@
+ENV["RAILS_ENV"] = "test"
+require File.expand_path(File.dirname(__FILE__) + "/../config/environment")
+require 'test_help'
+
+class Test::Unit::TestCase
+  # Transactional fixtures accelerate your tests by wrapping each test method
+  # in a transaction that's rolled back on completion.  This ensures that the
+  # test database remains unchanged so your fixtures don't have to be reloaded
+  # between every test method.  Fewer database queries means faster tests.
+  #
+  # Read Mike Clark's excellent walkthrough at
+  #   http://clarkware.com/cgi/blosxom/2005/10/24#Rails10FastTesting
+  #
+  # Every Active Record database supports transactions except MyISAM tables
+  # in MySQL.  Turn off transactional fixtures in this case; however, if you
+  # don't care one way or the other, switching from MyISAM to InnoDB tables
+  # is recommended.
+  #
+  # The only drawback to using transactional fixtures is when you actually 
+  # need to test transactions.  Since your test is bracketed by a transaction,
+  # any transactions started in your code will be automatically rolled back.
+  self.use_transactional_fixtures = true
+
+  # Instantiated fixtures are slow, but give you @david where otherwise you
+  # would need people(:david).  If you don't want to migrate your existing
+  # test cases which use the @david style and don't mind the speed hit (each
+  # instantiated fixtures translates to a database query per test method),
+  # then set this back to true.
+  self.use_instantiated_fixtures  = false
+
+  # Setup all fixtures in test/fixtures/*.(yml|csv) for all tests in alphabetical order.
+  #
+  # Note: You'll currently still have to declare fixtures explicitly in integration tests
+  # -- they do not yet inherit this setting
+  fixtures :all
+
+  # Add more helper methods to be used by all tests here...
+end

data/examples/rails_mysql_example/test/unit/user_favorite_test.rb

@@ -0,0 +1,8 @@
+require File.dirname(__FILE__) + '/../test_helper'
+
+class UserFavoriteTest < ActiveSupport::TestCase
+  # Replace this with your real tests.
+  def test_truth
+    assert true
+  end
+end

data/examples/rails_mysql_example/test/unit/user_test.rb

@@ -0,0 +1,8 @@
+require File.dirname(__FILE__) + '/../test_helper'
+
+class UserTest < ActiveSupport::TestCase
+  # Replace this with your real tests.
+  def test_truth
+    assert true
+  end
+end

data/extras/README

@@ -0,0 +1,7 @@
+I've included some extras to help you deploy and monitor skynet.
+
+init.d/skynet  # sample init.d script to start/stop/restart skynet
+
+nagios/check_skynet.sh  # A script we use to have nagios (a monitoring tool) check our skynet Q.
+
+rails/*  # I've included the controller and view we use to monitor our skynet Q

data/extras/init.d/skynet

@@ -0,0 +1,87 @@
+#!/bin/sh
+#
+# skynet	start/stop skynet
+#
+# processname: solr
+#
+
+# Source function library
+. /etc/init.d/functions
+
+# Get network config
+. /etc/sysconfig/network
+
+HOME=/skynet/current
+RAILS_ENV=production
+
+cd $HOME 
+RETVAL=0
+start() {
+        echo -n $"Starting Skynet: "
+	./script/skynet &
+        RETVAL=$?
+        echo
+        return $RETVAL
+}
+
+stop() {
+    echo -n $"Stopping Skynet: " 
+	pid=`cat /skynet/logs/skynet_production.pid`
+	kill ${pid}
+    RETVAL=$?
+    return $RETVAL
+}    
+
+restart() {
+        echo -n $"Restarting Skynet: "
+  ./script/skynet -r
+        RETVAL=$?
+        echo
+        return $RETVAL
+}
+
+
+reload() {
+    stop
+    start
+}
+
+case "$1" in
+  start)
+      start
+    ;;
+  stop)
+      stop
+    ;;
+  status)
+	exit
+    ;;
+  restart)
+      restart
+    ;;
+  condrestart)
+      [ -f /var/lock/subsys/skynet ] && restart || :
+    ;;
+  reload)
+    reload
+    ;;
+  *)
+    echo $"Usage: $0 {start|stop|status|restart|condrestart|reload}" 
+    exit 1
+esac
+get_pid_for_cmd ()
+{
+	pid=`cat /skynet/logs/skynet_worker7647.pid`
+	return pid
+
+    #for pid in `ls -t /proc`; do
+    #    if [ -d /proc/$pid ] && [ -f /proc/$pid/cmdline ]; then
+    #        if [ "$1" = "$(</proc/$pid/cmdline)" ]; then
+    #            return
+    #        fi
+    #    fi
+    #done
+    #pid=0
+}
+
+exit $?

data/extras/nagios/check_skynet.sh

@@ -0,0 +1,121 @@
+#! /bin/sh 
+#
+# Usage: ./check_skynet --help
+#
+# Examples: 
+#	./check_skynet -w 300 -c 2000 -u
+
+# Paths to commands used in this script.  These
+# may have to be modified to match your system setup.
+
+PATH=""
+
+ECHO="/bin/echo"
+SED="/bin/sed"
+GREP="/bin/grep"
+TAIL="/bin/tail"
+CAT="/bin/cat"
+CUT="/bin/cut"
+WC="/bin/wc"
+CURL="/usr/bin/curl -f"
+
+PROGNAME=`/bin/basename $0`
+PROGPATH=`echo $0 | /bin/sed -e 's,[\\/][^\\/][^\\/]*$,,'`
+REVISION=`echo '$Revision: 0.1 $' | /bin/sed -e 's/[^0-9.]//g'`
+
+. /usr/local/nagios/libexec/utils.sh
+
+print_usage() {
+    echo "Usage: $PROGNAME -w <threshold> -c <threshold> -u <url of skynet status page>"
+    echo "Usage: $PROGNAME --help"
+    echo "Usage: $PROGNAME --version"
+}
+
+print_help() {
+    print_revision $PROGNAME $REVISION
+    echo ""
+    print_usage
+    echo ""
+    echo "Check Skynet's untaken_tasks"
+    echo ""
+    support
+}
+
+# Make sure the correct number of command line
+# arguments have been supplied
+
+if [ $# -lt 3 ]; then
+    print_usage
+    exit $STATE_UNKNOWN
+fi
+
+# Grab the command line arguments
+
+exitstatus=$STATE_UNKNOWN #default
+while test -n "$1"; do
+    case "$1" in
+        --help)
+            print_help
+            exit $STATE_OK
+            ;;
+        -h)
+            print_help
+            exit $STATE_OK
+            ;;
+        --version)
+            print_revision $PROGNAME $VERSION
+            exit $STATE_OK
+            ;;
+        -V)
+            print_revision $PROGNAME $VERSION
+            exit $STATE_OK
+            ;;
+        -w)
+	    WARNING=$2;
+            shift;
+            ;;
+        -c)
+	    CRITICAL=$2;
+            shift;
+            ;;
+        -u)
+	    URL=$2;
+            shift;
+            ;;
+        *)
+            echo "Unknown argument: $1"
+            print_usage
+            exit $STATE_UNKNOWN
+            ;;
+    esac
+    shift
+done
+
+
+CURRENT=$($CURL $URL | $GREP "untaken_tasks\|down"| $CUT -d: -f2)
+
+if [ -z $CURRENT ] ;then
+	$ECHO "CANNOT GATHER SKYNET TASKS CALL NOPS"
+	exit $STATE_UNKNOWN
+fi
+
+if [ $CURRENT -ge $CRITICAL ]; then
+	$ECHO "Skynet untaken_tasks: $CURRENT threshold: $CRITICAL   CRITICAL"
+	$ECHO " "
+    exit $STATE_CRITICAL
+fi
+
+if [ $CURRENT -ge $WARNING ]; then
+	$ECHO "Skynet untaken_tasks: $CURRENT threshold: $WARNING   WARNING"
+	$ECHO " "
+    exit $STATE_WARNING
+fi
+
+if [[ $CURRENT -lt $CRITICAL  &&  $CURRENT -lt $WARNING ]]; then
+	$ECHO "Skynet untaken_tasks: $CURRENT threshold critical: $CRITICAL threshold warning: $WARNING OK"
+	$ECHO " "
+    exit $STATE_OK
+	
+fi
+$ECHO "NO SCRIPT OUTPUT CALL NOPS!"
+exit $STATE_CRITICAL

data/extras/rails/controllers/skynet_controller.rb

@@ -0,0 +1,43 @@
+class Admin::SkynetController < AdminController
+
+  def index
+    begin
+      start_time = Time.now
+      setup
+      if params[:skynet_message_queue]
+        Skynet.configure(:MYSQL_MESSAGE_QUEUE_TABLE => params[:skynet_message_queue]) do
+          @stats = @mq.stats
+        end
+      else
+        @stats = @mq.stats
+      end
+      @stats.merge!(Skynet::Manager.stats_for_hosts)
+      @stats[:hosts] = @stats[:servers].size
+      @stats[:processed_time] = Time.now - start_time
+    rescue Exception => e
+      logger.error "ERROR #{e.inspect} #{e.backtrace.join("\n")}"      
+    end
+  end
+  
+  # plain text page that will be used by monitoring scripts
+  def status
+    begin
+      setup
+      stats = @mq.stats
+      stats[:servers] = stats[:servers].keys.join(",")
+      stats.each { |k,v| stats[k.to_s] = stats.delete(k) }
+      text  = stats.keys.sort.collect{ |k| "#{k}:#{stats[k]}" }.join("\n") + "\n"
+      render :text => text, :content_type => 'text/plain'
+    rescue Exception => e
+      render :text => "skynet is down\n", :content_type => 'text/plain'
+    end
+  end  
+  
+  private
+  
+  def setup
+    @mq  ||= Skynet::MessageQueue.new(Skynet::CONFIG[:MESSAGE_QUEUE_ADAPTER])
+    @last_updated = Time.now.strftime('%r')
+  end
+  
+end

data/extras/rails/views/skynet/index.rhtml

@@ -0,0 +1,137 @@
+
+  <div class="hd">
+    <h1>SKYNET STATUS</h1>
+    <span>Last updated: <%= @last_updated || 'N/A' %></span>
+    <br />
+    <br />
+  </div>
+
+  <% if @stats %>
+  
+  <h2>Overall Stats</h2>
+  <table class="admin_table">
+    <thead>
+      <tr>
+        <th>Hosts</th>
+        <th>Workers</th>
+        <th>Active Workers<BR>(Masters / Tasks / Either)</th>
+        <th>Idle Workers<BR>(Masters / Tasks / Either)</th>
+          <th>Untaken Tasks<BR>(Master / Task) Future</th>
+        <th>Taken Tasks<BR>(Master / Task)</th>
+        <th>Results</th>
+        <th>Processed <br />(by active workers)</th>
+        <th>Process Time</th>
+      </tr>
+    </thead>
+    <tbody>
+      <tr>
+        <td class="numeric"><%= @stats[:hosts] %></td>
+        <td class="numeric"><%= @stats[:number_of_workers] %></td>
+        <td class="numeric"><b><%= @stats[:active_workers] %></b> (<%= @stats[:active_masters] %> / <%= @stats[:active_taskworkers] %> / <%= @stats[:active_master_or_task_workers] %>)</td>
+        <td class="numeric"><b><%= @stats[:idle_workers] %></b> (<%= @stats[:idle_masters] %> / <%= @stats[:idle_taskworkers] %> / <%= @stats[:idle_master_or_task_workers] %>)</td>
+        <td class="numeric"><%= @stats[:untaken_tasks] %> (<b><%= @stats[:untaken_master_tasks] %></b> / <%= @stats[:untaken_task_tasks] %>) <%= @stats[:untaken_future_tasks] %></td>
+        <td class="numeric"><b><%= @stats[:taken_tasks] %></b> (<%= @stats[:taken_master_tasks] %> / <%= @stats[:taken_task_tasks] %>)</td>
+        <td class="numeric"><%= @stats[:results] %></td>
+        <td class="numeric"><%= @stats[:processed] %> (<%= @stats[:processed_by_active_workers] %>)</td>
+        <td><%= @stats[:processed_time] %></td>
+      </tr>
+    </tbody>
+  </table>
+  
+
+  <BR>
+  <h2>Servers</h2>
+  <table class="admin_table">
+    <colgroup>
+      <col span="1" width="3%" />
+      <col span="1" width="10%" />
+      <col span="1" width="5%" />
+      <col span="1" width="5%" />
+      <col span="1" width="5%" />
+      <col span="1" width="5%" />
+    </colgroup>
+    <thead>
+      <tr>
+        <th class="numeric">#</th>
+        <th>Hostname</th>
+        <th>Number of Workers</th>
+        <th>Active Workers<BR>Masters / Tasks / Either</th>
+        <th>Idle Workers<BR>Masters / Tasks / Either</th>
+        <th>Processed <br />(by active workers)</th>
+      </tr>
+    </thead>
+
+      <% if @stats[:servers] %>
+      <tbody>
+        <% @stats[:servers].keys.sort.each do |hostname| %>
+          <% server = @stats[:servers][hostname] %>
+          <%   i = 1 %>
+          <tr>
+            <td class="numeric"><%= i %></td>
+            <td><%= server[:hostname] %></td>
+            <td class="numeric"><%= server[:number_of_workers] %></td>
+            <td class="numeric"><%= server[:active_workers] %>  (<%= server[:active_masters] %> / <%= server[:active_taskworkers] %> / <%= server[:active_master_or_task_workers] %>) </td>
+            <td class="numeric"><%= server[:idle_workers] %>  (<%= server[:idle_masters] %> / <%= server[:idle_taskworkers] %> / <%= server[:idle_master_or_task_workers] %>) </td>
+            <td class="numeric"><%= server[:processed] %> (<%= server[:processed_by_active_workers] %>)</td>
+          </tr>
+          <%     i += 1 %>
+        <% end %>
+      </tbody>
+      <% end %>
+
+    </table>  
+
+    <% end %>
+
+  <!-- <table class="admin_table">
+    <colgroup>
+      <col span="1" width="5%" />
+      <col span="1" width="8%" />
+      <col span="1" width="10%" />
+      <col span="1" width="10%" />
+      <col span="1" width="5%" />
+      <col span="1" width="5%" />
+    </colgroup>
+    <thead>
+      <tr>
+        <th class="numeric">#</th>
+        <th>Worker ID</th>
+        <th>Hostname</th>
+        <th>PID</th>
+        <th>Job ID</th>
+        <th>Task ID</th>
+        <th>Version</th>
+        <th>Processed</th>
+        <th>M/R</th>
+        <th>Name</th>
+      </tr>
+    </thead>
+    
+    <% if false and @servers %>
+    <tbody>
+      <% @servers.each do |server| %>
+      <tr>
+        <td colspan="10"><%= server[:manager] %></td>
+      </tr>
+      <%   i = 1 %>
+      <%   server[:workers].each do |w| %>
+      <tr>
+        <td class="numeric"><%= i %></td>
+        <td><%= w.worker_id %></td>
+        <td><%= w.hostname %></td>
+        <td class="numeric"><%= w.process_id %></td>
+        <td class="numeric"><%= w.job_id %></td>
+        <td class="numeric"><%= w.task_id %></td>
+        <td class="numeric"><%= w.version %></td>
+        <td class="numeric"><%= w.processed %></td>
+        <td><%= w.map_or_reduce || '-' %></td>
+        <td><%= w.name %></td>
+      </tr>
+      <%     i += 1 %>
+      <%   end %>
+      <% end %>
+    </tbody>
+    <% end %>
+    
+  </table>   -->
+