rq 0.1.7 → 3.0.0

data/TODO

@@ -1,13 +1,20 @@
 ---
-- re-think db schema: separate critical info/index for speed?
-- user docs
-- examples
-- installer script
-- api docs
-- rq relay mode (submit and track exit_status in local db)
-- housekeeping - make backups of queue every so often, integrity_checks at
-- scheduled intervals
+
+- confine/constrain mechansim
+- heartbeat to slave
+- ls show elapsed time
+- rq q 1234 234 bug???
 - full boolean resource monitoring and resource requests
-- in db message queue to send commands to remote nodes - ttl issues here...
-- track feeders in queue rather than using lock on local file?
-- use nodes to periodically generate stats (cache) 
+- pull out some infilter/outfilter classes for all the stdin/stdout parsing
+- rq relay mode (submit and track exit_status in local db)
+- config file configuration for feeder 
+- examples dir with shell commands/scripts
+- db message queue to send commands to remote nodes - ttl issues here...
+- consider/test tracking feeders in queue rather than using lock on local file?
+- use nodes to periodically generate stats and cache them
+
+X rotation bug when dest directory specified
+X backoff lockd recovery time
+X output for start/stop (include shush)
+X .rq dir for everything (log, pidfile)
+X cronify

data/TUTORIAL

@@ -0,0 +1,230 @@
+---
+=== SYNOPSIS
+---
+
+rq is a tool for instantly building simple linux clusters based on the concept
+of coordinated shared access to an nfs mounted priority job queue.  the idea is
+a simple one: one, or more, nodes take jobs from a priority queue, run them, and
+return their status and other information to the queue.  the priority queue
+itself is made available to all the 'feeding' nodes by virtue of it being placed
+on a globally visible nfs filesystem.
+
+                          -----------
+                          | priority |
+                          | queue    |
+                          -----------
+                         /      |     \
+                        /       |      \
+                       --------nfs-------
+                      /         |        \
+                     /          |         \
+                ----------  ----------  ----------
+                | node_a |  | node_b |  | node_c |
+                ----------  ----------  ----------
+
+all feeding nodes are equal, idependent, and isolated.  any node capable of
+mounting the nfs filesystem may submit to the queue.  so long as the nfs server
+and one node are up - the cluster can continute processing jobs.
+
+
+---
+=== EXAMPLE INSTALL
+---
+
+download rq from http://raa.ruby-lang.org/project/rq
+
+unpack rq-X.X.X.tgz
+
+cd ./depends/
+
+the ./depends/packages/ directory will contain ALL packages required to install
+rq including ruby[www.ruby-lang.org]
+
+the file ./depends/install.sh installs ALL required packages for ruby queue into
+an nfs mounted directory.  it is a simple script - feel free to read/edit.
+
+installed packages will include:
+
+* ruby
+* sqlite
+* sqlite-ruby
+* arrayfields
+* posixlock
+* lockfile
+* rq
+
+the install.sh procedure is reccomended since
+
+* a common nfs filesystem is required on which to store the queue anyhow
+
+* installing all packages into this common nfs filesystem means no
+  software will be installed locally on any node which simplifies maintainance
+  and the addition of new nodes to the cluster.
+
+* root privledges are not required for this technique
+
+* this technique implies only two requirements for any node to join the cluster
+  * the nfs filesystem is visible to it
+  * a single crontab entry has been added
+
+the user running ./depends/install.sh should have write permission (or sudo
+write permission) to the /nfs mounted directory.
+
+following are the steps for an install into the nfs mounted directory named
+'/nfs'.  absolutely NO packages or files will be installed outside this hierachy
+so simply use 'rm -rf' to un-install.  
+  
+  tar xvfz rq-X.X.X.tgz
+
+  cd rq-X.X.X/
+
+  cd depends/
+
+  ./install.sh /nfs  
+
+---
+=== EXAMPLE SETUP
+---
+
+= path setup
+
+the following instructions assume your PATH has been set appropriately on all
+nodes.  your path should be configured to include the bin directory of the nfs
+prefix used in the install.  eg:
+
+  export PATH=/nfs/bin/:$PATH
+
+if this has been done correctly the output of
+
+  which rq
+
+should report something like
+
+  /nfs/bin/rq
+
+/nfs being the location you just installed into
+
+
+= create the queue
+
+you must create/initialize the queue structure.  this should be done one time
+only from any host
+
+  rq /nfs/queue create
+
+
+= configure immortal feeders on all processing nodes
+
+add crontab entries similar to below to keep immortal feeders alive on ALL
+nodes.  this will not start feeders every 15 mintues - it will only attempt to
+start one every 15 minutes, failing silently if one is already running.
+
+  */15 * * * * /nfs/bin/rq /nfs/queue start 
+
+obviously you will need to edit the path to rq, the path to the queue, and
+possibly the log path.
+
+this can done automatically simply by running
+
+  rq /nfs/queue cron start
+
+essentially the crontab entry simply ensures that a feeder daemon is running on
+a node at all times, even after a reboot.  a majority of the time the cron entry
+will do nothing - only when no daemon is running will it be able to generate a
+pidfile and lock it in order to start feeding.  one feeder per queue per host is
+allowed by default.
+
+if you are not quite ready to setup crontab entries on a dozen machines and want
+to start a feeder from a terminal for debugging/testing purposes you might use
+something like this in a separate terminal/host from the one you'll submit jobs
+from:
+
+  /nfs/bin/rq /nfs/queue feed --min_sleep=2 --max_sleep=4
+
+which will log all output to STDERR and poll at a random interval between
+--min_sleep and --max_sleep.  a slight diversion is required here to explain
+min_sleep and max_sleep.
+
+once a feeder finds the queue to be empty it will begin polling the queue
+periodically to see if any new jobs have come in.  min_sleep and max_sleep
+define the lower and upper bound of the polling interval, which is actually
+chosen at random to be between these two numbers for performance reasons.
+
+_never_ would you use the polling interval shown above (min_sleep=2,
+max_sleep=4) in a production environment, such a rapid interval would _only_ be
+for debugging/testing.  
+  
+as reference the defaults for min_sleep and max_sleep are 42 and 240
+respectively, and this gives good responsiveness on a cluster of 12-20 machines.
+you can determine how long a job would sit in the queue, on average, before
+being noticed (assuming all nodes are not busy when the job is submitted) using
+this formula
+
+  median_sleep = ((max_sleep - min_sleep) / 2) + min_sleep
+
+  wait = n_nodes / median_sleep
+
+so, for example, if you use the default min_sleep and max_sleep for 10 feeding
+nodes a job would sit in the queue for, on average, about 10 seconds before
+being picked up.  
+
+it's very important to note all this talk of polling applies _only_ to nodes
+which have found the queue emtpy and are checking back at a regular interval to
+look for more work.  while the queue is full of jobs each node simply works to
+capacity to take jobs, runs them, and return them to the queue - so there is no
+polling.  in fact, it's best to consider the min_sleep and max_sleep options as
+affecting how quickly the cluster as a whole will tackle a freshly loaded queue:
+once the queue is loaded every node is 'too busy' to poll any longer.
+
+one other option which should be mentioned here is the --max_feed, -f option.
+this option simply determines the number of concurrent jobs rq will run at once.
+the default value is 2 - therefore two jobs will be running simoultaneously on
+each feeding node when the queue has been loaded with jobs.  this command
+
+  /nfs/bin/rq /nfs/queue feed --min_sleep=2 --max_sleep=4 --max_feed=1
+
+might be used in a terminal (logging to stderr) to watch rq in action.  limiting
+the number of feeders to 1 makes the output somewhat easier to understand.
+however, unless your machines cannot handle more than one of the jobs you plan
+to submit it's best to keep this number > 1 for production clusters.
+
+
+---
+=== EXAMPLE USAGE
+---
+
+
+= job submission
+
+submit a command line job to the nfs mounted queue
+
+  rq /nfs/queue submit 'echo 42'
+
+in this case the quoting is not needing but beware of shell expansion when
+submitting jobs from the shell
+
+submit a list of jobs from 'jobfile'.  jobfile is simply a text file with one
+command per line.  blank lines and comments (#) are ignored.
+
+  rq /nfs/queue submit - < jobfile 
+
+= cluster status
+
+check the status of your jobs (note that it may take cron a while to start
+feeders depending on the interval used to maintain them in your crontab file)
+
+  rq /nfs/queue status 
+
+= where to go from here
+
+for more info try
+
+  rq help
+
+
+---
+=== AUTHOR/ASSISTANCE/BUG REPORTS
+---
+
+  ara.t.howard@noaa.gov
+

data/all/install.rb

@@ -0,0 +1,127 @@
+class Installer
+  $VERBOSE = nil
+  getenv = lambda{|key| ENV[key] or abort("no ENV['#{ key }']")}
+  PREFIX = getenv["prefix"]
+  MAKE = getenv["make"] || getenv["MAKE"]
+  PACKAGEDIR = getenv["packagedir"]
+  BUILDDIR = getenv["builddir"]
+  SUDO = getenv["sudo"]
+  RUBY = getenv["ruby"]
+  LDFLAGS = getenv["LDFLAGS"]
+  CFLAGS = getenv["CFLAGS"]
+
+  INSTALL = "#{ PACKAGEDIR }/INSTALL"
+
+  def run
+    manifest = test ?e, INSTALL 
+
+    packages = 
+      if manifest 
+        lines = IO.readlines INSTALL
+        lines.delete_if{|line| line.strip.empty? or line =~ %r/^\s*#/}
+        lines.map{|line| File.join(PACKAGEDIR, File.basename(line.strip))}
+      else
+        filelist "#{ PACKAGEDIR }/*.{tar.gz,tgz}"
+      end
+
+    packages.delete_if{|package| package.strip.empty?}
+    packages.delete_if{|package| File.basename(package) !~ %r/\.(tar\.gz|tgz)$/}
+    packages.delete_if{|package| File.basename(package) =~ %r/^ruby.*\.(tar\.gz|tgz)$/}
+
+    Dir.chdir(BUILDDIR){
+      unless manifest
+        packages = packages.sort_by do |package|
+          lines = `tar tfz #{ package }`.split %/\n/
+          line = lines.grep(%r/(install\.rb|extconf\.rb|configure)/).first
+          dirname, first, rest = line.split(File::SEPARATOR, 2)
+          type = first unless rest
+          abort "cannot determine type of <#{ package }>" unless type
+          case type
+            when /configure/
+              0
+            when /extconf.rb/
+              1
+            when /install.rb/
+              2
+          end
+        end
+      end
+
+      packages.each{|package| install_package package}
+    }
+
+    Dir.chdir(".."){
+      pwd = File.basename(Dir.pwd)
+      log = "#{ pwd }.log" 
+      if test ?e, "install.rb"
+        puts "installing #{ pwd } (see #{ log } for details)..."
+        install_ruby_package log
+      end
+      if test ?e, "extconf.rb"
+        puts "installing #{ pwd } (see #{ log } for details)..."
+        install_ruby_extentsion log
+      end
+      if test ?e, "configure"
+        puts "installing #{ pwd } (see #{ log } for details)..."
+        install_src_package log
+      end
+    }
+  end
+  def install_package package
+    case package
+      when /(tgz|tar.gz)$/
+        install_from_tgz package
+      else
+        abort "can't install package <#{ package }>"
+    end
+  end
+  def install_from_tgz tgz
+    old = filelist '*' 
+    spawn "tar xvfz #{ tgz } >/dev/null 2>&1"
+    new = filelist '*'
+    created = new - old
+    expected = File.join(BUILDDIR, File.basename(tgz).gsub(%r/(\.tar\.gz|\.tgz)$/,''))
+    dir = created.first || expected 
+    log = "#{ BUILDDIR }/#{ File.basename tgz }.log"
+
+    Dir.chdir(dir){
+      puts "installing #{ tgz } (see #{ log } for details)..."
+      status = nil
+      if test ?e, 'install.rb'
+        status = install_ruby_package(log)
+      end
+      if test ?e, 'extconf.rb'
+        status = install_ruby_extentsion(log)
+      end
+      if test ?e, 'configure'
+        status = install_src_package(log)
+      end
+      if status
+        puts "success"
+      else
+        puts "failure"
+        exit 1
+      end
+    }
+  end
+  def install_ruby_package log
+    system "{ #{ SUDO } #{ RUBY } install.rb '--with-cflags=#{ CFLAGS }' '--with-ldflags=#{ LDFLAGS }'; } >#{ log } 2>&1" or
+      system "{ #{ SUDO } #{ RUBY } install.rb; } >#{ log } 2>&1"
+  end
+  def install_ruby_extentsion log
+    system "{ #{ RUBY } extconf.rb '--with-cflags=#{ CFLAGS }' '--with-ldflags=#{ LDFLAGS }' && stat [mM]akefile && #{ MAKE } && #{ SUDO } #{ MAKE } install; } >#{ log } 2>&1" or
+      system "{ #{ RUBY } extconf.rb && stat [mM]akefile && #{ MAKE } && #{ SUDO } #{ MAKE } install; } >#{ log } 2>&1"
+  end
+  def install_src_package log
+    #system "{ #{ MAKE } clean; } >/dev/null 2>&1"
+    system "{ ./configure --prefix=#{ PREFIX } && stat [mM]akefile && #{ MAKE } && #{ SUDO } #{ MAKE } install; } >#{ log } 2>&1"
+  end
+  def spawn cmd
+    system cmd or abort "cmd <#{ cmd }> failed"
+  end
+  def filelist glob
+    Dir.glob(glob).map{|path| File.expand_path path}.sort.uniq
+  end
+end
+
+Installer.new.run

data/all/install.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+
+
+###############################################################################
+#
+# CONFIG SECTION - EDIT TO SUIT YOUR SITE 
+#
+###############################################################################
+#
+# prefix : everything will be installed under this directory.  it should be an
+# absolute pathname.
+#
+  export prefix="$1"
+#
+# libdir : libs will installed here
+#
+  export libdir="${prefix}/lib"
+#
+# incdir : libs will installed here
+#
+  export incdir="${prefix}/include"
+#
+# bindir : executables and scripts will installed here
+#
+  export bindir="${prefix}/bin"
+#
+# sudo : set to "sudo" if you need this to install into prefix
+#
+  export sudo="" 
+  # export sudo="sudo"
+#
+# full path of dir containing packages to be installed
+#
+  export packagedir="`pwd`/packages"
+#
+# full path of dir where packages will be built 
+#
+  export builddir="`pwd`/build"
+#
+# set this to true if you want to install ruby into prefix too.  otherwise it
+# automatically determines if ruby needs to be installed.
+#
+  install_ruby=`test -f ${bindir}/ruby && echo false || echo true`
+  install_gem=`test -f ${bindir}/gem && echo false || echo true`
+#
+# determine ruby tar ball - this just picks up latest version
+#
+  ruby_tgz=`ls ${packagedir}/ruby-*gz|sort|tail -1`
+
+
+
+###############################################################################
+#
+# INSTALL SECTION - DO **NOT** EDIT
+#
+###############################################################################
+#
+# vars 
+#
+  div="==============================================================================="
+  line="-------------------------------------------------------------------------------"
+  usage="usage : $0 prefix"
+#
+# check/create prefix
+#
+  if [[ ! -n "$prefix" ]]; then
+    printf "$usage\n"
+    exit 1
+  fi
+  if [[ ! -d "$prefix" ]]; then
+    $sudo mkdir -p $prefix 
+  fi
+  if [[ ! -d "$prefix" ]]; then
+    "prefix <$prefix> does not exist and could not be created"
+    exit 1
+  fi
+#
+# info 
+#
+  printf "\n$div\n"
+  printf "CONFIG\n"
+  printf -- "$line\n"
+  printf "prefix <$prefix>\n"
+  printf "libdir <$libdir>\n"
+  printf "incdir <$incdir>\n"
+  printf "bindir <$bindir>\n"
+  printf "packagedir <$packagedir>\n"
+  printf "builddir <$builddir>\n"
+  printf -- "$line\n"
+#
+# important env settings for proper compilation
+#
+  export LD_RUN_PATH="${libdir}"
+  export LD_LIBRARY_PATH="${libdir}"
+  export LDFLAGS="-L ${libdir}"
+  export CFLAGS="-I ${incdir}"
+  export PATH="${bindir}:$PATH"
+#
+# important aliases for proper complilation and installation
+#
+  export make="env LD_RUN_PATH=${libdir} LD_LIBRARY_PATH=${libdir} make"
+  export MAKE="env LD_RUN_PATH=${libdir} LD_LIBRARY_PATH=${libdir} make"
+  export ruby="${bindir}/ruby"
+#
+# pre-install
+#
+  cwd=`pwd`
+  pushd . >/dev/null
+  mkdir -p "$builddir"
+  if [[ $? != 0 ]];then
+    printf "COULD NOT CREATE BUILDIR <${builddir}>" 1>&2
+    popd >/dev/null
+    exit 1
+  fi
+#
+# install ruby iff needed 
+#
+  if [[ "$install_ruby" == *true* ]]; then
+    cd "$builddir"
+    log="${builddir}/`basename $ruby_tgz`.log"
+    printf "\n$div\n"
+    printf "installing $ruby_tgz (see $log for details)...\n"
+
+    (tar xvfz "$ruby_tgz" &&\
+     cd ruby* &&\
+    ./configure "--prefix=${prefix}" &&\
+    $make &&\
+    $sudo $make install) > "$log" 2>&1
+
+    if [[ $? == 0 ]];then
+      printf "success\n"
+    else
+      printf "failure\n"
+      cd "$cwd"
+      exit 1
+    fi
+    printf -- "$line\n"
+    cd "$cwd"
+  fi
+#
+# bootstrap ruby to install everything else
+#
+  $ruby ./install.rb || exit 1
+#
+# serve notice
+#
+  printf -- "\n$div\n"
+  printf "ATTENTION\n"
+
+  printf -- "$line\n"
+  printf "IT APPEARS THAT INSTALLATION WAS COMPLETE AND SUCCESSFUL\n"
+  printf "\n"
+  printf "THE FOLLOWING ENVIRONMENT SETTINGS ARE RECCOMENDED WHEN RUNNING THE SOFTWARE\n"
+  printf "  bash/sh:\n"
+  printf "    export PATH=\"${bindir}:\$PATH\"\n"
+  printf "    export LD_LIBRARY_PATH=\"${libdir}\"\n"
+  printf "  tcsh/csh:\n"
+  printf "    setenv PATH \"${bindir}:\$PATH\"\n"
+  printf "    setenv LD_LIBRARY_PATH \"${libdir}\"\n"
+  printf "\n"
+#
+# post-install 
+#
+  popd >/dev/null
+  exit 0