rq-ruby1.8 3.4.3

data/lib/rq/util.rb

@@ -0,0 +1,304 @@
+unless defined? $__rq_util__
+  module RQ 
+#--{{{
+    LIBDIR = File::dirname(File::expand_path(__FILE__)) + File::SEPARATOR unless
+      defined? LIBDIR
+
+    require 'pathname'
+    require 'socket'
+    require 'tmpdir'
+
+    #
+    # of course - all the rest goes here
+    #
+    module  Util
+#--{{{
+      class << self
+        def export sym
+#--{{{
+          sym = "#{ sym }".intern
+          module_function sym 
+          public sym 
+#--}}}
+        end
+        def append_features c
+#--{{{
+          super
+          c.extend Util 
+#--}}}
+        end
+      end
+      def mcp obj
+#--{{{
+        Marshal.load(Marshal.dump(obj))
+#--}}}
+      end
+      export 'mcp'
+      def klass
+#--{{{
+        self.class
+#--}}}
+      end
+      export 'klass'
+      def realpath path
+#--{{{
+        path = File::expand_path "#{ path }"
+        begin
+          Pathname::new(path).realpath.to_s
+        rescue Errno::ENOENT, Errno::ENOTDIR
+          path
+        end
+#--}}}
+      end
+      export 'realpath'
+      def hashify(*hashes)
+#--{{{
+        hashes.inject(accum={}){|accum,hash| accum.update hash}
+#--}}}
+      end
+      export 'hashify'
+      def getopt opt, hash, default = nil
+#--{{{
+        key = opt
+        return hash[key] if hash.has_key? key
+
+        key = "#{ key }"
+        return hash[key] if hash.has_key? key
+
+        key = key.intern 
+        return hash[key] if hash.has_key? key
+
+        return default
+#--}}}
+      end
+      export 'getopt'
+      def alive? pid
+#--{{{
+        pid = Integer("#{ pid }")
+        begin
+          Process.kill 0, pid
+          true
+        rescue Errno::ESRCH
+          false
+        end
+#--}}}
+      end
+      export 'alive?'
+      def maim(pid, opts = {})
+#--{{{
+        sigs = getopt('signals', opts) || %w(SIGTERM SIGQUIT SIGKILL) 
+        suspend = getopt('suspend', opts) || 4
+        pid = Integer("#{ pid }")
+        sigs.each do |sig|
+          begin
+            Process.kill(sig, pid)
+          rescue Errno::ESRCH
+            return nil
+          end
+          sleep 0.2
+          unless alive?(pid)
+            break
+          else
+            sleep suspend
+          end
+        end
+        not alive?(pid)
+#--}}}
+      end
+      export 'maim'
+      def timestamp time = Time.now
+#--{{{
+        usec = "#{ time.usec }"
+        usec << ('0' * (6 - usec.size)) if usec.size < 6 
+        time.strftime('%Y-%m-%d %H:%M:%S.') << usec
+#--}}}
+      end
+      export 'timestamp'
+      def stamptime string, local = true 
+#--{{{
+        string = "#{ string }"
+        pat = %r/^\s*(\d\d\d\d)-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d).(\d\d\d\d\d\d)\s*$/o
+        match = pat.match string
+        raise ArgumentError, "<#{ string.inspect }>" unless match
+        yyyy,mm,dd,h,m,s,u = match.to_a[1..-1].map{|m| m.to_i}
+        if local
+          Time.local yyyy,mm,dd,h,m,s,u
+        else
+          Time.gm yyyy,mm,dd,h,m,s,u
+        end
+#--}}}
+      end
+      export 'stamptime'
+      def escape! s, char, esc
+#--{{{
+        re = %r/([#{0x5c.chr << esc}]*)#{char}/
+        s.gsub!(re) do
+          (($1.size % 2 == 0) ? ($1 << esc) : $1) + char 
+        end
+#--}}}
+      end
+      export 'escape!'
+      def escape s, char, esc
+#--{{{
+        ss = "#{ s }"
+        escape! ss, char, esc
+        ss
+#--}}}
+      end
+      export 'escape'
+      def fork(*args, &block)
+#--{{{
+        begin
+          verbose = $VERBOSE
+          $VERBOSE = nil
+          Process::fork(*args, &block)
+        ensure
+          $VERBOSE = verbose
+        end
+#--}}}
+      end
+      export 'fork'
+      def exec(*args, &block)
+#--{{{
+        begin
+          verbose = $VERBOSE
+          $VERBOSE = nil
+          Kernel::exec(*args, &block)
+        ensure
+          $VERBOSE = verbose
+        end
+#--}}}
+      end
+      export 'exec'
+      def system(*args, &block)
+#--{{{
+        begin
+          verbose = $VERBOSE
+          $VERBOSE = nil
+          Kernel::system(*args, &block)
+        ensure
+          $VERBOSE = verbose
+        end
+#--}}}
+      end
+      export 'system'
+      def hostname
+#--{{{
+        @__hostname__ ||= Socket::gethostname
+#--}}}
+      end
+      export 'hostname'
+      def host
+#--{{{
+        @__host__ ||= Socket::gethostname.gsub(%r/\..*$/o,'')
+#--}}}
+      end
+      export 'host'
+      def emsg e
+#--{{{
+        "#{ e.message } - (#{ e.class })"
+#--}}}
+      end
+      export 'emsg'
+      def btrace e
+#--{{{
+        (e.backtrace or []).join("\n")
+#--}}}
+      end
+      export 'btrace'
+      def errmsg e
+#--{{{
+        emsg(e) << "\n" << btrace(e)
+#--}}}
+      end
+      export 'errmsg'
+      def erreq a, b
+#--{{{
+        a.class == b.class and
+        a.message == b.message and
+        a.backtrace == b.backtrace
+#--}}}
+      end
+      export 'erreq'
+      def tmpnam dir = Dir.tmpdir, seed = File::basename($0)
+#--{{{
+        pid = Process.pid
+        path = "%s_%s_%s_%s_%d" % 
+          [Util::hostname, seed, pid, Util::timestamp.gsub(/\s+/o,'_'), rand(101010)]
+        File::join(dir, path)
+#--}}}
+      end
+      export 'tmpnam'
+      def uncache file 
+#--{{{
+        refresh = nil
+        begin
+          is_a_file = File === file
+          path = (is_a_file ? file.path : file.to_s) 
+          stat = (is_a_file ? file.stat : File::stat(file.to_s)) 
+          refresh = tmpnam(File::dirname(path))
+          File::link path, refresh rescue File::symlink path, refresh
+          File::chmod stat.mode, path
+          File::utime stat.atime, stat.mtime, path
+        ensure 
+          begin
+            File::unlink refresh if refresh
+          rescue Errno::ENOENT
+          end
+        end
+#--}}}
+      end
+      export 'uncache'
+      def columnize buf, width = 80, indent = 0
+#--{{{
+        column = []
+        words = buf.split %r/\s+/o
+        row = ' ' * indent
+        while((word = words.shift))
+          if((row.size + word.size) < (width - 1))
+            row << word
+          else
+            column << row
+            row = ' ' * indent
+            row << word
+          end
+          row << ' ' unless row.size == (width - 1)
+        end
+        column << row unless row.strip.empty?
+        column.join "\n"
+#--}}}
+      end
+      export 'columnize'
+      def defval var, default = nil
+#--{{{
+        v = "#{ var }"
+        c = "DEFAULT_#{ v }".upcase
+        begin
+          klass.send(v) || klass.const_get(c)
+        rescue NameError
+          default
+        end
+#--}}}
+      end
+      export 'defval'
+      def hms s
+#--{{{
+        h, s = s.divmod 3600
+        m, s = s.divmod 60
+        [h.to_i, m.to_i, s]
+#--}}}
+      end
+      export 'hms'
+      def which_ruby
+#--{{{
+        c = ::Config::CONFIG
+        realpath( File::join(c['bindir'], c['ruby_install_name']) << c['EXEEXT'] )
+#--}}}
+      end
+      export 'which_ruby'
+#--}}}
+    end # module Util
+#--}}}
+  end # module RQ
+$__rq_util__ = __FILE__ 
+end

data/rdoc.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+#
+# workaround so bin/* gets doc'd as ruby files
+#
+
+pushd .
+cd bin
+for f in *; do ln -s $f "$f.rb"; done
+popd
+
+rdoc -a -d -F -S -m README -I jpg -N [A-Z]* bin/rq.rb lib/rq.rb lib/*/*
+
+pushd .
+cd bin
+for f in *rb; do rm -f $f; done
+popd

data/rq-ruby1.8.gemspec

@@ -0,0 +1,120 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{rq-ruby1.8}
+  s.version = "3.4.3"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Pjotr Prins"]
+  s.date = %q{2011-07-21}
+  s.description = %q{Zero configuration job scheduler for computer clusters}
+  s.email = %q{pjotr.public01@thebird.nl}
+  s.executables = ["rqmailer", "rq"]
+  s.extra_rdoc_files = [
+    "LICENSE",
+    "README",
+    "TODO"
+  ]
+  s.files = [
+    "Gemfile",
+    "Gemfile.lock",
+    "INSTALL",
+    "LICENSE",
+    "Makefile",
+    "README",
+    "Rakefile",
+    "TODO",
+    "TUTORIAL",
+    "VERSION",
+    "bin/rq",
+    "bin/rqmailer",
+    "example/a.rb",
+    "extconf.rb",
+    "gemspec.rb",
+    "install.rb",
+    "lib/rq.rb",
+    "lib/rq/arrayfields.rb",
+    "lib/rq/backer.rb",
+    "lib/rq/configfile.rb",
+    "lib/rq/configurator.rb",
+    "lib/rq/creator.rb",
+    "lib/rq/cron.rb",
+    "lib/rq/defaultconfig.txt",
+    "lib/rq/deleter.rb",
+    "lib/rq/executor.rb",
+    "lib/rq/feeder.rb",
+    "lib/rq/ioviewer.rb",
+    "lib/rq/job.rb",
+    "lib/rq/jobqueue.rb",
+    "lib/rq/jobrunner.rb",
+    "lib/rq/jobrunnerdaemon.rb",
+    "lib/rq/lister.rb",
+    "lib/rq/locker.rb",
+    "lib/rq/lockfile.rb",
+    "lib/rq/logging.rb",
+    "lib/rq/mainhelper.rb",
+    "lib/rq/orderedautohash.rb",
+    "lib/rq/orderedhash.rb",
+    "lib/rq/qdb.rb",
+    "lib/rq/querier.rb",
+    "lib/rq/rails.rb",
+    "lib/rq/recoverer.rb",
+    "lib/rq/refresher.rb",
+    "lib/rq/relayer.rb",
+    "lib/rq/resource.rb",
+    "lib/rq/resourcemanager.rb",
+    "lib/rq/resubmitter.rb",
+    "lib/rq/rotater.rb",
+    "lib/rq/sleepcycle.rb",
+    "lib/rq/snapshotter.rb",
+    "lib/rq/sqlite.rb",
+    "lib/rq/statuslister.rb",
+    "lib/rq/submitter.rb",
+    "lib/rq/toucher.rb",
+    "lib/rq/updater.rb",
+    "lib/rq/usage.rb",
+    "lib/rq/util.rb",
+    "rdoc.sh",
+    "rq-ruby1.8.gemspec",
+    "test/.gitignore",
+    "test/test_rq.rb",
+    "white_box/crontab",
+    "white_box/joblist",
+    "white_box/killrq",
+    "white_box/rq_killer"
+  ]
+  s.homepage = %q{http://github.com/pjotrp/rq}
+  s.licenses = ["BSD"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{Ruby Queue scheduler}
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<posixlock>, [">= 0"])
+      s.add_runtime_dependency(%q<arrayfields>, [">= 0"])
+      s.add_runtime_dependency(%q<lockfile>, [">= 0"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0.15"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
+    else
+      s.add_dependency(%q<posixlock>, [">= 0"])
+      s.add_dependency(%q<arrayfields>, [">= 0"])
+      s.add_dependency(%q<lockfile>, [">= 0"])
+      s.add_dependency(%q<bundler>, ["~> 1.0.15"])
+      s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+    end
+  else
+    s.add_dependency(%q<posixlock>, [">= 0"])
+    s.add_dependency(%q<arrayfields>, [">= 0"])
+    s.add_dependency(%q<lockfile>, [">= 0"])
+    s.add_dependency(%q<bundler>, ["~> 1.0.15"])
+    s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+  end
+end
+

data/test/.gitignore

@@ -0,0 +1 @@
+test_queue

data/test/test_rq.rb

@@ -0,0 +1,145 @@
+#!/usr/bin/ruby
+#
+# Test frame work for rq - tests submitting jobs to a queue,
+# and the handling of them. Does not test NFS (yet).
+#
+# Copyright (C) 2011 Pjotr Prins <pjotr.prins@thebird.nl> 
+
+require 'yaml'
+
+print "rq integration test suite by Pjotr Prins 2011\n"
+
+src = File.join('..',File.dirname(__FILE__))
+$rq = File.join(src,'bin','rq');
+raise "Run from test folder!" if !File.executable?($rq)
+$queue = './test_queue'
+
+def rq_exec(args)
+  cmd = $rq + ' ' + $queue + ' ' + args
+  print "====> ",cmd," <===="
+  system(cmd)
+end
+
+def rq_status()
+  cmd = $rq + ' ' + $queue + ' status'
+  YAML.load(`#{cmd}`)
+end
+
+def error(line, msg)
+  p rq_status()
+  rq_exec('shutdown')
+  $stderr.print(__FILE__," ",line,": ",msg)
+  exit 1
+end
+
+def test_equal(line,s1,s2) 
+  error(line,"#{s1} does not equal #{s2}") if s1 != s2
+end
+
+def kill_rq()
+  pstab = `ps xau|grep rq`
+  pstab.grep(/#{$rq}/) do | s |
+    # rq_exec('shutdown')
+    s =~ /\S+\s+(\d+)/
+    pid = $1
+    print "+++#{pid}+++\n"
+    system("kill -9 #{pid}")
+    sleep(1)
+  end
+  print `rm -rf #{$queue}`
+  pstab = `ps xau|grep rq`
+  pstab.grep(/#{$rq}/) do | s |
+    error(__LINE__,"Sorry, still running:\n"+s)
+  end
+  pstab = `ps xau|grep rq`
+  pstab.grep(/rq_jobrunnerdaemon/) do | s |
+    s =~ /\S+\s+(\d+)/
+    pid = $1
+    print "+++#{pid}+++\n"
+    system("kill -9 #{pid}")
+    sleep(1)
+  end
+  pstab = `ps xau|grep rq`
+  pstab.grep(/rq_jobrunnerdaemon/) do | s |
+    error(__LINE__,"Still running "+s)
+  end
+end
+
+
+# Check dependencies
+print "Running tests...\n"
+
+# $:.unshift '/var/lib/gems/1.8/gems/sqlite-ruby-2.2.3/lib/'
+# test_equal(__LINE__,SQLite::Version::STRING,"2.2.3")
+
+gempath = '/var/lib/gems/1.8/gems/sqlite-1.3.1/lib'
+error("Expect "+gempath) if !File.directory?(gempath)
+$:.unshift gempath
+require 'sqlite'
+
+kill_rq()
+
+# In the first step we set up the queue
+print `rm -rf #{$queue}`
+rq_exec("create")
+p rq_status()
+
+# get status
+status = rq_status()
+test_equal(__LINE__,status["exit_status"],{"failures"=>0, "ok"=>0, "successes"=>0})
+
+# submit short job
+rq_exec('submit "/bin/ls /etc"')
+
+# first time is pending
+test_equal(__LINE__,rq_status()['jobs']['pending'],1)
+
+rq_exec('submit "/bin/ls /etc"')
+p rq_status()
+test_equal(__LINE__,rq_status()['jobs']['total'],2)
+
+# fire up daemon
+rq_exec("feed --daemon --log=rq.log --max_feed=1 --min_sleep 1 --max_sleep 1")
+5
+sleep(2)
+test_equal(__LINE__,rq_status()['jobs']['total'],2)
+
+# get status
+status = rq_status()
+p status
+test_equal(__LINE__,status["jobs"]['total'],2)
+
+# Now add longer jobs
+rq_exec('submit "sleep 15"')  # does not finish
+sleep(1)
+status = rq_status()
+test_equal(__LINE__,status['jobs']['pending']+status['jobs']['running'],1)
+rq_exec('submit "sleep 15"')   # will be removed
+p rq_status()
+test_equal(__LINE__,rq_status()['jobs']['total'],4)
+
+# Delete last job
+rq_exec('delete 4')
+sleep(2)
+print "\n---\n"
+print rq_exec('query state=pending')
+p rq_status
+
+# Kill rq
+rq_exec('shutdown')
+status = rq_status()
+test_equal(__LINE__,status['jobs']['pending']+status['jobs']['running'],1)
+sleep(1)
+p rq_status
+
+# Done!
+print <<MSG
+
+Congratulations, all tests passed!
+
+This means rq accepts and executes jobs. Note, the tests 
+are not exhaustive, but passing them is a good sign.
+
+MSG
+
+