rq 0.1.7

data/lib/rq-0.1.7/jobrunner.rb

@@ -0,0 +1,63 @@
+unless defined? $__rq_jobrunner__
+  module RQ 
+#{{{
+    LIBDIR = File::dirname(File::expand_path(__FILE__)) + File::SEPARATOR unless
+      defined? LIBDIR
+
+    require 'drb/drb'
+
+    require LIBDIR + 'util'
+
+    class  JobRunner
+#{{{
+      include DRbUndumped
+      attr :job
+      attr :jid
+      attr :cid
+      attr :shell
+      attr :command
+      alias pid cid
+      def initialize job
+#{{{
+        @job = job
+        @jid = job['jid']
+        @command = job['command']
+        @shell = job['shell'] || 'bash'
+        @r,@w = IO::pipe
+        @env = {}
+        @job.fields.each do |field|
+          key = "RQ_#{ field }".upcase.gsub(%r/\s+/,'_')
+          val = @job[field]
+          @env[key] = "#{ val }"
+        end
+        @cid = 
+          Util::fork do
+            begin
+              @env.each{|k,v| ENV[k] = v}
+              ENV['RQ_PID'] = "#{ $$ }"
+              @w.close
+              STDIN.reopen @r
+              if File::basename(@shell) == 'bash' || File::basename(@shell) == 'sh' 
+                exec [@shell, "__rq_job__#{ @jid }__#{ File::basename(@shell) }__"], '--login'
+              else
+                exec [@shell, "__rq_job__#{ @jid }__#{ File::basename(@shell) }__"], '-l'
+              end
+            rescue Exception => e
+              STDERR.puts(Util::errmsg(e))
+            end
+          end
+        @r.close
+#}}}
+      end
+      def run
+#{{{
+        @w.puts @command
+        @w.close
+#}}}
+      end
+#}}}
+    end # class JobRunner
+#}}}
+  end # module RQ
+$__rq_jobrunner__ = __FILE__ 
+end

data/lib/rq-0.1.7/jobrunnerdaemon.rb

@@ -0,0 +1,179 @@
+unless defined? $__rq_jobrunnerdaemon__
+  module RQ 
+#{{{
+    LIBDIR = File::dirname(File::expand_path(__FILE__)) + File::SEPARATOR unless
+      defined? LIBDIR
+
+    require 'drb/drb'
+    require 'fileutils'
+    require 'tmpdir'
+    require 'tempfile'
+
+    require LIBDIR + 'job'
+    require LIBDIR + 'jobrunner'
+
+    class  JobRunnerDaemon
+#{{{
+      include Logging
+
+      class << self
+#{{{
+        def daemon(*a,&b)
+#{{{
+          jrd = new(*a, &b) 
+
+          r, w = IO::pipe
+
+          unless((pid = fork)) # child
+            $0 = "#{ self }".gsub(%r/[^a-zA-Z]+/,'_').downcase
+            begin
+              r.close
+              n = 0
+              uri = nil
+              socket = nil
+
+              42.times do
+                begin
+                  s = "%s/%s_%s_%s_%s" %
+                    [Dir::tmpdir, File::basename($0), Process::ppid, n, rand(42)]
+                  u = "drbunix://#{ s }"
+                  DRb::start_service u, jrd 
+                  socket = s
+                  uri = u
+                rescue Errno::EADDRINUSE
+                  n += 1
+                end
+              end
+
+              if socket and uri
+                w.write socket 
+                w.close
+                pid = Process::pid
+                ppid = Process::ppid
+                cur = Thread::current
+                Thread::new(pid, ppid, cur) do |pid, ppid, cur|
+                  loop do
+                    begin
+                      Process::kill 0, ppid
+                      sleep 42
+                    rescue
+                      cur.raise "parent <#{ ppid }> died unexpectedly" 
+                    end
+                  end
+                end
+                DRb::thread.join
+              else
+                w.close
+              end
+            ensure
+              exit!
+            end
+          else # parent
+            w.close
+            socket = r.read
+            r.close
+
+            if socket and File::exist?(socket)
+              at_exit{ FileUtils::rm_f socket }
+              uri = "drbunix://#{ socket }"
+              DRb::start_service 'druby://localhost:0', nil
+              jrd = DRbObject::new nil, uri
+              jrd.pid = pid
+              jrd.uri = uri
+            else
+              raise "failed to start job runner daemon"
+            end
+          end
+
+          return jrd
+#}}}
+        end
+#}}}
+      end
+      attr :runners
+      attr :pid, true
+      attr :uri, true
+      def initialize
+#{{{
+        @runners = {}
+        @uri = nil
+        @pid = Process::pid 
+#}}}
+      end
+      def runner job 
+#{{{
+        r = nil
+        retried = false
+        begin
+          r = JobRunner::new job
+        rescue Errno::ENOMEM, Errno::EAGAIN
+          GC::start
+          unless retried
+            retried = true 
+            retry
+          else
+            raise
+          end
+        end
+        @runners[r.pid] = r
+        r
+#}}}
+      end
+      def wait
+#{{{
+        pid = Process::wait
+        @runners.delete pid
+        pid
+#}}}
+      end
+      def wait2
+#{{{
+        pid, status = Process::wait2
+        @runners.delete pid
+        [pid, status]
+#}}}
+      end
+      def waitpid pid = -1, flags = 0 
+#{{{
+        pid = pid.pid if pid.respond_to? 'pid'
+        pid = Process::waitpid pid, flags 
+        @runners.delete pid
+        pid
+#}}}
+      end
+      def waitpid2 pid = -1, flags = 0 
+#{{{
+        pid = pid.pid if pid.respond_to? 'pid'
+        pid, status = Process::waitpid2 pid, flags 
+        @runners.delete pid
+        [pid, status]
+#}}}
+      end
+      def shutdown
+#{{{
+        @death =
+          Thread::new do
+            begin
+              while not @runners.empty?
+                pid = Process::wait 
+                @runners.delete pid
+              end
+            ensure
+              sleep 4.2
+              DRb::thread.kill
+              Thread::main exit!
+            end
+          end
+#}}}
+      end
+      def install_signal_handlers
+#{{{
+        %w(TERM INT HUP).each{|sig| trap sig, 'SIG_IGN'}
+#}}}
+      end
+#}}}
+    end # class JobRunnerDaemon
+#}}}
+  end # module RQ
+$__rq_jobrunnerdaemon__ = __FILE__ 
+end

data/lib/rq-0.1.7/lister.rb

@@ -0,0 +1,22 @@
+unless defined? $__rq_lister__
+  module RQ 
+#{{{
+    LIBDIR = File::dirname(File::expand_path(__FILE__)) + File::SEPARATOR unless
+      defined? LIBDIR
+
+    require LIBDIR + 'mainhelper'
+
+    class  Lister < MainHelper
+#{{{
+      def list 
+    #{{{
+        set_q
+        @q.list(*@argv)
+    #}}}
+      end
+#}}}
+    end # class Lister
+#}}}
+  end # module RQ
+$__rq_lister__ = __FILE__ 
+end

data/lib/rq-0.1.7/locker.rb

@@ -0,0 +1,37 @@
+unless defined? $__rq_locker__
+  module RQ 
+#{{{
+    LIBDIR = File::dirname(File::expand_path(__FILE__)) + File::SEPARATOR unless
+      defined? LIBDIR
+
+    require LIBDIR + 'util'
+    require LIBDIR + 'locker'
+
+    class  Locker < MainHelper
+#{{{
+      def lock
+    #{{{
+        set_q
+        ltype = @argv.shift
+        debug{ "ltype <#{ ltype }>" }
+        read_only =
+          case ltype
+            when /^\s*r(?:ead)?|^\s*sh(?:ared)?/io
+              true
+            when /^\s*w(?:rite)?|^\s*ex(?:clusive)?/io
+              false
+            else
+              raise "lock type must be one of (r)ead|(sh)ared|(w)rite|(ex)clusive, not <#{ ltype }>"
+          end
+        cmd = @argv.join(' ').strip
+        raise "no command given for lock type <#{ ltype }>" if cmd.empty?
+        debug{ "cmd <#{ cmd }>" }
+        @q.lock(:read_only => read_only){ Util::system cmd }
+    #}}}
+      end
+#}}}
+    end # class Locker
+#}}}
+  end # module RQ
+$__rq_locker__ = __FILE__ 
+end

data/lib/rq-0.1.7/logging.rb

@@ -0,0 +1,117 @@
+unless defined? $__rq_logging__
+  module RQ 
+#{{{
+    LIBDIR = File::dirname(File::expand_path(__FILE__)) + File::SEPARATOR unless
+      defined? LIBDIR
+
+    require "logger"
+
+    require LIBDIR + 'util'
+#
+# module which adds logging methods to classes
+#
+    module Logging
+#{{{
+    #
+    # a module that adds an accessor to Logging objects in ored to fix a bug where
+    # not all logging devices are put into sync mode, resulting in improper log
+    # rolling.  this is a hack.
+    #
+      module LoggerExt
+#{{{
+        attr :logdev
+#}}}
+      end # module LoggerExt
+    #
+    # implementations of the methods shared by both classes and objects of classes
+    # which include Logging
+    #
+      module LogMethods
+#{{{
+        def logger
+#{{{
+          if defined?(@logger) and @logger
+            @logger
+          else
+            if Class === self
+              @logger = self.default_logger
+            else
+              @logger = self::class::logger
+            end
+            raise "@logger is undefined!" unless defined?(@logger) and @logger
+            @logger
+          end
+#}}}
+        end
+        def logger= log
+#{{{
+          @logger = log
+          @logger.extend LoggerExt
+          @logger.logdev.dev.sync = true
+          @logger
+#}}}
+        end
+        def debug(*args, &block); logger.debug(*args, &block); end
+        def info(*args, &block);  logger.info(*args, &block) ; end
+        def warn(*args, &block);  logger.warn(*args, &block) ; end
+        def error(*args, &block); logger.error(*args, &block); end
+        def fatal(*args, &block); logger.fatal(*args, &block); end
+        def logerr e
+#{{{
+          if logger.debug?
+            error{ Util::errmsg e } 
+          else
+            error{ Util::emsg e } 
+          end
+#}}}
+        end
+#}}}
+      end # module LogMethods
+      EOL    = "\n"
+      DIV0   = ("." * 79) << EOL 
+      DIV1   = ("-" * 79) << EOL 
+      DIV2   = ("=" * 79) << EOL 
+      DIV3   = ("#" * 79) << EOL 
+      SEC0   = ("." * 16) << EOL 
+      SEC1   = ("-" * 16) << EOL 
+      SEC2   = ("=" * 16) << EOL 
+      SEC3   = ("#" * 16) << EOL 
+      class << self
+#{{{
+        def append_features c
+#{{{
+          ret = super
+          c.extend LogMethods
+          class << c
+            def default_logger
+#{{{
+              if defined?(@default_logger) and @default_logger
+                @default_logger
+              else
+                self.default_logger = Logger.new STDOUT
+                @default_logger.debug{ "<#{ self }> using default logger"}
+                @default_logger
+              end
+#}}}
+            end
+            def default_logger= log
+#{{{
+              @default_logger = log
+              @default_logger.extend LoggerExt
+              @default_logger.logdev.dev.sync = true
+              @default_logger
+#}}}
+            end
+          end
+          ret
+#}}}
+        end
+#}}}
+      end
+      include LogMethods
+#}}}
+    end # module Logging
+#}}}
+  end # module rq
+$__rq_logging__ = __FILE__ 
+end

data/lib/rq-0.1.7/mainhelper.rb

@@ -0,0 +1,53 @@
+unless defined? $__rq_mainhelper__
+  module RQ 
+#{{{
+    LIBDIR = File::dirname(File::expand_path(__FILE__)) + File::SEPARATOR unless
+      defined? LIBDIR
+
+    require LIBDIR + 'util'
+    require LIBDIR + 'logging'
+
+    class  MainHelper
+#{{{
+      include Util
+      include Logging
+      attr :main
+      attr :logger
+      attr :argv
+      attr :env
+      attr :cmd
+      attr :options
+      attr :qpath
+      attr :mode
+      attr :q
+      def initialize main
+#{{{
+        @main = main
+        @logger = main.logger
+        @argv = main.argv
+        @env = main.env
+        @cmd = main.cmd
+        @options = main.options
+        @qpath = main.qpath
+        @mode = main.mode
+        @q = nil 
+#}}}
+      end
+      def set_q
+    #{{{
+        raise "q <#{ @qpath }> does not exist" unless test ?d, @qpath
+        @q = JobQueue::new @qpath, 'logger' => @logger
+        if @options['snapshot']
+          ss = "#{ $0 }_#{ Process::pid }_#{ Thread::current.id.abs }_#{ rand Time::now.to_i  }".gsub(%r|/|o,'_')
+          qtmp = File::join Dir::tmpdir, ss
+          @q = @q.snapshot qtmp, @options['retries']
+          at_exit{ FileUtils::rm_rf qtmp }
+        end
+    #}}}
+      end
+#}}}
+    end # class MainHelper
+#}}}
+  end # module RQ
+$__rq_mainhelper__ = __FILE__ 
+end