rq-ruby1.8 3.4.3

data/lib/rq/querier.rb

@@ -0,0 +1,98 @@
+unless defined? $__rq_queryier__
+  module RQ 
+#--{{{
+    LIBDIR = File::dirname(File::expand_path(__FILE__)) + File::SEPARATOR unless
+      defined? LIBDIR
+
+    require LIBDIR + 'mainhelper'
+
+    #
+    # a Querier simply takes an sql where clause (such as 'jid = 42') from the
+    # command line, queries the queue, and dumps a valid yaml representation of
+    # the tuples returned.  the output of a Querier can be used as the input to
+    # a Deleter or Submitter, etc.
+    #
+    #
+    class Querier < MainHelper
+#--{{{
+      def query
+#--{{{
+        set_q
+
+        @q.qdb.transaction_retries = 1
+
+        where_clause = @argv.join ' '
+
+        if where_clause.empty? and stdin? 
+          debug{ "reading where_clause from stdin" }
+          while((buf = stdin.gets))
+            buf.strip!
+            buf.gsub! %r/#.*$/o, ''
+            next if buf.empty?
+            where_clause << "#{ buf } "
+          end
+        end
+
+        @q.query(where_clause, &dumping_yaml_tuples)
+#--}}}
+      end
+      def query
+#--{{{
+        set_q
+
+        @q.qdb.transaction_retries = 1
+
+        simple_pat = %r/^\s*([^=\s!~]+)(=~|!~|!=|==|=)([^=\s]+)\s*$/ox
+        simple_query = @argv.select{|arg| arg !~ simple_pat }.empty?
+
+        where_clause = 
+          if simple_query 
+            wc = [] 
+
+            @argv.each do |arg|
+              m = simple_pat.match(arg).to_a[1..-1]
+              field, op, value = m[0], m[1], m[2..-1].join 
+              op =
+                case op
+                  when '=', '==' 
+                    '='
+                  when '!='
+                    '!='
+                  when '=~'
+                    'like'
+                  when '!~'
+                    'not like'
+                end
+
+              quoted = (value =~ %r/^\s*'.*'\s*$/o)
+              numeric = begin; Float(value); true; rescue; false; end 
+
+              value = "'#{ value }'" unless quoted or numeric
+
+              wc << "(#{ field } #{ op } #{ value })"
+            end
+
+            wc.join ' and '
+          else
+            @argv.join ' '
+          end
+
+        if where_clause.strip.empty? and stdin?
+          debug{ "reading where_clause from stdin" }
+          while((buf = stdin.gets))
+            buf.strip!
+            buf.gsub! %r/#.*$/o, ''
+            next if buf.empty?
+            where_clause << "#{ buf } "
+          end
+        end
+
+        @q.query(where_clause, &dumping_yaml_tuples)
+#--}}}
+      end
+#--}}}
+    end # class Queryier
+#--}}}
+  end # module RQ
+$__rq_queryier__ = __FILE__ 
+end

data/lib/rq/rails.rb

@@ -0,0 +1,80 @@
+if defined?(Rails)
+  require 'fileutils'
+
+  module Rails
+    class << self
+      def qpath *arg
+        @qpath = arg.shift unless arg.empty?
+        @qpath ||= File.join(RAILS_ROOT, 'q')
+        @qpath
+      end
+
+      def q
+        defined?(@q) ? @q : qinit
+      end
+
+      def qinit
+        unless test ?e, qpath
+          cmd = "rq #{ qpath } create"
+          system cmd or abort "cmd <#{ cmd }> failed with <#{ $?.inspect }>" 
+          cmd = "cp `which rqmailer` #{ qpath }/bin"
+          system cmd or abort "cmd <#{ cmd }> failed with <#{ $?.inspect }>" 
+        end
+        (( logger = Logger.new STDERR )).level = Logger::FATAL
+        @q = RQ::JobQueue.new qpath, 'logger' => logger
+        @q.extend qextension 
+        @q
+      end
+
+      def qextension
+        Module.new do
+          def rqmailer config, template = nil, submission = {}
+            config = YAML.load config if String === config
+
+            ### clean up keys
+            config = config.inject({}){|h,kv| k, v = kv; h.update k.to_s => v}
+            submission = submission.inject({}){|h,kv| k, v = kv; h.update k.to_s => v}
+
+            command = (config["command"] || config["cmd"]) or raise "no command in <#{ config.inspect }>!"
+
+            mconfig = config["mail"] || config
+            attachements = [ mconfig["attach"] ].flatten.compact
+
+            tag = "rqmailer"
+            command = "rqmailer ### #{ command }"
+
+            submission["tag"] ||= tag
+            submission["command"] = command 
+
+            tmpdir = File.join RAILS_ROOT, 'tmp', Process.pid.to_s, '.rqmailer.d'
+            FileUtils.mkdir_p tmpdir
+            begin
+              open(File.join(tmpdir, 'config'), 'w') do |fd|
+                fd.write config.to_yaml
+              end
+              if template
+                open(File.join(tmpdir, 'template'), 'w') do |fd|
+                  fd.write template 
+                end
+              end
+              d = File.join(tmpdir, "attachements")
+              FileUtils.mkdir_p d 
+              attachements.each do |attachment|
+                FileUtils.cp attachment, d
+              end
+
+              submission["data"] = tmpdir 
+              job = nil
+              submit(submission){|job|}
+              job
+            ensure
+              FileUtils.rm_rf tmpdir
+            end
+          end
+          alias_method "mailrun", "rqmailer"
+        end
+      end
+    end
+  end
+
+end

data/lib/rq/recoverer.rb

@@ -0,0 +1,28 @@
+unless defined? $__rq_recoverer__
+  module RQ 
+#--{{{
+    LIBDIR = File::dirname(File::expand_path(__FILE__)) + File::SEPARATOR unless
+      defined? LIBDIR
+
+    require LIBDIR + 'mainhelper'
+
+    class  Recoverer < MainHelper
+#--{{{
+      def recover
+#--{{{
+        set_q
+
+        bool = @q.recover! ? true : false
+        puts "---"
+        puts "recovered : #{ bool }" 
+
+        EXIT_SUCCESS
+#--}}}
+      end
+      alias recover! recover
+#--}}}
+    end # class Recoverer 
+#--}}}
+  end # module RQ
+$__rq_recoverer__ = __FILE__ 
+end

data/lib/rq/refresher.rb

@@ -0,0 +1,80 @@
+unless defined? $__rq_refresher__
+  module RQ 
+#--{{{
+    LIBDIR = File::dirname(File::expand_path(__FILE__)) + File::SEPARATOR unless
+      defined? LIBDIR
+
+    #
+    # the job of the Refresher is to maintain a _lease_ on a file that has been
+    # locked.  the method is simply to touch the file at a certain interval
+    # thereby keeping it 'fresh'.  a separate process, vs. a thread, is used for
+    # this task to eliminate any chance that the ruby interpreter might put all
+    # threads to sleep for some blocking tasks, like fcntl based locks which are
+    # used heavily in RQ, resulting in a a prematurely stale lockfile 
+    #
+    class Refresher
+#--{{{
+      SIGNALS = %w(SIGTERM SIGINT SIGKILL)
+      attr :path
+      attr :pid
+      attr :refresh_rate
+      def initialize path, refresh_rate = 8
+#--{{{
+        @path = path
+        File::stat path
+        @refresh_rate = Float refresh_rate
+        @pipe = IO::pipe
+        if((@pid = Util::fork))
+          @pipe.last.close
+          @pipe = @pipe.first
+          @thread = Thread::new{loop{@pipe.gets}}
+          Process::detach @pid
+        else
+          begin
+            pid = Process::pid
+            ppid = Process::ppid
+            $0 = "#{ path }.refresher.#{ pid }"
+            SIGNALS.each{|sig| trap(sig){ raise }}
+            @pipe.first.close
+            @pipe = @pipe.last
+            loop do
+              FileUtils::touch @path
+              sleep @refresh_rate
+              Process::kill 0, ppid
+              @pipe.puts pid
+            end
+          rescue Exception => e
+            exit!
+          end
+        end
+#--}}}
+      end
+      def kill
+#--{{{
+        begin
+          @thread.kill rescue nil
+          @pipe.close rescue nil
+          SIGNALS.each{|sig| Process::kill sig, @pid rescue nil}
+        ensure
+=begin
+          n = 42
+          dead = false
+          begin
+            n.times do |i|
+              Process::kill 0, @pid
+              sleep 1
+            end
+          rescue Errno::ESRCH
+            dead = true
+          end
+          raise "runaway refresher <#{ @pid }> must be killed!" unless dead
+=end
+        end
+#--}}}
+      end
+#--}}}
+    end # class Refresher
+#--}}}
+  end # module RQ
+$__rq_refresher__ = __FILE__ 
+end

data/lib/rq/relayer.rb

@@ -0,0 +1,283 @@
+unless defined? $__rq_relayer__
+  module RQ 
+#--{{{
+    LIBDIR = File::dirname(File::expand_path(__FILE__)) + File::SEPARATOR unless
+      defined? LIBDIR
+
+    require LIBDIR + 'mainhelper'
+    require LIBDIR + 'job'
+    require LIBDIR + 'jobrunner'
+    require LIBDIR + 'jobrunnerdaemon'
+    require LIBDIR + 'jobqueue'
+
+    class  Relayer < MainHelper
+#--{{{
+      DEFAULT_MIN_SLEEP = 42
+      DEFAULT_MAX_SLEEP = 240
+      DEFAULT_RELAY     = 16 
+
+      class << self
+#--{{{
+        attr :min_sleep, true
+        attr :max_sleep, true
+        attr :relay, true
+#--}}}
+      end
+
+      def relay 
+#--{{{
+        daemon do
+          gen_pidfile
+          @main.init_logging
+          @logger = @main.logger
+
+          set_q
+
+        #
+        # munge @q/@qpath to set there
+        #
+          @here = @q
+          @qpath = realpath @main.argv.shift
+          set_q
+          @there = @q
+          @q = @here
+          @hdb = @here.qdb
+          @tdb = @there.qdb
+
+          @pid = Process::pid
+          @cmd = @main.cmd 
+          @started = Util::timestamp
+          @min_sleep = Integer(@options['min_sleep'] || defval('min_sleep'))
+          @max_sleep = Integer(@options['max_sleep'] || defval('max_sleep'))
+          @relay = Integer(@options['number'] || defval('relay'))
+          
+          @transactions = {}
+
+
+          install_signal_handlers
+
+          info{ "** STARTED **" }
+          info{ "version <#{ RQ::VERSION }>" }
+          info{ "cmd <#{ @cmd }>" }
+          info{ "pid <#{ @pid }>" }
+          info{ "pidfile <#{ @pidfile.path }>" }
+          info{ "here <#{ @here.path }>" }
+          info{ "there <#{ @there.path }>" }
+
+          debug{ "mode <#{ @mode }>" }
+          debug{ "min_sleep <#{ @min_sleep }>" }
+          debug{ "max_sleep <#{ @max_sleep }>" }
+          debug{ "relay <#{ @relay }>" }
+
+exit
+
+          loop do
+            handle_signal if $rq_signaled
+            throttle(@min_sleep) do
+              reap_and_sow
+              relax
+            end
+          end
+=begin
+          loop do
+            handle_signal if $rq_signaled
+            throttle(@min_sleep) do
+              start_jobs unless busy?
+              if nothing_running?
+                relax
+              else
+                reap_jobs
+              end
+            end
+          end
+=end
+        end
+#--}}}
+      end
+
+      def daemon
+#--{{{
+        if @options['daemon']
+          fork do
+            Process::setsid
+            fork do
+              Dir::chdir(Util.realpath('~'))
+              File::umask 0
+              open('/dev/null','r+') do |f|
+                STDIN.reopen f 
+                STDOUT.reopen f 
+                STDERR.reopen f 
+              end
+              @daemon = true
+              yield
+              exit EXIT_SUCCESS
+            end
+            exit!
+          end
+          exit!
+        else
+          @daemon = false 
+          yield
+          exit EXIT_SUCCESS
+        end
+#--}}}
+      end
+      def gen_pidfile name = nil
+#--{{{
+        name ||= gen_relayer_name(@options['name'] || @qpath)
+        @pidfile = 
+          begin
+            open name, File::CREAT | File::EXCL | File::RDWR
+          rescue
+            open name, File::RDWR
+          end
+        unless @pidfile and @pidfile.posixlock(File::LOCK_EX | File::LOCK_NB)
+          pid = IO::read(name) rescue nil
+          pid ||= 'unknown'
+          if @options['quiet']
+            exit EXIT_FAILURE
+          else
+            raise "process <#{ pid }> is already relaying from this queue"
+          end
+        else
+          @pidfile.rewind
+          @pidfile.sync = true
+          @pidfile.print Process::pid
+          @pidfile.truncate @pidfile.pos
+          at_exit{ FileUtils::rm_f name rescue nil }
+        end
+#--}}}
+      end
+      def gen_relayer_name path 
+#--{{{
+        path = Util::realpath(path).gsub(%r|/|o, '_')
+        File::join(Util::realpath('~'), ".#{ path }.relayer")
+#--}}}
+      end
+      def install_signal_handlers
+#--{{{
+        if @daemon
+          $rq_signaled = false
+          $rq_sighup = false
+          $rq_sigterm = false
+          $rq_sigint = false
+          trap('SIGHUP') do
+            $rq_signaled = $rq_sighup = 'SIGHUP' 
+            warn{ "signal <SIGHUP>" }
+            warn{ "finishing running jobs before handling signal" }
+          end
+          trap('SIGTERM') do
+            $rq_signaled = $rq_sigterm = 'SIGTERM' 
+            warn{ "signal <SIGTERM>" }
+            warn{ "finishing running jobs before handling signal" }
+          end
+          trap('SIGINT') do
+            $rq_signaled = $rq_sigint = 'SIGINT' 
+            warn{ "signal <SIGINT>" }
+            warn{ "finishing running jobs before handling signal" }
+          end
+          @jrd.install_signal_handlers
+        end
+#--}}}
+      end
+      def handle_signal
+#--{{{
+        if $rq_sigterm or $rq_sigint
+          reap_jobs(reap_only = true) until nothing_running? 
+          info{ "** STOPPING **" }
+          @jrd.shutdown rescue nil
+          @pidfile.posixlock File::LOCK_UN
+          exit EXIT_SUCCESS
+        end
+
+        if $rq_sighup
+          reap_jobs(reap_only = true) until nothing_running? 
+          info{ "** RESTARTING **" }
+          @jrd.shutdown rescue nil
+          Util::uncache __FILE__ 
+          @pidfile.posixlock File::LOCK_UN
+          Util::exec @cmd
+        end
+#--}}}
+      end
+      def throttle rate = @min_sleep 
+#--{{{
+        if Numeric === rate and rate > 0 
+          if defined? @last_throttle_time and @last_throttle_time
+            elapsed = Time.now - @last_throttle_time
+            timeout = rate - elapsed
+            if timeout > 0
+              timeout = timeout + rand(rate * 0.10)
+              debug{ "throttle rate of <#{ rate }> exceeded - sleeping <#{ timeout }>" }
+              sleep timeout
+            end
+          end
+          @last_throttle_time = Time.now 
+        end
+        yield
+#--}}}
+      end
+      def transaction
+#--{{{
+        ret = nil
+        if @in_transaction
+          ret = yield
+        else
+          begin
+            @in_transaction = true
+            @here.transaction do
+              @there.transaction do
+                ret = yield
+              end
+            end
+          ensure
+            @in_transaction = false 
+          end
+        end
+        ret
+#--}}}
+      end
+      def relax
+#--{{{
+        seconds = rand(@max_sleep - @min_sleep + 1) + @min_sleep
+        debug{ "relaxing <#{ seconds }>" }
+        sleep seconds
+#--}}}
+      end
+
+#
+# TODO - this will need to map jids here to jids there
+#
+      def reap_and_sow
+#--{{{
+        transaction{ reap and sow }
+#--}}}
+      end
+      def reap
+#--{{{
+        debug{ "reaping finished/dead jobs" }
+
+        sql = <<-sql
+          select jid from jobs where or state='running'
+        sql
+        tuples = hdb.execute sql
+        hjids = tuples.map{|t| t['jid']}
+
+        unless jids.empty
+          where_clauses = hjids.map{|hjid| "jid=#{ hjid }" }
+          where_clause = where_clauses.join ' or '
+          sql = <<-sql
+            select jid from jobs where state='finished' or state='dead' and (#{ where_clause })
+          sql
+        end
+
+        debug{ "reaped finished/dead jobs" }
+        self
+#--}}}
+      end
+#--}}}
+    end # class Relayer 
+#--}}}
+  end # module RQ
+$__rq_relayer__ = __FILE__ 
+end