rq 0.1.7

data/lib/rq-0.1.7/feeder.rb

@@ -0,0 +1,367 @@
+unless defined? $__rq_feeder__
+  module RQ 
+#{{{
+    LIBDIR = File::dirname(File::expand_path(__FILE__)) + File::SEPARATOR unless
+      defined? LIBDIR
+
+    require LIBDIR + 'mainhelper'
+    require LIBDIR + 'job'
+    require LIBDIR + 'jobrunner'
+    require LIBDIR + 'jobrunnerdaemon'
+    require LIBDIR + 'jobqueue'
+
+    class  Feeder < MainHelper
+#{{{
+      DEFAULT_MIN_SLEEP = 42
+      DEFAULT_MAX_SLEEP = 240
+      DEFAULT_FEED      = 2
+
+      class << self
+#{{{
+        attr :min_sleep, true
+        attr :max_sleep, true
+        attr :feed, true
+#}}}
+      end
+
+      def feed
+    #{{{
+        daemon do
+          gen_pidfile
+          @main.init_logging
+          @logger = @main.logger
+          set_q
+
+          @pid = Process::pid
+          @cmd = @main.cmd 
+          @started = Util::timestamp
+          @min_sleep = Integer(@options['min_sleep'] || defval('min_sleep'))
+          @max_sleep = Integer(@options['max_sleep'] || defval('max_sleep'))
+          @max_feed = Integer(@options['feed'] || defval('feed'))
+          @children = Hash::new 
+          @jrd = JobRunnerDaemon::daemon
+
+          install_signal_handlers
+
+          info{ "** STARTED **" }
+          info{ "version <#{ RQ::VERSION }>" }
+          info{ "cmd <#{ @cmd }>" }
+          info{ "pid <#{ @pid }>" }
+          info{ "pidfile <#{ @pidfile.path }>" }
+          info{ "jobrunnerdaemon uri <#{ @jrd.uri }> pid <#{ @jrd.pid }>" }
+          info{ "qpath <#{ @qpath }>" }
+          info{ "mode <#{ @mode }>" }
+          debug{ "min_sleep <#{ @min_sleep }>" }
+          debug{ "max_sleep <#{ @max_sleep }>" }
+
+          fill_morgue
+
+          loop do
+            handle_signal if $rq_signaled
+            throttle(@min_sleep) do
+              start_jobs unless busy?
+              if nothing_running?
+                relax
+              else
+                reap_jobs
+              end
+            end
+          end
+        end
+    #}}}
+      end
+      def daemon
+    #{{{
+        if @options['daemon']
+          fork do
+            Process::setsid
+            fork do
+              Dir::chdir(Util.realpath('~'))
+              File::umask 0
+              open('/dev/null','r+') do |f|
+                STDIN.reopen f 
+                STDOUT.reopen f 
+                STDERR.reopen f 
+              end
+              @daemon = true
+              yield
+              exit EXIT_SUCCESS
+            end
+            exit!
+          end
+          exit!
+        else
+          @daemon = false 
+          yield
+          exit EXIT_SUCCESS
+        end
+    #}}}
+      end
+      def gen_pidfile name = nil
+    #{{{
+        name ||= gen_feeder_name(@options['name'] || @qpath)
+        @pidfile = 
+          begin
+            open name, File::CREAT | File::EXCL | File::RDWR
+          rescue
+            open name, File::RDWR
+          end
+        unless @pidfile and @pidfile.posixlock(File::LOCK_EX | File::LOCK_NB)
+          pid = IO::read(name) rescue nil
+          pid ||= 'unknown'
+          if @options['quiet']
+            exit EXIT_FAILURE
+          else
+            raise "process <#{ pid }> is already feeding from this queue"
+          end
+        else
+          @pidfile.rewind
+          @pidfile.sync = true
+          @pidfile.print Process::pid
+          @pidfile.truncate @pidfile.pos
+          at_exit{ FileUtils::rm_f name rescue nil }
+        end
+    #}}}
+      end
+      def gen_feeder_name path 
+    #{{{
+        path = Util::realpath(path).gsub(%r|/|o, '_')
+        File::join(Util::realpath('~'), ".#{ path }.feeder")
+    #}}}
+      end
+      def install_signal_handlers
+#{{{
+        if @daemon
+          $rq_signaled = false
+          $rq_sighup = false
+          $rq_sigterm = false
+          $rq_sigint = false
+          trap('SIGHUP') do
+            $rq_signaled = $rq_sighup = 'SIGHUP' 
+            warn{ "signal <SIGHUP>" }
+            warn{ "finishing running jobs before handling signal" }
+          end
+          trap('SIGTERM') do
+            $rq_signaled = $rq_sigterm = 'SIGTERM' 
+            warn{ "signal <SIGTERM>" }
+            warn{ "finishing running jobs before handling signal" }
+          end
+          trap('SIGINT') do
+            $rq_signaled = $rq_sigint = 'SIGINT' 
+            warn{ "signal <SIGINT>" }
+            warn{ "finishing running jobs before handling signal" }
+          end
+          @jrd.install_signal_handlers
+        end
+#}}}
+      end
+      def fill_morgue
+    #{{{
+        debug{ "filling morgue" }
+        sql = <<-sql
+          select * from jobs where state = 'running' and runner = '#{ Util::hostname }' and started <= '#{ @started }'
+        sql
+        db = @q.qdb
+        transaction do
+          tuples = db.execute sql
+          tuples.each do |tuple|
+            jid = tuple['jid']
+            if jid
+              sql = "update jobs set state='dead' where jid='#{ jid }'"
+              db.execute sql
+              info{ "burried job <#{ jid }>" }
+            end
+          end
+        end
+        debug{ "filled morgue" }
+    #}}}
+      end
+      def handle_signal
+#{{{
+        if $rq_sigterm or $rq_sigint
+          reap_jobs(reap_only = true) until nothing_running? 
+          info{ "** STOPPING **" }
+          @jrd.shutdown rescue nil
+          @pidfile.posixlock File::LOCK_UN
+          exit EXIT_SUCCESS
+        end
+
+        if $rq_sighup
+          reap_jobs(reap_only = true) until nothing_running? 
+          info{ "** RESTARTING **" }
+          @jrd.shutdown rescue nil
+          Util::uncache __FILE__ 
+          @pidfile.posixlock File::LOCK_UN
+          Util::exec @cmd
+        end
+#}}}
+      end
+      def throttle rate = @min_sleep 
+    #{{{
+        if Numeric === rate and rate > 0 
+          if defined? @last_throttle_time and @last_throttle_time
+            elapsed = Time.now - @last_throttle_time
+            timeout = rate - elapsed
+            if timeout > 0
+              timeout = timeout + rand(rate * 0.10)
+              debug{ "throttle rate of <#{ rate }> exceeded - sleeping <#{ timeout }>" }
+              sleep timeout
+            end
+          end
+          @last_throttle_time = Time.now 
+        end
+        yield
+    #}}}
+      end
+      def start_jobs
+    #{{{
+        n_started = 0 
+        transaction do
+          until busy?
+            break unless((job = @q.getjob))
+            start_job job
+            n_started += 1 
+          end
+        end
+        debug{ "<#{ n_started }> jobs started" }
+        n_started
+    #}}}
+      end
+      def start_job job
+    #{{{
+        jid, command = job['jid'], job['command']
+
+      #
+      # we setup state slightly prematurely so jobrunner will have it availible
+      #
+        job['state'] = 'running'
+        job['started'] = Util::timestamp Time::now
+        job['runner'] = Util::hostname 
+
+        jr = @jrd.runner job
+        cid = jr.cid
+    
+        if jr and cid
+          jr.run
+          job['pid'] = cid
+          @children[cid] = job
+          @q.jobisrunning job
+          info{ "started - jid <#{ job['jid'] }> pid <#{ job['pid'] }> command <#{ job['command'] }>" }
+        else
+          error{ "not started - jid <#{ job['jid'] }> command <#{ job['command'] }>" }
+        end
+    
+        cid
+    #}}}
+      end
+      def nothing_running?
+    #{{{
+        @children.size == 0
+    #}}}
+      end
+      def reap_jobs reap_only = false, blocking = true
+    #{{{
+        reaped = []
+
+        cid = status = nil
+    
+        if blocking
+          if busy? or reap_only
+            cid, status = @jrd.waitpid2 -1, Process::WUNTRACED 
+          else
+            loop do
+              debug{ "not busy wait loop" }
+              cid, status = @jrd.waitpid2 -1, Process::WNOHANG | Process::WUNTRACED 
+              break if cid
+              start_jobs unless $rq_signaled
+              break if busy?
+              cid, status = @jrd.waitpid2 -1, Process::WNOHANG | Process::WUNTRACED 
+              break if cid
+              sleep 4.2 
+            end
+            cid, status = @jrd.waitpid2 -1, Process::WUNTRACED unless cid
+          end
+        else
+          cid, status = @jrd.waitpid2 -1, Process::WNOHANG | Process::WUNTRACED 
+        end
+    
+        if cid and status
+          job = @children[cid]
+          finish_job job, status
+    
+          transaction do
+            loopno = 0
+            loop do
+              @q.jobisdone job
+              @children.delete cid
+              reaped << cid
+    
+              start_jobs unless reap_only or $rq_signaled
+    
+              if @children.size == 0 or loopno > 42
+                break
+              else
+                sleep 0.1
+                cid, status = @jrd.waitpid2 -1, Process::WNOHANG | Process::WUNTRACED 
+                break unless cid and status
+                job = @children[cid]
+                finish_job job, status
+              end
+              loopno += 1
+            end
+          end
+        end
+    
+        debug{ "<#{ reaped.size }> jobs reaped" }
+    
+        reaped
+    #}}}
+      end
+      def finish_job job, status
+    #{{{
+        job['finished'] = Util::timestamp(Time::now)
+        job['elapsed'] = Util::stamptime(job['finished']) - Util::stamptime(job['started'])
+        t = status.exitstatus rescue nil 
+        job['exit_status'] = t 
+        job['state'] = 'finished' 
+        if t and t == 0
+          info{ "finished - jid <#{ job['jid'] }> pid <#{ job['pid'] }> exit_status <#{ job['exit_status'] }>" }
+        else
+          warn{ "finished - jid <#{ job['jid'] }> pid <#{ job['pid'] }> exit_status <#{ job['exit_status'] }>" }
+        end
+    #}}}
+      end
+      def transaction
+    #{{{
+        ret = nil
+        if @in_transaction
+          ret = yield
+        else
+          begin
+            @in_transaction = true
+            @q.transaction{ ret = yield }
+          ensure
+            @in_transaction = false 
+          end
+        end
+        ret
+    #}}}
+      end
+      def busy?
+    #{{{
+        @children.size >= @max_feed
+    #}}}
+      end
+      def relax
+    #{{{
+        seconds = rand(@max_sleep - @min_sleep + 1) + @min_sleep
+        debug{ "relaxing <#{ seconds }>" }
+        sleep seconds
+    #}}}
+      end
+#}}}
+    end # class Feeder
+#}}}
+  end # module RQ
+$__rq_feeder__ = __FILE__ 
+end

data/lib/rq-0.1.7/job.rb

@@ -0,0 +1,51 @@
+unless defined? $__rq_job__
+  module RQ 
+#{{{
+    LIBDIR = File::dirname(File::expand_path(__FILE__)) + File::SEPARATOR unless
+      defined? LIBDIR
+
+    require LIBDIR + 'util'
+    require LIBDIR + 'qdb'
+
+    class  Job
+#{{{
+      module Methods
+#{{{
+        def method_missing(meth, *args, &block)
+#{{{
+          setpat = %r/=$/o
+          meth = "#{ meth }"
+          setter = meth =~ setpat 
+          meth.gsub! setpat, ''
+          if fields.include? "#{ meth }"
+            if setter
+              self.send('[]=', meth, *args, &block)
+            else
+              self.send('[]', meth, *args, &block)
+            end
+          else
+            super
+          end
+#}}}
+        end
+#}}}
+      end
+      class << self
+#{{{
+        def new(*a, &b)
+#{{{
+          t = QDB::tuple
+          kvs = Util::hashify(*a)
+          kvs.each{|k,v| t[k] = v}
+          t.extend Methods
+          t
+#}}}
+        end
+#}}}
+      end
+#}}}
+    end # class Job
+#}}}
+  end # module RQ
+$__rq_job__ = __FILE__ 
+end

data/lib/rq-0.1.7/jobqueue.rb

@@ -0,0 +1,432 @@
+unless defined? $__rq_jobqueue__
+  module RQ 
+#{{{
+    LIBDIR = File::dirname(File::expand_path(__FILE__)) + File::SEPARATOR unless
+      defined? LIBDIR
+
+    require LIBDIR + 'util'
+    require LIBDIR + 'logging'
+    require LIBDIR + 'qdb'
+
+    class  JobQueue
+#{{{
+      include Logging
+      include Util
+      class Error < StandardError; end
+    
+      MAX_JID = 2 ** 20 
+    
+      class << self
+    #{{{
+        def create path, opts = {}
+    #{{{
+          FileUtils::rm_rf path
+          FileUtils::mkdir_p path
+          db = File::join path, 'db'
+          qdb = QDB.create db, opts
+          opts['qdb'] = qdb
+          q = new path, opts
+          q
+    #}}}
+        end
+    #}}}
+      end
+      attr :path
+      attr :opts
+      attr :qdb
+      alias :db :qdb
+      def initialize path, opts = {}
+    #{{{
+        @path = path
+        @opts = opts
+        raise "q <#{ @path }> does not exist" unless test ?e, @path
+        raise "q <#{ @path }> is not a directory" unless test ?d, @path
+        @basename = File::basename(@path)
+        @dirname = File::dirname(@path)
+        @logger = getopt('logger', opts) || Logger::new(STDERR)
+        @qdb = getopt('qdb', opts) || QDB::new(File::join(@path, 'db'), 'logger' => @logger)
+    #}}}
+      end
+      def submit(*jobs)
+    #{{{
+        now = Util::timestamp Time.now
+        tuples = nil
+        sql = ''
+    
+        jobs.each do |job|
+          raise "no command for job <#{ job.inspect }>" unless job['command']
+
+          tuple = QDB::tuple
+
+          tuple['command'] = job['command']
+          tuple['priority'] = job['priority'] || 0 
+          tuple['tag'] = job['tag']
+          tuple['state'] = 'pending'
+          tuple['submitted'] = now
+          tuple['submitter'] = Util::hostname 
+
+          values = QDB::q tuple
+
+          sql << "insert into jobs values (#{ values.join ',' });\n"
+        end
+    
+        tuples = nil
+        transaction do
+          execute(sql){}
+          if block_given?
+            sql = "select * from jobs where submitted = '#{ now }'"
+            tuples = execute sql
+          end
+        end
+        tuples.each{|t| yield t} if tuples
+        tuples = nil
+    
+        self
+    #}}}
+      end
+      def list(*whats)
+    #{{{
+        whats.replace(['pending', 'running', 'finished', 'dead']) if 
+          whats.empty? or whats.include?('all')
+    
+        whats.map! do |what|
+          case what
+            when %r/^\s*p/io
+              'pending'
+            when %r/^\s*r/io
+              'running'
+            when %r/^\s*f/io
+              'finished'
+            when %r/^\s*d/io
+              'dead'
+            else
+              what
+          end
+        end
+
+        where_clauses = [] 
+
+        whats.each do |what|
+          if what =~ %r/^\s*\d+\s*$/o
+            where_clauses << "jid=#{ QDB::q what }\n"
+          else
+            where_clauses << "state=#{ QDB::q what }\n"
+          end
+        end
+
+        where_clause = where_clauses.join(" or \n")
+
+        sql = <<-sql
+          select * from jobs
+          where #{ where_clause } 
+        sql
+    
+        tuples = ro_transaction{ execute sql }
+        puts '---'
+        fields = @qdb.fields
+        tuples.each do |tuple|
+          puts '-'
+          fields.each{|f| puts " #{ f }: #{ tuple[f] }" }
+        end
+        tuples = nil
+    
+        self
+    #}}}
+      end
+      def status(*whats)
+    #{{{
+        whats.replace(['pending', 'running', 'finished', 'dead']) if 
+          whats.empty? or whats.include?('all')
+    
+        whats.map! do |what|
+          case what
+            when %r/^\s*p/io
+              'pending'
+            when %r/^\s*r/io
+              'running'
+            when %r/^\s*f/io
+              'finished'
+            when %r/^\s*d/io
+              'dead'
+            else
+              what
+          end
+        end
+    
+        puts '---'
+        ro_transaction do
+          whats.each do |what|
+            sql = <<-sql
+              select count(*) from jobs where state = '#{ what }';
+            sql
+            tuples = execute sql 
+            tuple = tuples.first
+            count = (tuple ? tuple.first : 0)
+            puts "#{ what } : #{ count }"
+          end
+        end
+    
+        self
+    #}}}
+      end
+      def query where_clause = nil
+    #{{{
+        sql = 
+          if where_clause
+
+          #
+          # turn =~ into like clauses 
+          #
+            #where_clause.gsub!(/(=~\s*([^\s')(=]+))/om){q = $2.gsub(%r/'+|\s+/o,''); "like '%#{ q }%'"}
+          #
+          # quote everything on the rhs of an '=' sign - helps with shell problems...
+          #
+            #where_clause.gsub!(/(==?\s*([^\s')(=]+))/om){q = $2.gsub(%r/'+|\s+/o,''); "='#{ q }'"}
+
+            "select * from jobs where #{ where_clause };"
+          else
+            "select * from jobs;"
+          end
+
+        tuples = ro_transaction{ execute sql }
+
+        puts '---'
+        fields = @qdb.fields
+        tuples.each do |tuple|
+          puts '-'
+          fields.each{|f| puts " #{ f }: #{ tuple[f] }"}
+        end
+        tuples = nil
+    
+        self
+    #}}}
+      end
+      def delete(*jids)
+    #{{{
+        what = jids.first || 'all'
+    
+        case what
+          when String
+            sql =
+              case what
+                when %r/^\s*p/io
+                  "delete from jobs where state = 'pending';"
+                when %r/^\s*r/io
+                  "delete from jobs where state = 'running';"
+                when %r/^\s*f/io
+                  "delete from jobs where state = 'finished';"
+                when %r/^\s*d/io
+                  "delete from jobs where state = 'dead';"
+                when %r/^\s*a/io
+                  "delete from jobs;"
+                else
+                  raise ArgumentError, "cannot delete <#{ what }>"
+              end
+          else
+            sql = ''
+            jids.each do |jid|
+              jid = Integer jid
+              sql << "delete from jobs where jid = #{ jid };"
+            end
+        end
+        transaction{ execute sql }
+        sql = nil
+        @qdb.vacuum
+        self
+    #}}}
+      end
+      def update(kvs,*jids)
+    #{{{
+        allowed = %w(priority command tag)
+        kvs.keys.each do |key|
+          raise "update of <#{ key }> not allowed" unless
+            allowed.include? key
+        end
+
+        raise "no jids" if jids.empty?
+        jids.sort!
+        jids.uniq!
+
+        pending = jids.delete 'pending'
+    
+        update_clause = kvs.map{|k,v| "#{ k }='#{ v }'"}.join(",\n")
+
+        if pending
+          update_sql = 
+            "update jobs\n" <<
+            "set\n#{ update_clause }\n" <<
+            "where\nstate='pending'"
+          select_sql = "select * from jobs where state='pending'"
+        else
+          where_clause = jids.map{|jid| "jid=#{ jid }"}.join(" or\n")
+          update_sql = 
+            "update jobs\n" <<
+            "set\n#{ update_clause }\n" <<
+            "where\nstate='pending' and\n#{ where_clause }"
+          select_sql = "select * from jobs where #{ where_clause }"
+        end
+
+        tuples = nil 
+        transaction do 
+          execute update_sql
+          tuples = execute select_sql 
+        end
+        tuples or []
+    #}}}
+      end
+      def transaction(*args, &block)
+    #{{{
+        @qdb.transaction(*args, &block)
+    #}}}
+      end
+      def ro_transaction &block
+    #{{{
+        @qdb.ro_transaction &block
+    #}}}
+      end
+      def execute(*args, &block)
+    #{{{
+        @qdb.execute(*args, &block)
+    #}}}
+      end
+      def integrity_check(*args, &block)
+    #{{{
+        @qdb.integrity_check(*args, &block)
+    #}}}
+      end
+      def snapshot qtmp = "#{ @basename }.snapshot", retries = nil 
+    #{{{
+        qtmp ||= "#{ @basename }.snapshot"
+        debug{ "snapshot <#{ @path }> -> <#{ qtmp }>" }
+        retries = Integer(retries || 16)
+        debug{ "retries <#{ retries }>" }
+
+        qss = nil
+        loopno = 0
+
+        take_snapshot = lambda do
+          FileUtils::rm_rf qtmp
+          FileUtils::mkdir_p qtmp
+          %w(db db.schema lock).each do |base|
+            src, dest = File::join(@path, base), File::join(qtmp, base)
+            debug{ "cp <#{ src }> -> <#{ dest }>" }
+            FileUtils::cp(src, dest)
+          end
+          ss = klass::new qtmp, @opts
+          if ss.integrity_check
+            ss
+          else
+            nil
+          end
+        end
+
+        loop do
+          break if loopno >= retries
+          if((ss = take_snapshot.call))
+            debug{ "snapshot <#{ qtmp }> created" }
+            qss = ss
+            break
+          else
+            debug{ "failure <#{ loopno + 1}> of <#{ retries }> attempts to create snapshot <#{ qtmp }> - retrying..." }
+          end
+          loopno += 1
+        end
+
+        unless qss
+          debug{ "locking <#{ @path }> as last resort" }
+          @qdb.write_lock do
+            if((ss = take_snapshot.call))
+              debug{ "snapshot <#{ qtmp }> created" }
+              qss = ss
+            else
+              raise "failed <#{ loopno }> times to create snapshot <#{ qtmp }>"
+            end
+          end
+        end
+
+        qss
+    #}}}
+      end
+      def lock(*args, &block)
+    #{{{
+        @qdb.lock(*args, &block)
+    #}}}
+      end
+      def getjob
+    #{{{
+        sql = <<-sql
+          select * from jobs 
+            where state = 'pending' 
+            order by priority desc, submitted asc, jid asc
+            limit 1;
+        sql
+        tuples = execute sql
+        job = tuples.first
+        job
+    #}}}
+      end
+      def jobisrunning job 
+    #{{{
+        sql = <<-sql
+          update jobs 
+            set
+              pid = '#{ job['pid'] }',
+              state = '#{ job['state'] }',
+              started = '#{ job['started'] }',
+              runner = '#{ job['runner'] }'
+            where jid = #{ job['jid'] };
+        sql
+        execute sql
+    #}}}
+      end
+      def jobisdone job
+    #{{{
+        sql = <<-sql
+          update jobs 
+            set
+              state = '#{ job['state'] }',
+              exit_status = '#{ job['exit_status'] }',
+              finished = '#{ job['finished'] }',
+              elapsed = '#{ job['elapsed'] }'
+            where jid = #{ job['jid'] };
+        sql
+        execute sql
+    #}}}
+      end
+      def mtime
+    #{{{
+        File::stat(@path).mtime
+    #}}}
+      end
+      def []= key, value
+    #{{{
+        sql = "select count(*) from attributes where key='#{ key }';"
+        tuples = @qdb.execute sql
+        tuple = tuples.first
+        count = Integer tuple['count(*)']
+        case count
+          when 0
+            sql = "insert into attributes values('#{ key }','#{ value }');"
+            @qdb.execute sql
+          when 1
+            sql = "update attributes set key='#{ key }', value='#{ value }' where key='#{ key }';"
+            @qdb.execute sql
+          else
+            raise "key <#{ key }> has become corrupt!"
+        end
+    #}}}
+      end
+      def attributes 
+    #{{{
+        h = {}
+        tuples = @qdb.execute "select * from attributes;"
+        tuples.map!{|t| h[t['key']] = t['value']}
+        h
+    #}}}
+      end
+#}}}
+    end # class JobQueue
+#}}}
+  end # module RQ
+$__rq_jobqueue__ = __FILE__ 
+end