rq 0.1.7 → 3.0.0

data/lib/rq-3.0.0/orderedhash.rb

@@ -0,0 +1,240 @@
+# AUTHOR
+#    jan molic /mig/at/1984/dot/cz/
+#
+# DESCRIPTION
+#    Hash with preserved order and some array-like extensions
+#    Public domain. 
+#
+# THANKS
+#    Andrew Johnson for his suggestions and fixes of Hash[],
+#    merge, to_a, inspect and shift
+#
+# USAGE
+#    just require this file and use OrderedHash instead of Hash (examples are at the end)
+#    you can try to run this file (ruby orderedhash.rb)
+#    
+
+class OrderedHash < Hash
+
+    attr_accessor :order
+
+    class << self
+	def [] *args
+	    hsh = OrderedHash.new
+	    if Hash === args[0]
+		hsh.replace args[0]
+	    elsif (args.size % 2) != 0
+		raise ArgumentError, "odd number of elements for Hash"
+	    else
+		hsh[args.shift] = args.shift while args.size > 0
+	    end
+	    hsh
+	end
+    end
+
+    def initialize 
+	@order = []
+    end
+    
+    def store_only a,b
+	store a,b
+    end
+    
+    alias orig_store store    
+    def store a,b
+	@order.push a unless has_key? a
+	super a,b
+    end
+    alias []= store
+    
+    def == hsh2
+	return false if @order != hsh2.order
+	super hsh2
+    end
+    
+    def clear
+	@order = []
+	super
+    end
+    
+    def delete key
+	@order.delete key
+	super
+    end
+    
+    def each_key
+	@order.each { |k| yield k }
+	self
+    end
+    
+    def each_value
+	@order.each { |k| yield self[k] }
+	self
+    end
+    
+    def each
+	@order.each { |k| yield k,self[k] }
+	self
+    end
+    alias each_pair each    
+
+    def delete_if
+	@order.clone.each { |k| 
+	    delete k if yield
+	}
+	self
+    end
+    
+    def values
+	ary = []
+	@order.each { |k| ary.push self[k] }
+	ary
+    end
+    
+    def keys
+	@order
+    end
+    
+    def invert
+	hsh2 = Hash.new    
+	@order.each { |k| hsh2[self[k]] = k }
+	hsh2
+    end
+    
+    def reject &block
+	self.dup.delete_if &block
+    end
+    
+    def reject! &block
+	hsh2 = reject &block
+	self == hsh2 ? nil : hsh2
+    end
+
+    def replace hsh2
+	@order = hsh2.keys 
+	super hsh2
+    end
+    
+    def shift
+	key = @order.first
+	key ? [key,delete(key)] : super
+    end
+
+    def unshift k,v
+	unless self.include? k
+	    @order.unshift k
+	    orig_store(k,v)
+	    true
+	else
+	    false
+	end
+    end
+    
+    def push k,v
+	unless self.include? k
+	    @order.push k
+	    orig_store(k,v)
+	    true
+	else
+	    false
+	end
+    end
+    
+    def pop
+	key = @order.last
+	key ? [key,delete(key)] : nil
+    end
+    
+    def to_a
+	ary = []
+	each { |k,v| ary << [k,v] }
+	ary
+    end
+    
+    def to_s
+	self.to_a.to_s
+    end
+    
+    def inspect
+	ary = []
+	each {|k,v| ary << k.inspect + "=>" + v.inspect}
+	'{' + ary.join(", ") + '}'
+    end
+    
+    def update hsh2
+	hsh2.each { |k,v| self[k] = v }
+	self
+    end
+    alias :merge! update
+    
+    def merge hsh2
+	self.dup update(hsh2)
+    end
+    
+    def select
+	ary = []
+	each { |k,v| ary << [k,v] if yield k,v }
+	ary
+    end
+
+    attr_accessor "to_yaml_style"
+    def yaml_inline= bool
+      if respond_to?("to_yaml_style")
+        self.to_yaml_style = :inline
+      else
+        unless defined? @__yaml_inline_meth
+          @__yaml_inline_meth =
+            lambda {|opts|
+              YAML::quick_emit(object_id, opts) {|emitter|
+                emitter << '{ ' << map{|kv| kv.join ': '}.join(', ') << ' }'
+              }
+            }
+          class << self
+            def to_yaml opts = {}
+              begin
+                @__yaml_inline ? @__yaml_inline_meth[ opts ] : super
+              rescue
+                @to_yaml_style = :inline
+                super
+              end
+            end
+          end
+        end
+      end
+      @__yaml_inline = bool
+    end
+    def yaml_inline!() self.yaml_inline = true end
+    
+end
+
+if __FILE__ == $0
+
+    # You can do simply
+    hsh = OrderedHash.new
+    hsh['z'] = 1
+    hsh['a'] = 2
+    hsh['c'] = 3
+    p hsh.keys     # ['z','a','c']
+
+    # or using OrderedHash[] method
+    hsh = OrderedHash['z', 1, 'a', 2, 'c', 3] 
+    p hsh.keys     # ['z','a','c']
+
+    # but this don't preserve order
+    hsh = OrderedHash['z'=>1, 'a'=>2, 'c'=>3]
+    p hsh.keys     # ['a','c','z']
+
+    # OrderedHash has useful extensions: push, pop and unshift
+    p hsh.push('to_end', 15)  # true, key added
+    p hsh.push('to_end', 30)  # false, already - nothing happen
+    p hsh.unshift('to_begin', 50)  # true, key added
+    p hsh.unshift('to_begin', 60)  # false, already - nothing happen
+    p hsh.keys     # ["to_begin", "a", "c", "z", "to_end"]
+    p hsh.pop      # ["to_end", 15], if nothing remains, return nil
+    p hsh.keys     # ["to_begin", "a", "c", "z"]
+    p hsh.shift    # ["to_begin", 30], if nothing remains, return nil
+
+end
+
+
+# END

data/lib/{rq-0.1.7 → rq-3.0.0}/qdb.rb

@@ -1,39 +1,50 @@
 unless defined? $__rq_qdb__
   module RQ 
-#{{{
+#--{{{
     LIBDIR = File::dirname(File::expand_path(__FILE__)) + File::SEPARATOR unless
       defined? LIBDIR
 
+    require 'arrayfields'
+
     require LIBDIR + 'util'
     require LIBDIR + 'logging'
     require LIBDIR + 'sleepcycle'
     require LIBDIR + 'refresher'
 
+    #
+    # the QDB class is the low level access point to the actual sqlite database.
+    # the primary function if performs is to serialize access to the queue db
+    # via the locking protocol
+    #
     class  QDB
-#{{{
+#--{{{
       include Util
       include Logging
+
+      class RollbackTransactionError < StandardError; end
+      class AbortedTransactionError < StandardError; end
     
       FIELDS = 
-    #{{{
+#--{{{
       %w(
         jid priority state 
         submitted started finished elapsed
         submitter runner
+        stdin stdout stderr
         pid exit_status
-        tag command
+        tag restartable command
       )
-    #}}}
+#--}}}
     
       PRAGMAS =
-    #{{{
+#--{{{
         <<-sql
           PRAGMA default_synchronous = FULL;
         sql
-    #}}}
+#--}}}
     
       SCHEMA = 
-    #{{{
+#--{{{
         <<-sql
           create table jobs
           (
@@ -47,7 +58,7 @@ unless defined? $__rq_qdb__
             primary key (key)
           );
         sql
-    #}}}
+#--}}}
     
       DEFAULT_LOGGER                         = Logger::new(STDERR)
       DEFAULT_SQL_DEBUG                      = false
@@ -55,13 +66,12 @@ unless defined? $__rq_qdb__
       DEFAULT_AQUIRE_LOCK_SC                 = SleepCycle::new(2, 16, 2)
       DEFAULT_TRANSACTION_RETRIES_SC         = SleepCycle::new(8, 24, 8)
       DEFAULT_ATTEMPT_LOCKD_RECOVERY         = true
-      DEFAULT_LOCKD_RECOVER_WAIT             = 1800
-      DEFAULT_AQUIRE_LOCK_LOCKFILE_STALE_AGE = 1800
-      DEFAULT_AQUIRE_LOCK_REFRESH_RATE       = 8
+      DEFAULT_LOCKD_RECOVER_WAIT             = 3600  # 1 hr
+      DEFAULT_AQUIRE_LOCK_LOCKFILE_STALE_AGE = 21600 # 6 hrs
+      DEFAULT_AQUIRE_LOCK_REFRESH_RATE       = 30
     
       class << self
-    #{{{
-        attr :logger, true
+#--{{{
         attr :sql_debug, true
         attr :transaction_retries, true
         attr :aquire_lock_sc, true
@@ -72,18 +82,23 @@ unless defined? $__rq_qdb__
         attr :aquire_lock_refresh_rate, true
 
         def fields
-#{{{
+#--{{{
           FIELDS
-#}}}
+#--}}}
         end
         def integrity_check dbpath
-#{{{
+#--{{{
           ret = false 
           tuple = nil
           begin
-            db = SQLite::Database::new dbpath, 0
+            db = 
+              begin
+                SQLite::Database::new dbpath, 0
+              rescue
+                SQLite::Database::new dbpath
+              end
             opened = true
-            db.use_array = true
+            db.use_array = true rescue nil
             tuple = db.execute 'PRAGMA integrity_check;'
             ret = (tuple and tuple.first and (tuple.first["integrity_check"] =~ /^\s*ok\s*$/io))
           ensure
@@ -91,42 +106,42 @@ unless defined? $__rq_qdb__
             db = nil
           end
           ret
-#}}}
+#--}}}
         end
         def t2h tuple
-    #{{{
+#--{{{
           h = {}
           FIELDS.each_with_index{|f,i| h[f] = tuple[i]}
           h
-    #}}}
+#--}}}
         end
         def h2t h
-    #{{{
+#--{{{
           t = tuple
           FIELDS.each{|f| t[f] = h[f]}
           t
-    #}}}
+#--}}}
         end
         def tuple
-    #{{{
+#--{{{
           t = Array::new FIELDS.size
           t.fields = FIELDS
           t
-    #}}}
+#--}}}
         end
         def q tuple
-    #{{{
-          tuple.map do |f| 
-            if f
+#--{{{
+          [ tuple ].flatten.map do |f| 
+            if f and not f.to_s.empty?
               "'" << Util.escape(f,"'","'") << "'"
             else
               'NULL'
             end
           end
-    #}}}
+#--}}}
         end
         def create path, opts = {}
-    #{{{
+#--{{{
           qdb = new path, opts
           FileUtils::touch qdb.lockfile
           create_schema qdb.schema
@@ -135,19 +150,19 @@ unless defined? $__rq_qdb__
             qdb.execute SCHEMA
           end
           qdb
-    #}}}
+#--}}}
         end
         def create_schema path
-#{{{
+#--{{{
           tmp = "#{ path }.tmp"
           open(tmp,'w') do |f| 
             f.puts PRAGMAS 
             f.puts SCHEMA
           end
           FileUtils::mv tmp, path
-#}}}
+#--}}}
         end
-    #}}}
+#--}}}
       end
     
       attr :path
@@ -168,7 +183,7 @@ unless defined? $__rq_qdb__
 
 
       def initialize path, opts = {}
-    #{{{
+#--{{{
         @path = path
         @opts = opts
 
@@ -230,21 +245,22 @@ unless defined? $__rq_qdb__
         @lockf = Lockfile::new("#{ @path }.lock") 
         @fields = FIELDS
         @in_transaction = false
+        @in_ro_transaction = false
         @db = nil
 
         @lockd_recover = "#{ @dirname }.lockd_recover"
         @lockd_recover_lockf = Lockfile::new "#{ @lockd_recover }.lock"
         @lockd_recovered = false
-    #}}}
+#--}}}
       end
       def ro_transaction(opts = {}, &block)
-    #{{{
+#--{{{
         opts['read_only'] = true
         transaction(opts, &block)
-    #}}}
+#--}}}
       end
       def transaction opts = {} 
-#{{{
+#--{{{
         raise 'nested transaction' if @in_transaction
         ro = Util::getopt 'read_only', opts 
         ret = nil
@@ -267,10 +283,49 @@ unless defined? $__rq_qdb__
           @in_transaction = false
         end
         ret
-#}}}
+#--}}}
+      end
+if false
+      def ro_transaction(opts = {}, &block)
+#--{{{
+        opts['read_only'] = true
+        transaction(opts, &block)
+#--}}}
+      end
+      def transaction opts = {} 
+#--{{{
+        ro = Util::getopt 'read_only', opts 
+        ret = nil
+        if @in_transaction
+      STDERR.puts 'continuing transaction...'
+          ret = yield
+        else
+          begin 
+      STDERR.puts 'starting transaction...'
+            @in_transaction = true
+            lockd_recover_wrap(opts) do
+              transaction_wrap(opts) do
+                aquire_lock(opts) do
+                  #sillyclean(opts) do
+                    connect do
+                      execute 'begin' unless ro
+                      ret = yield 
+                      execute 'commit' unless ro
+                    end
+                  #end
+                end
+              end
+            end
+          ensure
+            @in_transaction = false
+          end
+        end
+        ret
+#--}}}
       end
+end
       def lockd_recover_wrap opts = {}
-#{{{
+#--{{{
         ret = nil
         try_again = false
         begin
@@ -308,7 +363,7 @@ unless defined? $__rq_qdb__
           end
         end
         ret
-#}}}
+#--}}}
       end
 #
 # TODO - perhaps should not retry on SQLException??  yet errors seem to map to
@@ -316,7 +371,7 @@ unless defined? $__rq_qdb__
 # simply retry.
 #
       def transaction_wrap opts = {} 
-#{{{
+#--{{{
         ro = Util::getopt 'read_only', opts
         ret = nil
         if ro
@@ -328,25 +383,42 @@ unless defined? $__rq_qdb__
             ret = yield 
           rescue => e
           #rescue SQLite::DatabaseException, SQLite::SQLException, SystemCallError => e
-            if @transaction_retries == 0
-              raise
-            elsif errors.size >= @transaction_retries
-              error{ "MAXIMUM TRANSACTION RETRIES SURPASSED" }
-              raise
-            else
-              warn{ e } if(errors.empty? or not Util::erreq(errors[-1], e))
-              errors << e
-              warn{ "retry <#{ errors.size }>..." }
-            end
-            sleep @transaction_retries_sc.next
-            retry
+            case e
+              when AbortedTransactionError 
+                raise
+              when RollbackTransactionError 
+                raise
+              else
+                if @transaction_retries == 0
+                  raise
+                elsif errors.size >= @transaction_retries
+                  error{ "MAXIMUM TRANSACTION RETRIES SURPASSED" }
+                  raise
+                else
+                  warn{ e } if(errors.empty? or not Util::erreq(errors[-1], e))
+                  errors << e
+                  warn{ "retry <#{ errors.size }>..." }
+                end
+                sleep @transaction_retries_sc.next
+                retry
+              end
           end
         end
         ret
-#}}}
+#--}}}
+      end
+      def abort_transaction(*a)
+#--{{{
+        raise AbortedTransactionError, *a
+#--}}}
+      end
+      def rollback_transaction(*a)
+#--{{{
+        raise RollbackTransactionError, *a
+#--}}}
       end
       def sillyclean opts = {} 
-#{{{
+#--{{{
         ro = Util::getopt 'read_only', opts
         ret = nil
         if ro
@@ -360,10 +432,10 @@ unless defined? $__rq_qdb__
           silly.each{|path| FileUtils::rm_rf path}
         end
         ret
-#}}}
+#--}}}
       end
       def aquire_lock opts = {} 
-#{{{
+#--{{{
         ro = Util::getopt 'read_only', opts
         ret = nil
 
@@ -415,8 +487,16 @@ unless defined? $__rq_qdb__
                   stat = File::stat @lockfile
                   mtime = stat.mtime
                   stale = mtime < (Time::now - @aquire_lock_lockfile_stale_age)
-                  warn{ "detected stale lockfile of mtime <#{ mtime }>" }
-                  lockd_recover if stale and @attempt_lockd_recovery
+                  if stale
+                    Util::uncache @lockfile rescue nil
+                    stat = File::stat @lockfile
+                    mtime = stat.mtime
+                    stale = mtime < (Time::now - @aquire_lock_lockfile_stale_age)
+                    if stale
+                      warn{ "detected stale lockfile of mtime <#{ mtime }>" }
+                      lockd_recover if @attempt_lockd_recovery
+                    end
+                  end
                   sc = @aquire_lock_sc.next
                   debug{ "failed to aquire lock - sleep(#{ sc })" }
                   sleep sc 
@@ -445,19 +525,24 @@ unless defined? $__rq_qdb__
           end
         end
         ret
-#}}}
+#--}}}
       end
       def connect
-    #{{{
+#--{{{
         ret = nil
         opened = nil
         begin
           raise 'db has no schema' unless test ?e, @schema
           debug{"connecting to db <#{ @path }>..."}
-          $db = @db = SQLite::Database::new(@path, 0)
+          $db = @db = 
+            begin
+              SQLite::Database::new(@path, 0)
+            rescue
+              SQLite::Database::new(@path)
+            end
           debug{"connected."}
           opened = true
-          @db.use_array = true
+          @db.use_array = true rescue nil
           ret = yield @db
         ensure
           @db.close if opened
@@ -465,27 +550,27 @@ unless defined? $__rq_qdb__
           debug{"disconnected from db <#{ @path }>"}
         end
         ret
-    #}}}
+#--}}}
       end
       def execute sql, &block
-    #{{{
+#--{{{
         raise 'not in transaction' unless @in_transaction
         if @sql_debug
           logger << "SQL:\n#{ sql }\n"
         end
         #ret = retry_if_locked{ @db.execute sql, &block }
         ret = @db.execute sql, &block
-        if @sql_debug and ret and ret.first
+        if @sql_debug and ret and Array === ret and ret.first
           logger << "RESULT:\n#{ ret.first.inspect }\n...\n"
         end
         ret
-    #}}}
+#--}}}
       end
 #
 # TODO - add sleep cycle if this ends up getting used
 #
       def retry_if_locked
-    #{{{
+#--{{{
         ret = nil
         begin
           ret = yield 
@@ -495,10 +580,10 @@ unless defined? $__rq_qdb__
           retry
         end
         ret
-    #}}}
+#--}}}
       end
       def vacuum
-    #{{{
+#--{{{
         raise 'nested transaction' if @in_transaction
         begin 
           @in_transaction = true
@@ -507,10 +592,24 @@ unless defined? $__rq_qdb__
           @in_transaction = false
         end
         self
-    #}}}
+#--}}}
+      end
+      def recover!
+#--{{{
+        raise 'nested transaction' if @in_transaction
+        begin 
+          @in_transaction = true
+          connect{ execute 'vacuum' }
+          require 'timeout'
+          Timeout::timeout(60){ system "sqlite #{ @path } .tables >/dev/null 2>&1" }
+        ensure
+          @in_transaction = false
+        end
+        integrity_check
+#--}}}
       end
       def lockd_recover
-#{{{
+#--{{{
         return nil unless @attempt_lockd_recovery
         warn{ "attempting lockd recovery" }
         time = Time::now
@@ -598,16 +697,16 @@ unless defined? $__rq_qdb__
           end
         end
         ret
-#}}}
+#--}}}
       end
       def integrity_check path = @path
-#{{{
+#--{{{
         debug{ "running integrity_check on <#{ path }>" }
         klass.integrity_check(path)  
-#}}}
+#--}}}
       end
       def lock opts = {} 
-#{{{
+#--{{{
         ret = nil
         lockd_recover_wrap do
           aquire_lock(opts) do
@@ -615,20 +714,20 @@ unless defined? $__rq_qdb__
           end
         end
         ret
-#}}}
+#--}}}
       end
       alias write_lock lock
       alias wlock write_lock
       def read_lock(opts = {}, &block)
-#{{{
+#--{{{
         opts['read_only'] = true
         lock opts, &block
-#}}}
+#--}}}
       end
       alias rlock read_lock
-#}}}
+#--}}}
     end # class QDB
-#}}}
+#--}}}
   end # module RQ
 $__rq_qdb__ = __FILE__ 
 end