rescheduler 0.5.3 → 0.5.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 85c352b4ca6259305ae8fa32ba0fc16c0c00aef0
-  data.tar.gz: 7d54913d094dced68ae4210a791471e022495aad
+  metadata.gz: acc7eb77ebb2e60efc00c38a654b6eb3ac1fe6e3
+  data.tar.gz: cf12bd7f36e0e8135a2b1d88cd1de3f58deee044
 SHA512:
-  metadata.gz: 6625da10818cf1aefdc48e3561c17c93d059939225bfdb784b0267771fc6b23db4d0ca89bff2b19fa112fa3bf2a5ad309624323dcf14dd09adb13cd75cc4ff3a
-  data.tar.gz: a6838adb3e3e6f1618df27dbce9f6a9d361539ecebd173e1a458e0dcf2008e6a3ebc42728835e9d63e0b8791a09804de4f2929579bdb5af57bbd957232d755c2
+  metadata.gz: 99ea938055d68a8dd63ec75c459c2195b9ba6c3a2751a23dc5c3a523e482eb6632c0edc85daec05b557ef1314c893b2a2c83857c8f7aea0d30ee6789b0f6fa2d
+  data.tar.gz: ccb0f16410fe88f4aac4f091fb0f52a49c9e80371790dfc01d830dbf1eaa69869e0839d5bb58a486b0bcaff3ad729fb1be1eaa8418a245c8d0081afede248cf7

data/lib/rescheduler.rb

@@ -98,6 +98,34 @@ module Rescheduler
     return {:jobs=>stats, :workers=>Worker.stats}
   end
 
+  def eta(sample_time = 30)
+    print "Measuring ... "
+    jobs1 = stats[:jobs]
+    start = Time.zone.now
+    sleep sample_time
+    stop = Time.zone.now
+    jobs2 = stats[:jobs]
+    print "done!\n"
+
+    secs = stop - start
+    jobs = {}
+    jobs1.select {|k, v| (v[:immediate] || 0) > 1}.keys.each do |job|
+      j1_i = jobs1[job][:immediate]
+      j2_i = (jobs2[job] || {})[:immediate] || 0
+      diff = j1_i - j2_i
+      if diff == 0
+        jobs[job] = "No progress"
+      elsif diff < 0
+        jobs[job] = "Negative progress. Queue size increased."
+      else
+        secs_per_job = secs/diff.to_f
+        remaining = jobs2[job][:immediate] * secs_per_job
+        jobs[job] = {per_job: '%.2f' % secs_per_job, remaining: '%.2f' % remaining}
+      end
+    end
+    jobs
+  end
+
   #----------------------------------------------
   # Queue management
   #----------------------------------------------
@@ -276,6 +304,9 @@ module Rescheduler
       raise Exception, 'Can not start worker without defining job handlers.'
     end
 
+    start_opts = {}
+    start_opts = tubes.pop if tubes.last.is_a?(Hash)
+
     tubes.each do |t|
       next if @@runners.include?(t)
       raise Exception, "Handler for queue #{t} is undefined."
@@ -288,12 +319,8 @@ module Rescheduler
     # Generate a random clientkey for maintenance token ring maintenance.
     client_key = [[rand(0xFFFFFFFF)].pack('L')].pack('m0')[0...6]
 
-    keys = tubes.map {|t| rk_queue(t)}
-    keys << rk_maintenace
-
-    # Queue to control a named worker
     worker_queue = Worker.rk_queue if Worker.named?
-    keys.unshift(worker_queue) if worker_queue # worker control queue is the first we respond to
+    queues = construct_pri_queues(tubes.map {|t| rk_queue(t)})
 
     dopush = nil
 
@@ -318,7 +345,7 @@ module Rescheduler
       # This may not be desirable...
       # (too bad BRPOPLPUSH does not support multiple queues...)
       # TODO: Maybe LUA script is the way out of this.
-      result = redis.brpop(keys, :timeout=>timeout)
+      result = redis.brpop(queues, :timeout=>timeout)
 
       # Handle task
       if result # Got a task
@@ -332,6 +359,8 @@ module Rescheduler
           Worker.handle_command(qnid)
         else
           run_job(qnid)
+          # Regenerate prioritized queue list based on current handled tasks, if we do round_robin
+          queues = construct_pri_queues(tubes.map {|t| rk_queue(t)}, tube) if start_opts[:round_robin]
         end
       else
         # Do nothing when got timeout, the run_maintenance will take care of deferred jobs
@@ -341,6 +370,21 @@ module Rescheduler
 
   def end_job_loop; @@end_job_loop = true; end
 
+  # Perform a "fair" round robin rotation of queues by rotating the "current" queue to the end of the list
+  private def construct_pri_queues(queues, current = nil)
+    idx = queues.index(current) if current
+    # Rotate if needed
+    queues = queues[(idx+1)..-1] + queues[0..idx] if idx && idx < queues.size - 1
+
+    # Prepend Queue to control a named worker
+    worker_queue = Worker.rk_queue if Worker.named?
+    queues.unshift(worker_queue) if worker_queue # worker control queue is the first we respond to
+
+    # Append rk_maintenance
+    queues << rk_maintenace
+    return queues
+  end
+
 
   # Logging facility
   def log_debug(msg)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rescheduler
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.5.4
 platform: ruby
 authors:
 - Dongyi Liao
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-25 00:00:00.000000000 Z
+date: 2016-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis