boy_band 0.1.6 → 0.1.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d7584b735b0671735458dcf190fac4d6469b4fc3e165e9df54f909bceaa89b83
-  data.tar.gz: 691d966ef8eabf44ec431cf4a182ca81b10202c17f96c0f88d5a702ff6492eb6
+  metadata.gz: 6931c59174a8d6eda05b764b143c11c38532c46ae0d68f41b35700539db6e45a
+  data.tar.gz: 7df882de888ee4ac4680d37006f437d5b26e9055f193f596486f6259bb75997e
 SHA512:
-  metadata.gz: 06723c500fc807e94c6f1937154033dd87d82f9c3ce53d4cc699a4b6071ddbb3f180100a2d9a0cb002dee06b87c9a8248326b8de84cf03953d54423e8e44197e
-  data.tar.gz: 61166b0bc26eae3181cbf10f195bfb461199cc0f40839f7717a8765abac549dc632826bf2160dc4fef5bff08fec1ffb8c528c999746da2e3b5cfcdb47105329e
+  metadata.gz: 2c19f3dd8e840f87b903d97450adbbfe0ce92b236cbbdec0905b7e6f18c283dcf2dccc2372f731cfbcc53bde6e530e79061b6da072b1cf49f61fe1ab461daeb4
+  data.tar.gz: 41cbb91e46bc9dcb8169c95c5b0b458626692679fa21668a09281d2f4a40568c7595917f8ac35268612c674470ea09bda8c557c85ba931a475bfd068dd22167a

data/lib/boy_band.rb

@@ -39,12 +39,22 @@ module BoyBand
     def set_job_chain(val)
       @@job_chain = val
     end
+
+    def queue_size(queue)
+      size = Resque.redis.get("sizeof/#{queue}").to_i
+      if !size || size == 0
+        size = Resque.size(queue)
+        Resque.redis.setex("sizeof/#{queue}", 30.seconds.to_i, size)
+      end
+      size
+    end
   
     def schedule_for(queue, klass, method_name, *args)
+      queue = queue.to_sym
       @queue = queue.to_s
       job_hash = Digest::MD5.hexdigest(args.to_json)
       note_job(job_hash)
-      size = Resque.size(queue)
+      size = queue_size(queue)
       args.push("domain::#{self.domain_id}")
       chain = self.job_chain.split(/##/)
       job_id = "j#{Time.now.iso8601}_#{rand(9999)}"
@@ -53,10 +63,19 @@ module BoyBand
         Resque.redis.incr("jobs_from_#{chain[0]}") 
         Resque.redis.expire("jobs_from_#{chain[0]}", 24.hours.to_i)
       end
-      chain.push("#{klass.to_s},#{method_name.to_s},#{args.join('+')}")
-      Rails.logger.warn("jobchain set, #{chain[0]} #{chain.join('##')}") if chain.length > 1
+      Resque.redis.setex("scheduled/#{klass.to_s}/#{job_hash}", 6.hours, "t")
+      chain_args = args[0..-2]
+      if chain_args.length == 1 && chain_args[0].is_a?(Hash)
+        chain_args = [chain_args[0]['method'],chain_args[0]['id'],chain_args[0]['arguments'].to_s[0, 20]]
+      end
+      chain.push("#{klass.to_s},#{method_name.to_s},#{chain_args.join('+')}")
+      Rails.logger.warn("jobchain set, #{chain[0]} #{chain.join('##')}") if chain.length > 2
       if chain.length > 5
-        Rails.logger.error("jobchain too long: job_id, #{chain.length} entries")
+        Rails.logger.error("jobchain too deep: #{chain[0]}, #{chain.length} entries")
+      end
+      job_count = Resque.redis.get("jobs_from_#{chain[0]}")
+      if job_count && job_count.to_i > 50
+        Rails.logger.error("jobchain too many sub-jobs: #{chain[0]}, #{job_count} so far")
       end
       args.push("chain::#{chain.join('##')}")
       if queue == :slow
@@ -75,21 +94,21 @@ module BoyBand
     end
   
     def note_job(hash)
-      if Resque.redis
-        timestamps = JSON.parse(Resque.redis.hget('hashed_jobs', hash) || "[]")
-        cutoff = 6.hours.ago.to_i
-        timestamps = timestamps.select{|ts| ts > cutoff }
-        timestamps.push(Time.now.to_i)
-  #      Resque.redis.hset('hashed_jobs', hash, timestamps.to_json)
-      end
+  #     if Resque.redis
+  #       timestamps = JSON.parse(Resque.redis.hget('hashed_jobs', hash) || "[]")
+  #       cutoff = 6.hours.ago.to_i
+  #       timestamps = timestamps.select{|ts| ts > cutoff }
+  #       timestamps.push(Time.now.to_i)
+  # #      Resque.redis.hset('hashed_jobs', hash, timestamps.to_json)
+  #     end
     end
   
     def clear_job(hash)
-      if Resque.redis
-        timestamps = JSON.parse(Resque.redis.hget('hashed_jobs', hash) || "[]")
-        timestamps.shift
-  #      Resque.redis.hset('hashed_jobs', hash, timestamps.to_json)
-      end
+  #     if Resque.redis
+  #       timestamps = JSON.parse(Resque.redis.hget('hashed_jobs', hash) || "[]")
+  #       timestamps.shift
+  # #      Resque.redis.hset('hashed_jobs', hash, timestamps.to_json)
+  #     end
     end
   
     def schedule(klass, method_name, *args)
@@ -120,6 +139,7 @@ module BoyBand
       klass = Object.const_get(klass_string)
       method_name = args_copy.shift
       job_hash = Digest::MD5.hexdigest(args_copy.to_json)
+      Resque.redis.del("scheduled/#{klass_string}/#{method_name}/#{job_hash}")
       hash = args_copy[0] if args_copy[0].is_a?(Hash)
       hash ||= {'method' => method_name}
       action = "#{klass_string} . #{hash['method']} (#{hash['id']})"
@@ -174,6 +194,68 @@ module BoyBand
       end
       res
     end
+
+    def root_actions(queue='default')
+      idx = Resque.size(queue)
+      job_ids = {}
+      idx.times do |i|
+        item = Resque.peek(queue, i)
+        chain = nil
+        if item && item['args'] && item['args'][-1].match(/^chain::/)
+          chain = item['args'].pop
+        end
+        if chain
+          parts = chain.split(/##/)
+          job_ids[parts[0]] ||= [parts[1], 0, []]
+          job_ids[parts[0]][1] += 1
+          job_ids[parts[0]][2] << parts.length - 2
+        end
+      end
+      job_ids.each{|k, v| job_ids.delete(k) if v[1] <= 5}.length
+      job_ids
+    end
+
+    def action_types(queue='default')
+      idx = Resque.size(queue)
+      list = []
+      idx.times do |i|
+        item = Resque.peek(queue, i)
+        chain = nil
+        if item && item['args'] && item['args'][-1].match(/^chain::/)
+          chain = item['args'].pop
+        end
+        if chain
+          match = chain.scan(/##/)
+          if match && match.length == 1
+            item['root'] = true
+            list << item
+          elsif match && match.length > 1
+            list << item
+          end
+        end
+      end
+      count = {'root' => {}, 'non_root' => {}}
+      list.each do |item|
+        key = "#{item['args'][0]}::#{item['args'][2].is_a?(Hash) ? item['args'][2]['method'] : item['args'][1]}"
+        count[item['root'] ? 'root' : 'non_root'][key] ||= 0
+        count[item['root'] ? 'root' : 'non_root'][key] += 1
+      end.length
+      count
+    end
+
+    def find_actions(method)
+      queue = 'default'
+      idx = Resque.size(queue)
+      list = []
+      idx.times do |i|
+        item = Resque.peek(queue, i)
+        if item['args'] && item['args'][2].is_a?(Hash) && item['args'][2]['method'] == method
+          list << item
+          puts item.to_json
+        end
+      end
+      list
+    end
   
     def scheduled_for?(queue, klass, method_name, *args)
       args_copy = [] + args
@@ -184,6 +266,10 @@ module BoyBand
         set_domain_id(args_copy.pop.split(/::/, 2)[1])
       end
 
+      idx = queue_size(queue)
+      job_hash = args_copy.to_json
+      return true if Resque.redis.get("scheduled/#{klass.to_s}/#{method_name}/#{job_hash}") == "t"
+      return false if idx > 500 # big queues mustn't be searched this way
       idx = Resque.size(queue)
       queue_class = (queue == :slow ? 'SlowWorker' : 'Worker')
       if false
@@ -308,6 +394,10 @@ module BoyBand
     
   module AsyncInstanceMethods
     def schedule(method, *args)
+      schedule_for('default', method, *args)
+    end
+
+    def schedule_for(queue, method, *args)
       return nil unless method
       id = self.id
       settings = {
@@ -316,19 +406,23 @@ module BoyBand
         'scheduled' => self.class.scheduled_stamp,
         'arguments' => args
       }
-      Worker.schedule(self.class, :perform_action, settings)
+      Worker.schedule_for(queue, self.class, :perform_action, settings)
     end
-  
+
     def schedule_once(method, *args)
+      schedule_once_for('default', method, *args)
+    end
+
+    def schedule_once_for(queue, method, *args)
       return nil unless method && id
-      already_scheduled = Worker.scheduled?(self.class, :perform_action, {
+      already_scheduled = Worker.scheduled_for?(queue, self.class, :perform_action, {
         'id' => id,
         'method' => method,
         'scheduled' => self.class.scheduled_stamp,
         'arguments' => args
       })
       if !already_scheduled
-        schedule(method, *args)
+        schedule_for(queue, method, *args)
       else
         false
       end
@@ -353,23 +447,37 @@ module BoyBand
         'scheduled' => self.scheduled_stamp,
         'arguments' => args
       }
-      Worker.schedule(self, :perform_action, settings)
+      schedule_for('default', method, *args)
     end
   
+    def schedule_for(queue, method, *args)
+      return nil unless method
+      settings = {
+        'method' => method,
+        'scheduled' => self.scheduled_stamp,
+        'arguments' => args
+      }
+      Worker.schedule_for(queue, self, :perform_action, settings)
+    end
+
     def schedule_once(method, *args)
+      schedule_once_for('default', method, *args)
+    end
+
+    def schedule_once_for(queue, method, *args)
       return nil unless method
-      already_scheduled = Worker.scheduled?(self, :perform_action, {
+      already_scheduled = Worker.scheduled_for?(queue, self, :perform_action, {
         'method' => method,
         'scheduled' => self.scheduled_stamp,
         'arguments' => args
       })
       if !already_scheduled
-        schedule(method, *args)
+        schedule_for(queue, method, *args)
       else
         false
       end
     end
-  
+
     def perform_action(settings)
       obj = self
       if settings['id']

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: boy_band
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.11
 platform: ruby
 authors:
 - Brian Whitmer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-13 00:00:00.000000000 Z
+date: 2020-05-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails