sidekiq-batch 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1fe037d539fb1492eb690004dfd48a03727beadf
-  data.tar.gz: 3bad40562aa7abd37fc677ae4bf65c6393053d4b
+  metadata.gz: d41af0ded27d9233c6731d110bcc24d3c31a7e0f
+  data.tar.gz: 2a6a63c2b2bd2159cc0456970d013819344c4670
 SHA512:
-  metadata.gz: d12d34ce2b8d9ee4d823f1c32406403322a402ab5fc4104c46b6b1cfffca94d12732ff50c09c238666fd333787b3e546eac412529a8c840860bc182ad57014a8
-  data.tar.gz: bdbc01d26ef53401a420fc78baf0e715ff3d609a950b0987c318d17a5ac9ce67c6e97fb9005fff79a31e2f1e33eb16b9c26ffbb9fb09e6f645a9e3c4977aa8a6
+  metadata.gz: c297e8094b0de083870db5850dee5eacccc59cb30aa85961fd71bff9076119092b7721cf1502b9d2556372337a5d4bdb3a1bc1c2d1630d8949d06b1bc6b4a03c
+  data.tar.gz: 3c9d0365850379328118ed6a8f25b59a1e96c2fde7f4a7ae0ef8695024d589ba33de38d4344d321f480d2b41e74115fe8bae7d0c815efa8b55e2171f9bc5164b

data/.gitignore

@@ -9,3 +9,4 @@
 /tmp/
 dump.rdb
 sidekiq-batch-*.gem
+.DS_Store

data/Gemfile

@@ -5,4 +5,5 @@ gemspec
 group :test do
   gem "simplecov"
   gem "codeclimate-test-reporter", "~> 1.0.0"
+  gem 'pry-byebug'
 end

data/README.md

@@ -4,6 +4,8 @@
 
 # Sidekiq::Batch
 
+[![Join the chat at https://gitter.im/breamware/sidekiq-batch](https://badges.gitter.im/breamware/sidekiq-batch.svg)](https://gitter.im/breamware/sidekiq-batch?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+
 [![Gem Version](https://badge.fury.io/rb/sidekiq-batch.svg)][gem]
 [![Build Status](https://travis-ci.org/breamware/sidekiq-batch.svg?branch=master)][travis]
 [![Code Climate](https://codeclimate.com/github/breamware/sidekiq-batch/badges/gpa.svg)][codeclimate]

data/lib/sidekiq/batch.rb

@@ -12,16 +12,15 @@ module Sidekiq
 
     BID_EXPIRE_TTL = 108_000
 
-    attr_reader :bid, :description, :callback_queue
+    attr_reader :bid, :description, :callback_queue, :created_at
 
     def initialize(existing_bid = nil)
       @bid = existing_bid || SecureRandom.urlsafe_base64(10)
-      Sidekiq.redis do |r|
-        r.multi do
-          r.hset("BID-#{bid}", 'created_at', Time.now)
-          r.expire("BID-#{bid}", BID_EXPIRE_TTL)
-        end
-      end
+      @existing = !(!existing_bid || existing_bid.empty?)  # Basically existing_bid.present?
+      @initialized = false
+      @created_at = Time.now.utc.to_f
+      @bidkey = "BID-" + @bid.to_s
+      @ready_to_queue = []
     end
 
     def description=(description)
@@ -36,11 +35,14 @@ module Sidekiq
 
     def on(event, callback, options = {})
       return unless %w(success complete).include?(event.to_s)
+      callback_key = "#{@bidkey}-callbacks-#{event}"
       Sidekiq.redis do |r|
         r.multi do
-          r.hset("BID-#{bid}", "callback_#{event}", callback)
-          r.hset("BID-#{bid}", "callback_#{event}_opts", options.to_json)
-          r.expire("BID-#{bid}", BID_EXPIRE_TTL)
+          r.sadd(callback_key, JSON.unparse({
+            callback: callback,
+            opts: options
+          }))
+          r.expire(callback_key, BID_EXPIRE_TTL)
         end
       end
     end
@@ -48,11 +50,82 @@ module Sidekiq
     def jobs
       raise NoBlockGivenError unless block_given?
 
-      Sidekiq.redis { |r| r.hincrby("BID-#{bid}", 'to_process', 1) }
-      Thread.current[:bid] = bid
-      yield
-      Thread.current[:bid] = nil
-      Sidekiq.redis { |r| r.hincrby("BID-#{bid}", 'to_process', -1) }
+      bid_data, Thread.current[:bid_data] = Thread.current[:bid_data], []
+
+      begin
+        if !@existing && !@initialized
+          parent_bid = Thread.current[:bid].bid if Thread.current[:bid]
+
+          Sidekiq.redis do |r|
+            r.multi do
+              r.hset(@bidkey, "created_at", @created_at)
+              r.hset(@bidkey, "parent_bid", parent_bid.to_s) if parent_bid
+              r.expire(@bidkey, BID_EXPIRE_TTL)
+            end
+          end
+
+          @initialized = true
+        end
+
+        @ready_to_queue = []
+
+        begin
+          parent = Thread.current[:bid]
+          Thread.current[:bid] = self
+          yield
+        ensure
+          Thread.current[:bid] = parent
+        end
+
+        return [] if @ready_to_queue.size == 0
+
+        Sidekiq.redis do |r|
+          r.multi do
+            if parent_bid
+              r.hincrby("BID-#{parent_bid}", "children", 1)
+              r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
+            end
+
+            r.hincrby(@bidkey, "pending", @ready_to_queue.size)
+            r.hincrby(@bidkey, "total", @ready_to_queue.size)
+            r.expire(@bidkey, BID_EXPIRE_TTL)
+
+            r.sadd(@bidkey + "-jids", @ready_to_queue)
+            r.expire(@bidkey + "-jids", BID_EXPIRE_TTL)
+          end
+        end
+
+        @ready_to_queue
+      ensure
+        Thread.current[:bid_data] = bid_data
+      end
+    end
+
+    def increment_job_queue(jid)
+      @ready_to_queue << jid
+    end
+
+    def invalidate_all
+      Sidekiq.redis do |r|
+        r.setex("invalidated-bid-#{bid}", BID_EXPIRE_TTL, 1)
+      end
+    end
+
+    def parent_bid
+      Sidekiq.redis do |r|
+        r.hget(@bidkey, "parent_bid")
+      end
+    end
+
+    def parent
+      if parent_bid
+        Sidekiq::Batch.new(parent_bid)
+      end
+    end
+
+    def valid?(batch = self)
+      valid = !Sidekiq.redis { |r| r.exists("invalidated-bid-#{batch.bid}") }
+      batch.parent ? valid && valid?(batch.parent) : valid
     end
 
     private
@@ -60,57 +133,88 @@ module Sidekiq
     def persist_bid_attr(attribute, value)
       Sidekiq.redis do |r|
         r.multi do
-          r.hset("BID-#{bid}", attribute, value)
-          r.expire("BID-#{bid}", BID_EXPIRE_TTL)
+          r.hset(@bidkey, attribute, value)
+          r.expire(@bidkey, BID_EXPIRE_TTL)
         end
       end
     end
 
     class << self
       def process_failed_job(bid, jid)
-        to_process = Sidekiq.redis do |r|
+        _, pending, failed, children, complete = Sidekiq.redis do |r|
           r.multi do
             r.sadd("BID-#{bid}-failed", jid)
+
+            r.hincrby("BID-#{bid}", "pending", 0)
             r.scard("BID-#{bid}-failed")
-            r.hget("BID-#{bid}", 'to_process')
+            r.hincrby("BID-#{bid}", "children", 0)
+            r.scard("BID-#{bid}-complete")
+
             r.expire("BID-#{bid}-failed", BID_EXPIRE_TTL)
           end
         end
-        if to_process[2].to_i == to_process[1].to_i
-          Callback.call_if_needed(:complete, bid)
-        end
+
+        enqueue_callbacks(:complete, bid) if pending.to_i == failed.to_i && children == complete
       end
 
-      def process_successful_job(bid)
-        out = Sidekiq.redis do |r|
+      def process_successful_job(bid, jid)
+        failed, pending, children, complete, success, total, parent_bid = Sidekiq.redis do |r|
           r.multi do
-            r.hincrby("BID-#{bid}", 'to_process', -1)
             r.scard("BID-#{bid}-failed")
-            r.hincrby("BID-#{bid}", 'pending', -1)
+            r.hincrby("BID-#{bid}", "pending", -1)
+            r.hincrby("BID-#{bid}", "children", 0)
+            r.scard("BID-#{bid}-complete")
+            r.scard("BID-#{bid}-success")
+            r.hget("BID-#{bid}", "total")
+            r.hget("BID-#{bid}", "parent_bid")
+
+            r.srem("BID-#{bid}-failed", jid)
+            r.srem("BID-#{bid}-jids", jid)
             r.expire("BID-#{bid}", BID_EXPIRE_TTL)
           end
         end
 
         puts "processed process_successful_job"
-        Callback.call_if_needed(:complete, bid) if out[1].to_i == out[0].to_i
-        Callback.call_if_needed(:success, bid) if out[0].to_i.zero?
+
+        enqueue_callbacks(:complete, bid) if pending.to_i == failed.to_i && children == complete
+        enqueue_callbacks(:success, bid) if pending.to_i.zero? && children == success
       end
 
-      def cleanup_redis(bid)
-        Sidekiq.redis do |r|
-          r.del("BID-#{bid}",
-                "BID-#{bid}-failed")
+      def enqueue_callbacks(event, bid)
+        batch_key = "BID-#{bid}"
+        callback_key = "#{batch_key}-callbacks-#{event}"
+        needed, _, callbacks, queue, parent_bid = Sidekiq.redis do |r|
+          r.multi do
+            r.hget(batch_key, event)
+            r.hset(batch_key, event, true)
+            r.smembers(callback_key)
+            r.hget(batch_key, "callback_queue")
+            r.hget(batch_key, "parent_bid")
+          end
+        end
+        return if needed == 'true'
+
+        begin
+          parent_bid = !parent_bid || parent_bid.empty? ? nil : parent_bid    # Basically parent_bid.blank?
+          Sidekiq::Client.push_bulk(
+            'class' => Sidekiq::Batch::Callback::Worker,
+            'args' => callbacks.reduce([]) do |memo, jcb|
+              cb = Sidekiq.load_json(jcb)
+              memo << [cb['callback'], event, cb['opts'], bid, parent_bid]
+            end,
+            'queue' => queue ||= 'default'
+          ) unless callbacks.empty?
+        ensure
+          cleanup_redis(bid) if event == :success
         end
       end
 
-      def increment_job_queue(bid)
+      def cleanup_redis(bid)
         Sidekiq.redis do |r|
-          r.multi do
-            %w(to_process pending total).each do |c|
-              r.hincrby("BID-#{bid}", c, 1)
-            end
-            r.expire("BID-#{bid}", BID_EXPIRE_TTL)
-          end
+          r.del("BID-#{bid}",
+                "BID-#{bid}-callbacks-complete",
+                "BID-#{bid}-callbacks-success",
+                "BID-#{bid}-failed")
         end
       end
     end

data/lib/sidekiq/batch/callback.rb

@@ -4,38 +4,64 @@ module Sidekiq
       class Worker
         include Sidekiq::Worker
 
-        def perform(clazz, event, opts, bid)
+        def perform(clazz, event, opts, bid, parent_bid)
           return unless %w(success complete).include?(event)
           clazz, method = clazz.split("#") if (clazz.class == String && clazz.include?("#"))
           method = "on_#{event}" if method.nil?
-          clazz.constantize.new.send(method, Sidekiq::Batch::Status.new(bid), opts) rescue nil
+          status = Sidekiq::Batch::Status.new(bid)
+          clazz.constantize.new.send(method, status, opts) rescue nil
+
+          send(event.to_sym, bid, status, parent_bid)
         end
-      end
 
-      class << self
-        def call_if_needed(event, bid)
-          needed = Sidekiq.redis do |r|
-            r.multi do
-              r.hget("BID-#{bid}", event)
-              r.hset("BID-#{bid}", event, true)
+
+        def success(bid, status, parent_bid)
+          if (parent_bid)
+            _, _, success, pending, children = Sidekiq.redis do |r|
+              r.multi do
+                r.sadd("BID-#{parent_bid}-success", bid)
+                r.expire("BID-#{parent_bid}-success", Sidekiq::Batch::BID_EXPIRE_TTL)
+                r.scard("BID-#{parent_bid}-success")
+                r.hincrby("BID-#{parent_bid}", "pending", 0)
+                r.hincrby("BID-#{parent_bid}", "children", 0)
+              end
+            end
+
+            Batch.enqueue_callbacks(:success, parent_bid) if pending.to_i.zero? && children == success
+          end
+
+          Sidekiq.redis do |r|
+            r.del "BID-#{bid}-success", "BID-#{bid}-complete", "BID-#{bid}-jids", "BID-#{bid}-failed"
+          end
+        end
+
+        def complete(bid, status, parent_bid)
+          if (parent_bid)
+            _, complete, pending, children, failure = Sidekiq.redis do |r|
+              r.multi do
+                r.sadd("BID-#{parent_bid}-complete", bid)
+                r.scard("BID-#{parent_bid}-complete")
+                r.hincrby("BID-#{parent_bid}", "pending", 0)
+                r.hincrby("BID-#{parent_bid}", "children", 0)
+                r.hlen("BID-#{parent_bid}-failed")
+              end
             end
+
+            Batch.enqueue_callbacks(:complete, parent_bid) if complete == children && pending == failure
           end
-          return if 'true' == needed[0]
-          callback, opts, queue = Sidekiq.redis do |r|
-            r.hmget("BID-#{bid}",
-                    "callback_#{event}", "callback_#{event}_opts",
-                    'callback_queue')
+
+          pending, children, success = Sidekiq.redis do |r|
+            r.multi do
+              r.hincrby("BID-#{bid}", "pending", 0)
+              r.hincrby("BID-#{bid}", "children", 0)
+              r.scard("BID-#{bid}-success")
+            end
           end
-          return unless callback
-          opts    = JSON.parse(opts) if opts
-          opts  ||= {}
-          queue ||= 'default'
-          Sidekiq::Client.push('class' => Sidekiq::Batch::Callback::Worker,
-                               'args' => [callback, event, opts, bid],
-                               'queue' => queue)
-        ensure
-          Sidekiq::Batch.cleanup_redis(bid) if event == :success
+
+          Batch.enqueue_callbacks(:success, bid) if pending.to_i.zero? && children == success
+
         end
+
       end
     end
   end

data/lib/sidekiq/batch/extension/worker.rb

@@ -0,0 +1,15 @@
+module Sidekiq::Batch::Extension
+  module Worker
+    def bid
+      Thread.current[:bid]
+    end
+
+    def batch
+      Sidekiq::Batch.new(Thread.current[:bid].bid) if Thread.current[:bid]
+    end
+
+    def valid_within_batch?
+      batch.valid?
+    end
+  end
+end

data/lib/sidekiq/batch/middleware.rb

@@ -1,10 +1,12 @@
+require_relative 'extension/worker'
+
 module Sidekiq
   class Batch
     module Middleware
       class ClientMiddleware
         def call(_worker, msg, _queue, _redis_pool = nil)
-          if (bid = Thread.current[:bid])
-            Batch.increment_job_queue(bid) if (msg[:bid] = bid)
+          if (batch = Thread.current[:bid])
+            batch.increment_job_queue(msg['jid']) if (msg[:bid] = batch.bid)
           end
           yield
         end
@@ -14,10 +16,10 @@ module Sidekiq
         def call(_worker, msg, _queue)
           if (bid = msg['bid'])
             begin
-              Thread.current[:bid] = bid
+              Thread.current[:bid] = Sidekiq::Batch.new(bid)
               yield
               Thread.current[:bid] = nil
-              Batch.process_successful_job(bid)
+              Batch.process_successful_job(bid, msg['jid'])
             rescue
               Batch.process_failed_job(bid, msg['jid'])
               raise
@@ -44,12 +46,7 @@ module Sidekiq
             chain.add Sidekiq::Batch::Middleware::ServerMiddleware
           end
         end
-        Sidekiq::Worker.send(:define_method, 'bid') do
-          Thread.current[:bid]
-        end
-        Sidekiq::Worker.send(:define_method, 'batch') do
-          Sidekiq::Batch.new(Thread.current[:bid]) if Thread.current[:bid]
-        end
+        Sidekiq::Worker.send(:include, Sidekiq::Batch::Extension::Worker)
       end
     end
   end

data/lib/sidekiq/batch/status.rb

@@ -27,6 +27,10 @@ module Sidekiq
         Sidekiq.redis { |r| r.hget("BID-#{bid}", 'total') }.to_i
       end
 
+      def parent_bid
+        Sidekiq.redis { |r| r.hget("BID-#{bid}", "parent_bid") }
+      end
+
       def failure_info
         Sidekiq.redis { |r| r.smembers("BID-#{bid}-failed") } || []
       end
@@ -35,6 +39,10 @@ module Sidekiq
         'true' == Sidekiq.redis { |r| r.hget("BID-#{bid}", 'complete') }
       end
 
+      def child_count
+        Sidekiq.redis { |r| r.hget("BID-#{bid}", 'children') }.to_i
+      end
+
       def data
         {
           total: total,
@@ -42,7 +50,8 @@ module Sidekiq
           pending: pending,
           created_at: created_at,
           complete: complete?,
-          failure_info: failure_info
+          failure_info: failure_info,
+          parent_bid: parent_bid
         }
       end
     end

data/lib/sidekiq/batch/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   class Batch
-    VERSION = '0.1.2'.freeze
+    VERSION = '0.1.3'.freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-batch
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Marcin Naglik
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-12-29 00:00:00.000000000 Z
+date: 2017-05-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -97,6 +97,7 @@ files:
 - Rakefile
 - lib/sidekiq/batch.rb
 - lib/sidekiq/batch/callback.rb
+- lib/sidekiq/batch/extension/worker.rb
 - lib/sidekiq/batch/middleware.rb
 - lib/sidekiq/batch/status.rb
 - lib/sidekiq/batch/version.rb
@@ -121,7 +122,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: Sidekiq Batch Jobs