sidekiq-batch 0.1.0.pre → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 279810838005ed551056776d77629a5cb9f244c2
-  data.tar.gz: d4f09f5eab7044a6811b69d2090f9ecf3ac53e02
+  metadata.gz: cf8a9c0f01008a49c2b6584a0b1238eb76e3db57
+  data.tar.gz: b30363e23b7e82c8092b5691089ebc61ed10cef2
 SHA512:
-  metadata.gz: bcc10bdb6bc8e2188ebfe97b4382f9f9de9f83a283ff3c72fbdc98cebf899b1e4e360964ed0a8ef1fe42c9fbc4d4fcdcd6f280576afaa7dd4a07de8cc87a0967
-  data.tar.gz: 2d646ba8e04fb71229bfe42f0cd5975e771b7c42ecd62c981fd5971709c229bb9e8a129ae13d1e22210fe3a5f1d7fed31f5ab5e083f46c937f7feba7fcec8a0b
+  metadata.gz: 761296bf005a0328ca8f152c776fbc1891e86b6e996ce4ead7cfe85f1c97b37b39111a2e6fb3950af2a228fc9b8aa427ce2666dc66b68dac27e87ab856db4751
+  data.tar.gz: 8b268734d9c29148eeb661fb6cdc3ba6c8ff08a128c4ca8bdf28e4f313007614189191242bab085a7a8f043667df50af75a87585bbd1c76af1207e120b70c209

data/.gitignore

@@ -8,3 +8,4 @@
 /spec/reports/
 /tmp/
 dump.rdb
+sidekiq-batch-*.gem

data/Gemfile

@@ -1,4 +1,7 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in sidekiq-batch.gemspec
 gemspec
+
+group :test do
+  gem 'codeclimate-test-reporter'
+end

data/README.md

@@ -1,5 +1,15 @@
+[gem]: https://rubygems.org/gems/sidekiq-batch
+[travis]: https://travis-ci.org/breamware/sidekiq-batch
+[codeclimate]: https://codeclimate.com/github/breamware/sidekiq-batch
+
 # Sidekiq::Batch
 
+[![Gem Version](https://badge.fury.io/rb/sidekiq-batch.svg)][gem]
+[![Build Status](https://travis-ci.org/breamware/sidekiq-batch.svg?branch=master)][travis]
+[![Code Climate](https://codeclimate.com/github/breamware/sidekiq-batch/badges/gpa.svg)][codeclimate]
+[![Code Climate](https://codeclimate.com/github/breamware/sidekiq-batch/badges/coverage.svg)][codeclimate]
+[![Code Climate](https://codeclimate.com/github/breamware/sidekiq-batch/badges/issue_count.svg)][codeclimate]
+
 Simple Sidekiq Batch Job implementation.
 
 ## Installation

data/lib/sidekiq/batch/callback.rb

@@ -8,7 +8,7 @@ module Sidekiq
           return unless %w(success complete).include?(event)
           instance = clazz.constantize.send(:new) rescue nil
           return unless instance
-          instance.send("on_#{event}", Status.new(bid), opts) rescue nil
+          instance.send("on_#{event}", Sidekiq::Batch::Status.new(bid), opts) rescue nil
         end
       end
 
@@ -16,13 +16,13 @@ module Sidekiq
         def call_if_needed(event, bid)
           needed = Sidekiq.redis do |r|
             r.multi do
-              r.hget(bid, event)
-              r.hset(bid, event, true)
+              r.hget("BID-#{bid}", event)
+              r.hset("BID-#{bid}", event, true)
             end
           end
           return if 'true' == needed[0]
           callback, opts, queue = Sidekiq.redis do |r|
-            r.hmget(bid,
+            r.hmget("BID-#{bid}",
                     "callback_#{event}", "callback_#{event}_opts",
                     'callback_queue')
           end
@@ -33,6 +33,8 @@ module Sidekiq
           Sidekiq::Client.push('class' => Sidekiq::Batch::Callback::Worker,
                                'args' => [callback, event, opts, bid],
                                'queue' => queue)
+        ensure
+          Sidekiq::Batch.cleanup_redis(bid) if event == :success
         end
       end
     end

data/lib/sidekiq/batch/middleware.rb

@@ -1,29 +1,10 @@
 module Sidekiq
   class Batch
     module Middleware
-      def self.extended(base)
-        base.class_eval do
-          register_middleware
-        end
-      end
-
-      def register_middleware
-        Sidekiq.configure_server do |config|
-          config.client_middleware do |chain|
-            chain.add ClientMiddleware
-          end
-          config.server_middleware do |chain|
-            chain.add ClientMiddleware
-            chain.add ServerMiddleware
-          end
-        end
-      end
-
       class ClientMiddleware
         def call(_worker, msg, _queue, _redis_pool = nil)
           if (bid = Thread.current[:bid])
-            Batch.increment_job_queue(bid) if
-            msg[:bid] = bid
+            Batch.increment_job_queue(bid) if (msg[:bid] = bid)
           end
           yield
         end
@@ -33,19 +14,45 @@ module Sidekiq
         def call(_worker, msg, _queue)
           if (bid = msg['bid'])
             begin
+              Thread.current[:bid] = bid
               yield
+              Thread.current[:bid] = nil
               Batch.process_successful_job(bid)
             rescue
               Batch.process_failed_job(bid, msg['jid'])
               raise
+            ensure
+              Thread.current[:bid] = nil
             end
           else
             yield
           end
         end
       end
+
+      def self.configure
+        Sidekiq.configure_client do |config|
+          config.client_middleware do |chain|
+            chain.add Sidekiq::Batch::Middleware::ClientMiddleware
+          end
+        end
+        Sidekiq.configure_server do |config|
+          config.client_middleware do |chain|
+            chain.add Sidekiq::Batch::Middleware::ClientMiddleware
+          end
+          config.server_middleware do |chain|
+            chain.add Sidekiq::Batch::Middleware::ServerMiddleware
+          end
+        end
+        Sidekiq::Worker.send(:define_method, 'bid') do
+          Thread.current[:bid]
+        end
+        Sidekiq::Worker.send(:define_method, 'batch') do
+          Sidekiq::Batch.new(Thread.current[:bid]) if Thread.current[:bid]
+        end
+      end
     end
   end
 end
 
-Sidekiq::Batch::Middleware.send(:extend, Sidekiq::Batch::Middleware)
+Sidekiq::Batch::Middleware.configure

data/lib/sidekiq/batch/status.rb

@@ -1,7 +1,7 @@
 module Sidekiq
   class Batch
     class Status
-      attr_reader :bid, :total, :failures, :created_at, :failure_info
+      attr_reader :bid
 
       def initialize(bid)
         @bid = bid
@@ -12,11 +12,27 @@ module Sidekiq
       end
 
       def pending
-        Sidekiq.redis { |r| r.get("#{bid}-to_process") }.to_i
+        Sidekiq.redis { |r| r.hget("BID-#{bid}", 'pending') }.to_i
+      end
+
+      def failures
+        Sidekiq.redis { |r| r.scard("BID-#{bid}-failed") }.to_i
+      end
+
+      def created_at
+        Sidekiq.redis { |r| r.hget("BID-#{bid}", 'created_at') }
+      end
+
+      def total
+        Sidekiq.redis { |r| r.hget("BID-#{bid}", 'total') }.to_i
+      end
+
+      def failure_info
+        Sidekiq.redis { |r| r.smembers("BID-#{bid}-failed") } || []
       end
 
       def complete?
-        'true' == Sidekiq.redis { |r| r.hget(bid, 'complete') }
+        'true' == Sidekiq.redis { |r| r.hget("BID-#{bid}", 'complete') }
       end
 
       def data

data/lib/sidekiq/batch/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   class Batch
-    VERSION = '0.1.0.pre'.freeze
+    VERSION = '0.1.0'.freeze
   end
 end

data/lib/sidekiq/batch.rb

@@ -10,47 +10,70 @@ module Sidekiq
   class Batch
     class NoBlockGivenError < StandardError; end
 
+    BID_EXPIRE_TTL = 108_000
+
     attr_reader :bid, :description, :callback_queue
 
     def initialize(existing_bid = nil)
       @bid = existing_bid || SecureRandom.urlsafe_base64(10)
-      Sidekiq.redis { |r| r.set("#{bid}-to_process", 0) }
+      Sidekiq.redis do |r|
+        r.multi do
+          r.hset("BID-#{bid}", 'created_at', Time.now)
+          r.expire("BID-#{bid}", BID_EXPIRE_TTL)
+        end
+      end
     end
 
     def description=(description)
       @description = description
-      Sidekiq.redis { |r| r.hset(bid, 'description', description) }
+      persist_bid_attr('description', description)
     end
 
     def callback_queue=(callback_queue)
       @callback_queue = callback_queue
-      Sidekiq.redis { |r| r.hset(bid, 'callback_queue', callback_queue) }
+      persist_bid_attr('callback_queue', callback_queue)
     end
 
     def on(event, callback, options = {})
       return unless %w(success complete).include?(event.to_s)
       Sidekiq.redis do |r|
-        r.hset(bid, "callback_#{event}", callback)
-        r.hset(bid, "callback_#{event}_opts", options.to_json)
+        r.multi do
+          r.hset("BID-#{bid}", "callback_#{event}", callback)
+          r.hset("BID-#{bid}", "callback_#{event}_opts", options.to_json)
+          r.expire("BID-#{bid}", BID_EXPIRE_TTL)
+        end
       end
     end
 
     def jobs
       raise NoBlockGivenError unless block_given?
 
-      Batch.increment_job_queue(bid)
+      Sidekiq.redis { |r| r.hincrby("BID-#{bid}", 'to_process', 1) }
       Thread.current[:bid] = bid
       yield
-      Batch.process_successful_job(bid)
+      Thread.current[:bid] = nil
+      Sidekiq.redis { |r| r.hincrby("BID-#{bid}", 'to_process', -1) }
+    end
+
+    private
+
+    def persist_bid_attr(attribute, value)
+      Sidekiq.redis do |r|
+        r.multi do
+          r.hset("BID-#{bid}", attribute, value)
+          r.expire("BID-#{bid}", BID_EXPIRE_TTL)
+        end
+      end
     end
 
     class << self
       def process_failed_job(bid, jid)
         to_process = Sidekiq.redis do |r|
           r.multi do
-            r.sadd("#{bid}-failed", jid)
-            r.scard("#{bid}-failed")
-            r.get("#{bid}-to_process")
+            r.sadd("BID-#{bid}-failed", jid)
+            r.scard("BID-#{bid}-failed")
+            r.hget("BID-#{bid}", 'to_process')
+            r.expire("BID-#{bid}-failed", BID_EXPIRE_TTL)
           end
         end
         if to_process[2].to_i == to_process[1].to_i
@@ -59,20 +82,36 @@ module Sidekiq
       end
 
       def process_successful_job(bid)
-        to_process = Sidekiq.redis do |r|
+        out = Sidekiq.redis do |r|
           r.multi do
-            r.decr("#{bid}-to_process")
-            r.get("#{bid}-to_process")
+            r.hincrby("BID-#{bid}", 'to_process', -1)
+            r.scard("BID-#{bid}-failed")
+            r.hincrby("BID-#{bid}", 'pending', -1)
+            r.expire("BID-#{bid}", BID_EXPIRE_TTL)
           end
         end
-        if to_process[1].to_i == 0
-          Callback.call_if_needed(:success, bid)
-          Callback.call_if_needed(:complete, bid)
+
+        puts "processed process_successful_job"
+        Callback.call_if_needed(:complete, bid) if out[1].to_i == out[0].to_i
+        Callback.call_if_needed(:success, bid) if out[0].to_i.zero?
+      end
+
+      def cleanup_redis(bid)
+        Sidekiq.redis do |r|
+          r.del("BID-#{bid}",
+                "BID-#{bid}-failed")
         end
       end
 
       def increment_job_queue(bid)
-        Sidekiq.redis { |r| r.incr("#{bid}-to_process") }
+        Sidekiq.redis do |r|
+          r.multi do
+            %w(to_process pending total).each do |c|
+              r.hincrby("BID-#{bid}", c, 1)
+            end
+            r.expire("BID-#{bid}", BID_EXPIRE_TTL)
+          end
+        end
       end
     end
   end

data/sidekiq-batch.gemspec

@@ -19,9 +19,10 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "sidekiq", "~> 4"
+  spec.add_dependency "sidekiq", "~> 3"
 
   spec.add_development_dependency "bundler", "~> 1.12"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "fakeredis", "~> 0.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-batch
 version: !ruby/object:Gem::Version
-  version: 0.1.0.pre
+  version: 0.1.0
 platform: ruby
 authors:
 - Marcin Naglik
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-08-22 00:00:00.000000000 Z
+date: 2016-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4'
+        version: '3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4'
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: fakeredis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
 description: Sidekiq Batch Jobs Implementation
 email:
 - marcin.naglik@gmail.com
@@ -102,9 +116,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.5.1