sidekiq-paquet 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2d30b4621c71e1966308bbe0a68f10aeb776a332
+  data.tar.gz: 13476e7dbafc931566f3e9cd6d2861fa1175583f
+SHA512:
+  metadata.gz: d17cefb200e17b149424f2c1403b053e11c23da4c016e670b445f9108da393cc081c9281a91cb4a72faa320c5fd51fc4641d2579e0210daacae43367a7e9a525
+  data.tar.gz: 46be4131522f343c047226c2ae9b3b06082590a2054ac98cfd11bec473f2e4de72d1172fc3a532998165e3ff0001ad61e2a7fa52b674c9d00535e77b25d31ee8

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+services:
+  - redis-server
+rvm:
+  - 2.2.2
+before_install: gem install bundler -v 1.10.6

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,13 @@
+# Contributor Code of Conduct
+
+As contributors and maintainers of this project, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free experience for everyone, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, ethnicity, age, or religion.
+
+Examples of unacceptable behavior by participants include the use of sexual language or imagery, derogatory comments or personal attacks, trolling, public or private harassment, insults, or other unprofessional conduct.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. Project maintainers who do not follow the Code of Conduct may be removed from the project team.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers.
+
+This Code of Conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org), version 1.0.0, available at [http://contributor-covenant.org/version/1/0/0/](http://contributor-covenant.org/version/1/0/0/)

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in sidekiq-paquet.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 ccocchi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,66 @@
+# Sidekiq::Paquet
+
+Instead of enqueueing and processing jobs one at a time, enqueue them one by one process them in bulk.
+Useful for grouping background API calls or intensive database inserts coming from multiple sources.
+
+## Installation
+
+```ruby
+gem install 'sidekiq-paquet'
+```
+
+sidekiq-bulk requires Sidekiq 4+. If you're using Sidekiq < 4, take a look at [sidekiq-grouping](https://github.com/gzigzigzeo/sidekiq-grouping/) for similar features.
+
+## Usage
+
+Add `bulk: true` option to your worker's `sidekiq_options` to have jobs processed in bulk. The size of the bulk can be configured per worker. If not specified, the `Sidekiq::Paquet.options[:default_bulk_size]` is used.
+
+```ruby
+class ElasticIndexerWorker
+  include Sidekiq::Worker
+
+  sidekiq_options bulk: true, bulk_size: 100
+
+  def perform(values)
+    # Perform work with the array of values
+  end
+end
+```
+
+Instead of being processed by Sidekiq, jobs will be stored into a separate queue and periodically, a poller will retrieve them by slice of `bulk_size` and enqueue a regular Sidekiq job with that bulk as argument.
+Thus, your worker will only be invoked with an array of values, never with single values themselves.
+
+For example, if you call `perform_async` twice on the previous worker
+
+```ruby
+ElasticIndexerWorker.perform_async({ delete: { _index: 'users', _id: 1, _type: 'user' } })
+ElasticIndexerWorker.perform_async({ delete: { _index: 'users', _id: 2, _type: 'user' } })
+```
+
+the worker instance will receive these values as a single argument
+
+```ruby
+[
+  { delete: { _index: 'users', _id: 1, _type: 'user' } },
+  { delete: { _index: 'users', _id: 2, _type: 'user' } }
+]
+```
+
+## Configuration
+
+You can change global configuration by modifying the `Sidekiq::Paquet.options` hash.
+
+```
+  Sidekiq::Paquet.options[:default_bulk_size] = 500 # Default is 100
+  Sidekiq::Paquet.options[:average_bulk_flush_interval] = 30 # Default is 15
+```
+
+The `average_bulk_flush_interval` represent the average time elapsed between two polling of values. This scales with the number of sidekiq processes you're running. So if you have 5 sidekiq processes, and set the `average_bulk_flush_interval` to 15, each process will check for new bulk jobs every 75 seconds -- so that in average, the bulk queue will be checked every 15 seconds.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/ccocchi/sidekiq-paquet. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,12 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+
+  t.warning    = false
+  t.test_files = FileList['test/**/test_*.rb']
+end
+
+task :default => :test

data/lib/sidekiq/paquet/batch.rb

@@ -0,0 +1,42 @@
+module Sidekiq
+  module Paquet
+    module Batch
+
+      def self.append(item)
+        worker_name = item['class'.freeze]
+        args = item.fetch('args'.freeze, [])
+
+        Sidekiq.redis do |conn|
+          conn.multi do
+            conn.sadd('bulks'.freeze, worker_name)
+            conn.rpush("bulk:#{worker_name}", Sidekiq.dump_json(args))
+          end
+        end
+      end
+
+      def self.enqueue_jobs
+        Sidekiq.redis do |conn|
+          workers = conn.smembers('bulks'.freeze)
+
+          workers.each do |worker|
+            klass = worker.constantize
+            opts  = klass.get_sidekiq_options
+            items = conn.lrange("bulk:#{worker}", 0, -1)
+            items.map! { |i| Sidekiq.load_json(i) }
+
+            items.each_slice(opts.fetch('bulk_size'.freeze, Sidekiq::Paquet.options[:default_bulk_size])) do |vs|
+              Sidekiq::Client.push(
+                'class' => worker,
+                'queue' => opts['queue'.freeze],
+                'args'  => vs
+              )
+            end
+
+            conn.ltrim("bulk:#{worker}", items.size, -1)
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/sidekiq/paquet/list.rb

@@ -0,0 +1,21 @@
+module Sidekiq
+  module Paquet
+    class List
+      def initialize(name)
+        @lname = "bulk:#{name}"
+      end
+
+      def size
+        Sidekiq.redis { |c| c.llen(@lname) }
+      end
+
+      def items
+        Sidekiq.redis { |c| c.lrange(@lname, 0, -1) }
+      end
+
+      def clear
+        Sidekiq.redis { |c| c.del(@lname) }
+      end
+    end
+  end
+end

data/lib/sidekiq/paquet/middleware.rb

@@ -0,0 +1,14 @@
+module Sidekiq
+  module Paquet
+    class Middleware
+      def call(worker, item, queue, redis_pool = nil)
+        if item['bulk'.freeze]
+          Batch.append(item)
+          false
+        else
+          yield
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/paquet/poller.rb

@@ -0,0 +1,84 @@
+require 'sidekiq/util'
+require 'sidekiq/scheduled'
+
+module Sidekiq
+  module Paquet
+    class Poller < Sidekiq::Scheduled::Poller
+
+      def initialize
+        @sleeper  = ConnectionPool::TimedStack.new
+        @done     = false
+      end
+
+      def start
+        @thread ||= safe_thread('bulk') do
+          initial_wait
+
+          while !@done
+            enqueue
+            wait
+          end
+          Sidekiq.logger.info('Bulk exiting...')
+        end
+      end
+
+      def enqueue
+        begin
+          Batch.enqueue_jobs
+        rescue => ex
+          # Most likely a problem with redis networking.
+          # Punt and try again at the next interval
+          logger.error ex.message
+          logger.error ex.backtrace.first
+        end
+      end
+
+      private
+
+      # Calculates a random interval that is ±50% the desired average.
+      def random_poll_interval
+        avg = poll_interval_average.to_f
+        avg * rand + avg / 2
+      end
+
+      # We do our best to tune the poll interval to the size of the active Sidekiq
+      # cluster.  If you have 30 processes and poll every 15 seconds, that means one
+      # Sidekiq is checking Redis every 0.5 seconds - way too often for most people
+      # and really bad if the retry or scheduled sets are large.
+      #
+      # Instead try to avoid polling more than once every 15 seconds.  If you have
+      # 30 Sidekiq processes, we'll poll every 30 * 15 or 450 seconds.
+      # To keep things statistically random, we'll sleep a random amount between
+      # 225 and 675 seconds for each poll or 450 seconds on average.  Otherwise restarting
+      # all your Sidekiq processes at the same time will lead to them all polling at
+      # the same time: the thundering herd problem.
+      #
+      # We only do this if poll_interval_average is unset (the default).
+      def poll_interval_average
+        if Sidekiq::Paquet.options[:dynamic_interval_scaling]
+          scaled_poll_interval
+        else
+          Sidekiq::Paquet.options[:bulk_flush_interval] ||= scaled_poll_interval
+        end
+      end
+
+      # Calculates an average poll interval based on the number of known Sidekiq processes.
+      # This minimizes a single point of failure by dispersing check-ins but without taxing
+      # Redis if you run many Sidekiq processes.
+      def scaled_poll_interval
+        pcount = Sidekiq::ProcessSet.new.size
+        pcount = 1 if pcount == 0
+        pcount * Sidekiq::Paquet.options[:average_bulk_flush_interval]
+      end
+
+      def initial_wait
+        # Have all processes sleep between 5-15 seconds.  10 seconds
+        # to give time for the heartbeat to register (if the poll interval is going to be calculated by the number
+        # of workers), and 5 random seconds to ensure they don't all hit Redis at the same time.
+        total = INITIAL_WAIT + (15 * rand)
+        @sleeper.pop(total)
+      rescue Timeout::Error
+      end
+    end
+  end
+end

data/lib/sidekiq/paquet/version.rb

@@ -0,0 +1,5 @@
+module Sidekiq
+  module Paquet
+    VERSION = '0.1.0'
+  end
+end

data/lib/sidekiq/paquet.rb

@@ -0,0 +1,47 @@
+require 'sidekiq'
+require 'sidekiq/paquet/version'
+
+require 'sidekiq/paquet/list'
+require 'sidekiq/paquet/batch'
+require 'sidekiq/paquet/middleware'
+require 'sidekiq/paquet/poller'
+
+module Sidekiq
+  module Paquet
+    DEFAULTS = {
+      default_bulk_size: 100,
+      bulk_flush_interval: nil,
+      average_bulk_flush_interval: 15,
+      dynamic_interval_scaling: true
+    }
+
+    def self.options
+      @options ||= DEFAULTS.dup
+    end
+
+    def self.options=(opts)
+      @options = opts
+    end
+  end
+end
+
+Sidekiq.configure_client do |config|
+  config.client_middleware do |chain|
+    chain.add Sidekiq::Paquet::Middleware
+  end
+end
+
+Sidekiq.configure_server do |config|
+  config.client_middleware do |chain|
+    chain.add Sidekiq::Paquet::Middleware
+  end
+
+  config.on(:startup) do
+    config.options[:bulk_poller] = Sidekiq::Paquet::Poller.new
+    config.options[:bulk_poller].start
+  end
+
+  config.on(:shutdown) do
+    config.options[:bulk_poller].terminate
+  end
+end

data/sidekiq-bulk.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'sidekiq/paquet/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "sidekiq-paquet"
+  spec.version       = Sidekiq::Paquet::VERSION
+  spec.authors       = ["ccocchi"]
+  spec.email         = ["cocchi.c@gmail.com"]
+
+  spec.summary       = "Bulk processing for sidekiq 4+"
+  spec.homepage      = "https://github.com/ccocchi/sidekiq-paquet"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "sidekiq", ">= 4"
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest"
+end

metadata

@@ -0,0 +1,115 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq-paquet
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- ccocchi
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-12-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- cocchi.c@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/sidekiq/paquet.rb
+- lib/sidekiq/paquet/batch.rb
+- lib/sidekiq/paquet/list.rb
+- lib/sidekiq/paquet/middleware.rb
+- lib/sidekiq/paquet/poller.rb
+- lib/sidekiq/paquet/version.rb
+- sidekiq-bulk.gemspec
+homepage: https://github.com/ccocchi/sidekiq-paquet
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Bulk processing for sidekiq 4+
+test_files: []
+has_rdoc: