sidekiq-addons 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e0d728ca590a42de552c8001f5989e80202d2586
+  data.tar.gz: 47a28b8b736713b12b1443f35eb44f61b5fc6b3e
+SHA512:
+  metadata.gz: 526bb3c5ad5841f0bda1e7a94f0ce9879d00bbe0c7932e9b7ba5773ea55f1a45d13be1725f67964a846bd5130f4d4e6dfc7c6983c8339870b56e7941cfd44fa7
+  data.tar.gz: f1f8c38a884c4030965d030afbfe35969cdb60e17e4dae3212de15c159a9bde99b1debaaf6f80cb1ce564fdabdf8b827557e7025696a38cc8a8f328424bebc4c

data/.gitignore

@@ -0,0 +1 @@
+pkg

data/.travis.yml

@@ -0,0 +1,17 @@
+language: ruby
+cache: bundler
+
+services:
+  redis-server
+
+rvm:
+  - 2.2.0
+
+script: 'bundle exec rake'
+
+notifications:
+  email:
+    recipients:
+      - asviresh@gmail.com
+    on_failure: change
+    on_success: never

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in sidekiq-addons.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,81 @@
+PATH
+  remote: .
+  specs:
+    sidekiq-addons (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    celluloid (0.15.2)
+      timers (~> 1.1.0)
+    connection_pool (2.2.0)
+    coveralls (0.7.1)
+      multi_json (~> 1.3)
+      rest-client
+      simplecov (>= 0.7)
+      term-ansicolor
+      thor
+    diff-lcs (1.2.5)
+    docile (1.1.5)
+    domain_name (0.5.24)
+      unf (>= 0.0.5, < 1.0.0)
+    http-cookie (1.0.2)
+      domain_name (~> 0.5)
+    json (1.8.3)
+    mime-types (2.99)
+    multi_json (1.12.1)
+    netrc (0.10.3)
+    rake (10.5.0)
+    redis (3.2.1)
+    redis-namespace (1.5.2)
+      redis (~> 3.0, >= 3.0.4)
+    rest-client (1.8.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 3.0)
+      netrc (~> 0.7)
+    rspec (3.2.0)
+      rspec-core (~> 3.2.0)
+      rspec-expectations (~> 3.2.0)
+      rspec-mocks (~> 3.2.0)
+    rspec-core (3.2.1)
+      rspec-support (~> 3.2.0)
+    rspec-expectations (3.2.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-mocks (3.2.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-support (3.2.2)
+    sidekiq (3.2.6)
+      celluloid (= 0.15.2)
+      connection_pool (>= 2.0.0)
+      json
+      redis (>= 3.0.6)
+      redis-namespace (>= 1.3.1)
+    simplecov (0.9.2)
+      docile (~> 1.1.0)
+      multi_json (~> 1.0)
+      simplecov-html (~> 0.9.0)
+    simplecov-html (0.9.0)
+    term-ansicolor (1.3.0)
+      tins (~> 1.0)
+    thor (0.19.1)
+    timers (1.1.0)
+    tins (1.3.2)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.12)
+  coveralls
+  rake (~> 10.0)
+  rspec (~> 3.0)
+  sidekiq (~> 3.2)
+  sidekiq-addons!
+
+BUNDLED WITH
+   1.12.5

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Viresh S
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,109 @@
+# Sidekiq::Addons
+
+[![Join the chat at https://gitter.im/vireshas/sidekiq-addons](https://badges.gitter.im/vireshas/sidekiq-addons.svg)](https://gitter.im/vireshas/sidekiq-addons?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+[![Build Status](https://travis-ci.org/vireshas/sidekiq-addons.svg?branch=master)](https://travis-ci.org/vireshas/sidekiq-addons)
+[![Coverage Status](https://coveralls.io/repos/github/vireshas/sidekiq-addons/badge.svg?branch=master)](https://coveralls.io/github/vireshas/sidekiq-addons?branch=master)
+
+
+  * Dynamically change job priority with in the same queue.  
+ 
+## Overview
+  
+Enqueue job with default priority
+```ruby
+  MockWorker.perform_async(1)
+```
+Enqueue job with a priority
+```ruby
+ job1 = MockWorker.perform_async(1, {:with_priority => 80})
+ job2 = MockWorker.perform_async(1, {:with_priority => 90})
+ job3 = MockWorker.perform_async(1, {:with_priority => 100})
+ job4 = MockWorker.perform_async(1, {:with_priority => 70})
+```
+When jobs are enqueued in this order, this gems makes sure that, job3 is the one that will be executed next(as it has the highest priority), this will be followed by job2, job1 and job4.
+
+## Usage
+
+To always use default priority set ignore_priority => true at worker level
+```ruby
+class IgnoreWorker
+  include Sidekiq::Worker
+  sidekiq_options :ignore_priority => true
+
+  def perform(arg)
+  end
+end
+```
+
+Moves job to priority queue when with_priority or blocks value is greater than min_priority
+```ruby
+class MinPriorityWorker
+  include Sidekiq::Worker
+  sidekiq_options :min_priority => 50
+
+  def perform(arg)
+  end
+end
+```
+
+Use a block to assign priority dynamically
+```ruby
+class LazyEvalWorker
+  include Sidekiq::Worker
+  sidekiq_options :lazy_eval => Proc.new { |param|
+    param.first.to_i > 50
+  }
+
+  def perform(arg)
+  end
+end
+```
+Proc should either return a number or true to use priority scheduling
+
+
+## Benefits  
+  * Doesnt interrupt those jobs that are already getting executed, but, makes sure that the next job that will be executed will be a highest priority job.   
+  * Minimal code changes: You just have to pass an extra param when you enqueue a job and jobs will be scheduled based on this param value.
+  * Minimal network transfer: loads a script in Redis and uses SHA to execute it. This greatly reduces network data transfer.
+  * Can talk to remote Redis: From your stack, you can pass a REDIS_URL in Sidekiq.options and it can talk to that Redis.
+  * When Sidekiq is interruppted, active jobs are re-enqueued with the existing priority. When Sidekiq boots-up, it will still pick the highest prortized job.
+  * Has automic ZPOP
+
+## Coming up
+  * Uniqueness: removes duplicate jobs
+  * Cron(?)
+  * Stats
+  * Monitor
+  * Sidekiq-UI: integrate with sidekiq-UI
+  * 
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'sidekiq-addons'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install sidekiq-addons
+
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/vireshas/sidekiq-addons.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "sidekiq/addons"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/sidekiq/addons/prioritize/deqr.rb

@@ -0,0 +1,59 @@
+require "celluloid"
+require "sidekiq/fetch"
+
+module Sidekiq::Addons::Prioritize
+  class Deqr < Sidekiq::BasicFetch
+
+    attr_accessor :script_sha
+
+    def zpop(queue)
+      Sidekiq.redis do |con|
+        #TODO: for some reason pipeline and multi didnt work; revisit
+        if ( self.script_sha.nil? ) or !con.script(:exists, self.script_sha)
+          self.script_sha = con.script(:load, Sidekiq::Addons::Util::ZPOP)
+        end
+
+        con.evalsha(self.script_sha, [queue])
+      end
+    end
+
+    def retrieve_work
+      priority_job = nil
+      @queues.uniq.each do |q|
+        q_name = Sidekiq::Addons::Util.priority_job_queue_name(q.split("queue:").last)
+        priority_job = zpop(q_name)
+        break if priority_job
+      end
+
+      if priority_job.nil?
+        return super
+      else
+        work = JSON.load(priority_job)
+        return UnitOfWork.new(work["queue"], priority_job)
+      end
+    end
+
+    def self.bulk_requeue(inprogress, options)
+      return if inprogress.empty?
+
+      jobs_unhandled = []
+      inprogress.each do |unit_work|
+        unit_work = JSON.load(unit_work)
+        priority = Sidekiq::Addons::Prioritize::Enqr.get_priority_from_msg(unit_work)
+        if ( priority > 0 )
+          Sidekiq.redis do |con|
+            Sidekiq::Addons::Prioritize::Enqr.enqueue_with_priority(con, unit_work["queue"], priority, unit_work)
+          end
+        else
+          jobs_unhandled << unit_work.to_json
+        end
+      end
+
+      Sidekiq.logger.info("Pushed #{inprogress.size - jobs_unhandled.size} jobs back to priority_queue in Redis")
+
+      super(jobs_unhandled, options)
+    end
+  end
+
+end
+

data/lib/sidekiq/addons/prioritize/enqr.rb

@@ -0,0 +1,67 @@
+module Sidekiq::Addons::Prioritize
+  class Enqr
+
+    def compute_priority(queue, msg)
+      to_ignore_qs = Sidekiq.options[:ignore_priority] || []
+      min_priority = msg["min_priority"] || Sidekiq.options[:min_priority] || 0
+      ignore_priority = msg["ignore_priority"]
+      lazy_eval = msg["lazy_eval"]
+
+      priority = min_priority
+      if ignore_priority or (to_ignore_qs.include?(queue))
+        priority = 0
+
+      elsif lazy_eval
+        priority = lazy_eval.call(msg["args"])
+        unless priority.is_a?(Integer)
+          priority = priority ? (min_priority + 1) : (min_priority - 1)
+        end
+        msg.delete("lazy_eval")
+      else
+        priority = self.class.get_priority_from_msg(msg)
+
+      end
+
+      return (priority > min_priority) ? priority : nil
+    end
+
+    def call(worker_class, msg, queue, redis_pool)
+      priority = compute_priority(queue, msg)
+      if priority
+        msg["queue"] = "queue:#{queue}"
+        redis_pool.with {|con| self.class.enqueue_with_priority(con, queue, priority, msg) }
+        return false
+      else
+        yield
+      end
+    end
+
+    class << self
+
+      def get_priority_from_msg(msg)
+        priority = nil
+
+        if msg["args"].is_a?(Array)
+          msg["args"].each do |param|
+            if param.is_a?(Hash) and ( param.has_key?(:with_priority) \
+               or param.has_key?("with_priority") )
+
+              priority = param[:with_priority]
+              priority = param["with_priority"] unless priority
+              break
+            end
+          end
+        end
+
+        return priority.to_i
+      end
+
+      def enqueue_with_priority(con, queue, priority, msg)
+        q_name = Sidekiq::Addons::Util.priority_job_queue_name(queue)
+        return con.zadd(q_name, priority, msg.to_json)
+      end
+
+    end
+
+  end
+end

data/lib/sidekiq/addons/util.rb

@@ -0,0 +1,22 @@
+module Sidekiq
+  module Addons
+    module Util
+
+      ZPOP = %q(
+        local resp = redis.call('zrevrange', KEYS[1], '0', '0')
+        if (resp[1] ~= nil) then
+          local val = resp[# resp]
+          redis.call('zrem', KEYS[1], val)
+          return val
+        else
+          return false
+        end
+      )
+
+      def self.priority_job_queue_name(q)
+        return "sidekiq-addons:pq:#{q}"
+      end
+
+    end
+  end
+end

data/lib/sidekiq/addons/version.rb

@@ -0,0 +1,5 @@
+module Sidekiq
+  module Addons
+    VERSION = "0.1.0"
+  end
+end

data/lib/sidekiq/addons.rb

@@ -0,0 +1,34 @@
+require "sidekiq"
+require "sidekiq/addons/util"
+require "sidekiq/addons/version"
+require "sidekiq/addons/prioritize/enqr"
+require "sidekiq/addons/prioritize/deqr"
+
+module Sidekiq
+  module Addons
+
+    #set redis_url to default redis setttings if its not passed
+    unless Sidekiq.options[:redis_url]
+      Sidekiq.options[:redis_url] = {
+        url: 'redis://127.0.0.1:6379/0'
+      }
+    end
+
+    Sidekiq.configure_client do |config|
+      config.client_middleware do |chain|
+        chain.add Sidekiq::Addons::Prioritize::Enqr
+        config.redis = Sidekiq.options[:redis_url]
+      end
+    end
+
+    Sidekiq.configure_server do |config|
+      Sidekiq.options[:fetch] = Sidekiq::Addons::Prioritize::Deqr
+      config.redis = Sidekiq.options[:redis_url]
+      config.client_middleware do |chain|
+        chain.add Sidekiq::Addons::Prioritize::Enqr
+        config.redis = Sidekiq.options[:redis_url]
+      end
+    end
+
+  end
+end

data/sidekiq-addons.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'sidekiq/addons/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "sidekiq-addons"
+  spec.version       = Sidekiq::Addons::VERSION
+  spec.authors       = ["Viresh S"]
+  spec.email         = ["asviresh@gmail.com"]
+
+  spec.summary       = %q{ Prioritize jobs in a queue, Uniqueness in jobs and Sidekiq based cron.}
+  spec.description   = %q{ Prioritize jobs in a queue, Uniqueness in jobs and Sidekiq based cron.}
+  spec.homepage      = "https://github.com/vireshas/sidekiq-addons"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|pkg)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "coveralls"
+  spec.add_development_dependency "sidekiq", "~> 3.2"
+  spec.add_development_dependency "bundler", "~> 1.12"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+end

metadata

@@ -0,0 +1,129 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq-addons
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Viresh S
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-08-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: " Prioritize jobs in a queue, Uniqueness in jobs and Sidekiq based cron."
+email:
+- asviresh@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/sidekiq/addons.rb
+- lib/sidekiq/addons/prioritize/deqr.rb
+- lib/sidekiq/addons/prioritize/enqr.rb
+- lib/sidekiq/addons/util.rb
+- lib/sidekiq/addons/version.rb
+- sidekiq-addons.gemspec
+homepage: https://github.com/vireshas/sidekiq-addons
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.3
+signing_key: 
+specification_version: 4
+summary: Prioritize jobs in a queue, Uniqueness in jobs and Sidekiq based cron.
+test_files: []