sidekiq-recycler 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: eb953e946bdabfda52012df96add214084291736
+  data.tar.gz: 74e8809c6ae424ba44927c0c84509a7784bd1066
+SHA512:
+  metadata.gz: 558a89279d337b8f4d9fea71c9e561b49e24a509bf35500e64f26ae54144e9e5e062757e9e331e67732d4097e8a82de8a6338a304b2ef685fd8885b5b4312385
+  data.tar.gz: e86bb903e9188e6ce52e1df1706841d96d27491a4433977ef20b3a8d45d48c9299c9e783a4e794ba2655124db90e7be42176579b152d713829256d40c2865891

data/Gemfile

@@ -0,0 +1,17 @@
+
+source "https://rubygems.org"
+
+gem "sidekiq"
+
+group :local do
+  gem "sidekiq-recycler", :path => "."
+end
+
+group :development do
+  gem "rake"
+  gem "jeweler"
+  gem "yard"
+  gem "logging"
+  gem "easycov"
+  gem "micron", :github => "chetan/micron"
+end

data/Gemfile.lock

@@ -0,0 +1,107 @@
+GIT
+  remote: git://github.com/chetan/micron.git
+  revision: d722ad58ca390cdaf01d27d77aea1fbe951d0bca
+  specs:
+    micron (0.1.0)
+      colorize
+      easycov
+      hitimes
+
+PATH
+  remote: .
+  specs:
+    sidekiq-recycler (0.1.0)
+      sidekiq
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.3.5)
+    builder (3.2.2)
+    celluloid (0.15.2)
+      timers (~> 1.1.0)
+    colorize (0.6.0)
+    connection_pool (1.1.0)
+    easycov (0.3.1)
+      multi_json
+      simplecov
+      simplecov-console
+      simplecov-html
+    faraday (0.8.8)
+      multipart-post (~> 1.2.0)
+    git (1.2.6)
+    github_api (0.10.1)
+      addressable
+      faraday (~> 0.8.1)
+      hashie (>= 1.2)
+      multi_json (~> 1.4)
+      nokogiri (~> 1.5.2)
+      oauth2
+    hashie (2.0.5)
+    highline (1.6.19)
+    hirb (0.7.1)
+    hitimes (1.2.1)
+    httpauth (0.2.0)
+    jeweler (1.8.8)
+      builder
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      github_api (= 0.10.1)
+      highline (>= 1.6.15)
+      nokogiri (= 1.5.10)
+      rake
+      rdoc
+    json (1.8.0)
+    jwt (0.1.8)
+      multi_json (>= 1.5)
+    little-plugger (1.1.3)
+    logging (1.8.1)
+      little-plugger (>= 1.1.3)
+      multi_json (>= 1.3.6)
+    multi_json (1.8.1)
+    multi_xml (0.5.5)
+    multipart-post (1.2.0)
+    nokogiri (1.5.10)
+    oauth2 (0.9.2)
+      faraday (~> 0.8)
+      httpauth (~> 0.2)
+      jwt (~> 0.1.4)
+      multi_json (~> 1.0)
+      multi_xml (~> 0.5)
+      rack (~> 1.2)
+    rack (1.5.2)
+    rake (10.1.0)
+    rdoc (4.0.1)
+      json (~> 1.4)
+    redis (3.0.5)
+    redis-namespace (1.3.1)
+      redis (~> 3.0.0)
+    sidekiq (2.15.1)
+      celluloid (>= 0.15.1)
+      connection_pool (>= 1.0.0)
+      json
+      redis (>= 3.0.4)
+      redis-namespace (>= 1.3.1)
+    simplecov (0.7.1)
+      multi_json (~> 1.0)
+      simplecov-html (~> 0.7.1)
+    simplecov-console (0.1.3)
+      colorize
+      hirb
+      simplecov
+    simplecov-html (0.7.1)
+    timers (1.1.0)
+    yard (0.8.7.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  easycov
+  jeweler
+  logging
+  micron!
+  rake
+  sidekiq
+  sidekiq-recycler!
+  yard

data/README.md

@@ -0,0 +1,34 @@
+# Sidekiq Recycler
+Gracefully recycle sidekiq processes that use too much memory
+
+sidekiq-recycler is a simple middleware which checks the process's RSS usage on
+the completion of each job. When the usage surpasses a predefined limit, the process
+will gracefully terminate. If any jobs are still running beyond a further time threshold,
+they will be killed and requeued.
+
+
+## Quickstart
+
+```
+$ gem install sidekiq-recycler
+```
+
+```ruby
+# Add the middleware
+require "sidekiq"
+require "sidekiq/recycler"
+
+Sidekiq.configure_server do |config|
+  config.server_middleware do |chain|
+    chain.add Sidekiq::Recycler, :mem_limit => 300_000, :hard_limit_sec => 300
+  end
+end
+```
+
+
+## Configuration
+
+Two options are exposed by the middleware:
+
+ * mem_limit: RSS usage limit, in megabytes
+ * hard_limit_sec: time in seconds to wait for jobs to finish, after graceful shutdown initiated

data/Rakefile

@@ -0,0 +1,33 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+require 'jeweler'
+
+Jeweler::Tasks.new do |gemspec|
+  gemspec.name        = "sidekiq-recycler"
+  gemspec.summary     = "Recycle large sidekiq processes"
+  gemspec.description = "Gracefully recycle sidekiq processes that use too much memory"
+  gemspec.email       = "chetan@pixelcop.net"
+  gemspec.homepage    = "http://github.com/chetan/sidekiq-recycler"
+  gemspec.authors     = ["Chetan Sarva"]
+  gemspec.license     = "MIT"
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+Dir['tasks/**/*.rake'].each { |rake| load rake }
+
+task :default => :test
+
+require 'yard'
+YARD::Rake::YardocTask.new
+
+require "easycov/rake"

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/lib/sidekiq/recycler.rb

@@ -0,0 +1,78 @@
+
+module Sidekiq
+  class Recycler
+
+    # used to avoid race conditions when recycling
+    @@mutex = Mutex.new
+
+    # avoid extra spam after hard limit reached
+    @@recycled = false
+
+    def initialize(opts={})
+      @mem_limit      = opts[:mem_limit] || 300_000 # default is 300mb
+      @hard_limit_sec = opts[:hard_limit_sec] || 300 # default to 300 sec
+    end
+
+    def call(worker, job, queue)
+      begin
+        yield
+
+      ensure
+        # check mem usage here
+        rss = `ps -o rss= -p #{$$}`.to_i
+        if rss > @mem_limit then
+
+          # handle race conditions with many jobs/threads completing
+          # at the same time
+          return if !@@mutex.try_lock or @@recycled
+          @@recycled = true
+
+          Sidekiq.logger.warn "Recycler threshold reached: #{rss} > #{@mem_limit}"
+          Sidekiq.logger.warn "Attempting to stop gracefully"
+
+          # soft_limit_sec = @soft_limit_sec
+          hard_limit_sec = @hard_limit_sec
+          launcher = nil
+
+          Thread.new do
+            Celluloid::Actor.all.each do |actor|
+              # tell sidekiq to exit gracefully
+              # stops accepting new work and kills all waiting ("ready") threads
+              if actor.kind_of? Sidekiq::Launcher then
+                launcher = actor
+                Thread.new do
+                  actor.manager.async.stop
+                end
+              end
+            end
+          end
+
+          Thread.new do
+            # wait until all threads have exited
+            while true do
+              sleep 1
+              next if launcher.nil?
+              if launcher.manager.ready.empty? and launcher.manager.busy.empty? then
+                Sidekiq.logger.info "All threads stopped; exiting now!"
+                exit
+              end
+            end
+          end
+
+          Thread.new do
+            # wait for hard limit sec then kill
+            sleep hard_limit_sec
+            Sidekiq.logger.warn "Hard limit of #{hard_limit_sec}sec reached; sending TERM signal"
+            if !launcher.nil? then
+              launcher.stop
+            else
+              Process.kill("TERM", $$)
+            end
+          end
+
+        end
+      end
+    end
+
+  end
+end

data/sidekiq-recycler.gemspec

@@ -0,0 +1,69 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+# stub: sidekiq-recycler 0.1.0 ruby lib
+
+Gem::Specification.new do |s|
+  s.name = "sidekiq-recycler"
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Chetan Sarva"]
+  s.date = "2013-10-14"
+  s.description = "Gracefully recycle sidekiq processes that use too much memory"
+  s.email = "chetan@pixelcop.net"
+  s.extra_rdoc_files = [
+    "README.md"
+  ]
+  s.files = [
+    "Gemfile",
+    "Gemfile.lock",
+    "README.md",
+    "Rakefile",
+    "VERSION",
+    "lib/sidekiq/recycler.rb",
+    "sidekiq-recycler.gemspec",
+    "test/helper.rb",
+    "test/support/create_jobs.rb",
+    "test/support/sidekiq_mock_fetcher.rb",
+    "test/support/workers.rb",
+    "test/test_recycler.rb"
+  ]
+  s.homepage = "http://github.com/chetan/sidekiq-recycler"
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = "2.1.5"
+  s.summary = "Recycle large sidekiq processes"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 4
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<sidekiq>, [">= 0"])
+      s.add_development_dependency(%q<rake>, [">= 0"])
+      s.add_development_dependency(%q<jeweler>, [">= 0"])
+      s.add_development_dependency(%q<yard>, [">= 0"])
+      s.add_development_dependency(%q<logging>, [">= 0"])
+      s.add_development_dependency(%q<easycov>, [">= 0"])
+      s.add_development_dependency(%q<micron>, [">= 0"])
+    else
+      s.add_dependency(%q<sidekiq>, [">= 0"])
+      s.add_dependency(%q<rake>, [">= 0"])
+      s.add_dependency(%q<jeweler>, [">= 0"])
+      s.add_dependency(%q<yard>, [">= 0"])
+      s.add_dependency(%q<logging>, [">= 0"])
+      s.add_dependency(%q<easycov>, [">= 0"])
+      s.add_dependency(%q<micron>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<sidekiq>, [">= 0"])
+    s.add_dependency(%q<rake>, [">= 0"])
+    s.add_dependency(%q<jeweler>, [">= 0"])
+    s.add_dependency(%q<yard>, [">= 0"])
+    s.add_dependency(%q<logging>, [">= 0"])
+    s.add_dependency(%q<easycov>, [">= 0"])
+    s.add_dependency(%q<micron>, [">= 0"])
+  end
+end
+

data/test/helper.rb

@@ -0,0 +1,22 @@
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+
+require "logging"
+require "micron"
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+EasyCov.path = "coverage"
+EasyCov.filters << EasyCov::IGNORE_GEMS << EasyCov::IGNORE_STDLIB
+EasyCov.filters << lambda { |filename|
+  filename =~ %r(#{EasyCov.root}/test/)
+}
+EasyCov.start

data/test/support/create_jobs.rb

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+require File.expand_path("../workers", __FILE__)
+
+ForeverWorker.perform_async()
+
+10.times do
+  HardWorker.perform_async(Time.now.to_f.to_s)
+end

data/test/support/sidekiq_mock_fetcher.rb

@@ -0,0 +1,91 @@
+
+require "thread"
+require "securerandom"
+
+class MockFetcher
+
+  UnitOfWork = Struct.new(:queue, :message) do
+    def acknowledge
+    end
+    def queue_name
+      "jobs"
+    end
+    def requeue
+    end
+  end
+
+  attr_accessor :reader, :writer
+
+  def self.instance(opts=nil)
+    if @inst.nil? then
+      @inst = self.allocate
+      @inst.setup
+    end
+    @inst
+  end
+
+  class << self
+    alias_method :new, :instance
+  end
+
+  def setup
+    @reader, @writer = IO.pipe
+    @mutex = Mutex.new
+  end
+
+  def retrieve_work
+    @mutex.synchronize {
+      begin
+        return internal_fetch()
+      rescue Exception => ex
+        $stderr.puts ex
+      end
+    }
+  end
+
+  def self.bulk_requeue(*args)
+    # noop
+  end
+
+
+  private
+
+  def internal_fetch
+
+    data = case reader.readline.strip
+    when "forever"
+      {:jid => SecureRandom.hex(4),
+       :class => ForeverWorker.name, :args => []}
+
+    when "hard"
+      {:jid => SecureRandom.hex(4),
+       :class => HardWorker.name, :args => ["test"]}
+    end
+
+    return UnitOfWork.new("foobar", MultiJson.dump(data))
+  end
+
+end
+
+module Sidekiq
+  class Fetcher
+    def self.strategy
+      MockFetcher
+    end
+  end
+end
+
+Sidekiq::Logging.logger = Logging.logger[Sidekiq]
+
+module Sidekiq
+  module Logging
+    def self.with_context(msg)
+      begin
+        ::Logging.mdc["sidekiq"] = msg
+        yield
+      ensure
+        ::Logging.mdc["sidekiq"] = nil
+      end
+    end
+  end
+end

data/test/support/workers.rb

@@ -0,0 +1,40 @@
+
+require "sidekiq"
+require "sidekiq/recycler"
+
+Sidekiq.configure_server do |config|
+  config.server_middleware do |chain|
+    chain.add Sidekiq::Recycler, :mem_limit => 100_000, :hard_limit_sec => 30
+  end
+end
+
+class ForeverWorker
+  @@worked = 0
+  include Sidekiq::Worker
+  def perform()
+    @@worked += 1
+    puts "working forever!"
+    while true
+      sleep 1
+    end
+  end
+
+  def self.times_worked
+    @@worked
+  end
+end
+
+class HardWorker
+  @@worked = 0
+  @@foo = []
+  include Sidekiq::Worker
+  def perform(msg)
+    @@foo << "hello world" * (4**10)
+    puts "perform.."
+    @@worked += 1
+  end
+
+  def self.times_worked
+    @@worked
+  end
+end

data/test/test_recycler.rb

@@ -0,0 +1,62 @@
+
+require "helper"
+
+require 'celluloid'
+require 'sidekiq'
+require 'sidekiq'
+require 'sidekiq/cli'
+require 'sidekiq/launcher'
+require 'support/sidekiq_mock_fetcher'
+require 'support/workers'
+
+require 'micron/test_case/redir_logging'
+
+class TestRecycler < Micron::TestCase
+
+  include Micron::TestCase::RedirLogging
+  @@redir_logger = Logging.logger.root
+
+  def setup
+  end
+
+  def teardown
+    @launcher.stop
+    Celluloid.shutdown
+  end
+
+  def test_recycler
+
+    opts = {
+      :timeout => 0,
+    }
+    @launcher = Sidekiq::Launcher.new(opts)
+    @launcher.run
+
+    # do something..
+    MockFetcher.instance.writer.puts "forever"
+
+    sleep 1
+    assert_equal 1, ForeverWorker.times_worked
+
+    2.times do
+      MockFetcher.instance.writer.puts "hard"
+    end
+    sleep 1
+    assert_equal 2, HardWorker.times_worked
+
+    5.times do
+      MockFetcher.instance.writer.puts "hard"
+    end
+
+    while HardWorker.times_worked < 7 do
+      sleep 0.1
+      Thread.pass
+    end
+
+    puts "over limit?!"
+    sleep 1
+    assert_equal 0, @launcher.manager.ready.size, "waiting threads should have been stopped"
+    assert_equal 1, @launcher.manager.busy.size, "only one thread still busy (sleeping forever)"
+  end
+
+end

metadata

@@ -0,0 +1,154 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq-recycler
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Chetan Sarva
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-10-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logging
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: easycov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: micron
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Gracefully recycle sidekiq processes that use too much memory
+email: chetan@pixelcop.net
+executables: []
+extensions: []
+extra_rdoc_files:
+- README.md
+files:
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- VERSION
+- lib/sidekiq/recycler.rb
+- sidekiq-recycler.gemspec
+- test/helper.rb
+- test/support/create_jobs.rb
+- test/support/sidekiq_mock_fetcher.rb
+- test/support/workers.rb
+- test/test_recycler.rb
+homepage: http://github.com/chetan/sidekiq-recycler
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.1.5
+signing_key: 
+specification_version: 4
+summary: Recycle large sidekiq processes
+test_files: []