sidekiq_memlimit 0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e4f49d07f4bedb1eb8dfa509c7d1bcc8b6327a78
+  data.tar.gz: 600eb7890bde5487893f5206c628c30fab32b67a
+SHA512:
+  metadata.gz: 1156ff65b00c614403d1ff708ecb3dd38571c272444fb5a2ef1fc220beddd13434bf2239b5b2a236156e530e3f6dd3ca9de672dec5b1a883a5a39a79a16d9710
+  data.tar.gz: 67e227ea0d6ebd3ae27d76b56b327141030c1727c68cdc1487b45b891f3d2d127cd53691ecb985ecaecd97b5960a2884b2fbd02569f268e08fb3c69f3ad75c55

data/CHANGELOG

@@ -0,0 +1 @@
+v0.1. first version

data/Manifest

@@ -0,0 +1,6 @@
+CHANGELOG
+Manifest
+README.md
+Rakefile
+lib/sidekiq_memlimit.rb
+test/sidekiq_memlimit.rb

data/README.md

@@ -0,0 +1,9 @@
+## Sidekiq memory limit
+
+This ruby gem gracefully kills Sidekiq worker processes (with USR1 signal)
+when the RSS memory exceeds the limit set in the SIDEKIQ_MAX_MB environment
+variable.
+
+The memory limit is checked once per 5 seconds in a background thread and
+this is only run when inside a Sidekiq worker.
+

data/Rakefile

@@ -0,0 +1,9 @@
+require 'echoe'
+
+Echoe.new("sidekiq_memlimit") do |p|
+  p.author = "Andrew Snow"
+  p.email = 'andrew@modulus.org'
+  p.summary = "Restart Sidekiq gracefully when exceeding preset memory limit"
+  p.url = "https://github.com/andys/sidekiq_memlimit"
+  p.runtime_dependencies = ['sidekiq >=3.0']
+end

data/lib/sidekiq_memlimit.rb

@@ -0,0 +1,56 @@
+class SidekiqMemlimit
+  class << self
+    attr_accessor :max_mb, :sleep_time
+    attr_reader :monitorthread
+
+    def start_monitorthread
+      if !@monitorthread || !@monitorthread.alive?
+        @monitorthread = Thread.new do
+          begin
+            loop do
+              mb = rss_mb
+              if max_mb && mb > max_mb
+                run_gc
+                if max_mb && mb > max_mb
+                  Sidekiq.logger.error "#{self}: Exceeded max memory limit (#{mb} > #{max_mb} MB)"
+                  Process.kill('USR1', $$)
+                end
+              end
+              sleep sleep_time
+            end
+          rescue Exception => e
+            Sidekiq.logger.error "#{self}: #{$!.class} exception: #{$!}"
+          end
+        end
+      end
+    end
+
+    def run_gc
+      GC.start
+    end
+
+    def pagesize
+      4096
+    end
+
+    def rss_mb
+      rss_kb >> 10
+    end
+
+    def rss_kb
+      pagesize * (File.read("/proc/#{$$}/statm").split(' ')[1].to_i rescue 0) >> 10
+    end
+
+    def setup
+      self.sleep_time ||= 5
+      if ENV['SIDEKIQ_MAX_MB']
+        self.max_mb = ENV['SIDEKIQ_MAX_MB'].to_i
+        self.max_mb = nil unless max_mb > 1
+      end
+      self.start_monitorthread
+    end
+  end
+
+end
+
+SidekiqMemlimit.setup if defined?(Sidekiq::CLI)

data/sidekiq_memlimit.gemspec

@@ -0,0 +1,33 @@
+# -*- encoding: utf-8 -*-
+# stub: sidekiq_memlimit 0.1 ruby lib
+
+Gem::Specification.new do |s|
+  s.name = "sidekiq_memlimit"
+  s.version = "0.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
+  s.require_paths = ["lib"]
+  s.authors = ["Andrew Snow"]
+  s.date = "2014-08-07"
+  s.description = "Restart Sidekiq gracefully when exceeding preset memory limit"
+  s.email = "andrew@modulus.org"
+  s.extra_rdoc_files = ["CHANGELOG", "README.md", "lib/sidekiq_memlimit.rb"]
+  s.files = ["CHANGELOG", "Manifest", "README.md", "Rakefile", "lib/sidekiq_memlimit.rb", "sidekiq_memlimit.gemspec", "test/sidekiq_memlimit.rb"]
+  s.homepage = "https://github.com/andys/sidekiq_memlimit"
+  s.rdoc_options = ["--line-numbers", "--title", "Sidekiq_memlimit", "--main", "README.md"]
+  s.rubyforge_project = "sidekiq_memlimit"
+  s.rubygems_version = "2.3.0"
+  s.summary = "Restart Sidekiq gracefully when exceeding preset memory limit"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 4
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<sidekiq>, [">= 3.0"])
+    else
+      s.add_dependency(%q<sidekiq>, [">= 3.0"])
+    end
+  else
+    s.add_dependency(%q<sidekiq>, [">= 3.0"])
+  end
+end

data/test/sidekiq_memlimit.rb

@@ -0,0 +1,31 @@
+ENV['SIDEKIQ_MAX_MB'] = '100'
+require "minitest/autorun"
+require 'sidekiq/cli'
+require "#{File.dirname(__FILE__)}/../lib/sidekiq_memlimit"
+
+class TestSidekiqMemlimit < MiniTest::Unit::TestCase
+  def test_rss_kb
+    SidekiqMemlimit.run_gc
+    kb = SidekiqMemlimit.rss_kb
+    assert(kb > 0)
+    assert_equal "VmRSS:\t%8d kB\n" % kb,
+                 File.read("/proc/#{$$}/status").lines.grep(/VmRSS/).first
+  end
+
+  def test_monitorthread
+    assert SidekiqMemlimit.monitorthread
+  end
+  
+  def test_signal
+    received_signal = false
+    trap("USR1") { received_signal = true }
+    Sidekiq.logger = nil
+    assert_equal false, received_signal
+    
+    # now take up >100MB memory
+    x = 'a' * (101 * 1024 * 1024)
+    
+    sleep 1.1
+    assert_equal true, received_signal
+  end
+end

metadata

@@ -0,0 +1,71 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq_memlimit
+version: !ruby/object:Gem::Version
+  version: '0.1'
+platform: ruby
+authors:
+- Andrew Snow
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-08-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: Restart Sidekiq gracefully when exceeding preset memory limit
+email: andrew@modulus.org
+executables: []
+extensions: []
+extra_rdoc_files:
+- CHANGELOG
+- README.md
+- lib/sidekiq_memlimit.rb
+files:
+- CHANGELOG
+- Manifest
+- README.md
+- Rakefile
+- lib/sidekiq_memlimit.rb
+- sidekiq_memlimit.gemspec
+- test/sidekiq_memlimit.rb
+homepage: https://github.com/andys/sidekiq_memlimit
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options:
+- --line-numbers
+- --title
+- Sidekiq_memlimit
+- --main
+- README.md
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '1.2'
+requirements: []
+rubyforge_project: sidekiq_memlimit
+rubygems_version: 2.3.0
+signing_key: 
+specification_version: 4
+summary: Restart Sidekiq gracefully when exceeding preset memory limit
+test_files: []