resque-multi-job-forks 0.4.1 → 0.4.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: fdc96f5c13d05b9537644689803bc6b95a49a232
+  data.tar.gz: 15d6f6dce37c4f2c47c3b202b29ade84efd12aeb
+SHA512:
+  metadata.gz: bfd6fe81a46c5bd9e4cbec35b8b29af8f274bcb31a27340edeb5fb984bc14b094b79709df91d2e09eceb9dd75a6d2c2708f9021bdb81034c23be2916dc83f026
+  data.tar.gz: 6766f1baf6a4866e36ed7096a18d509e4599ad78daf0e29b9dfc94fce46ad87b26431d6f25f4df77a6668284a3926741dfd2a4d94da62ce5bcfc2b90081637d2

data/lib/resque-multi-job-forks.rb

@@ -1,10 +1,13 @@
 require 'resque'
 require 'resque/worker'
+require 'resque/plugins/multi_job_forks/rss_reader'
 
 module Resque
   class Worker
+    include Plugins::MultiJobForks::RssReader
     attr_accessor :seconds_per_fork
     attr_accessor :jobs_per_fork
+    attr_accessor :memory_threshold
     attr_reader   :jobs_processed
 
     def self.multi_jobs_per_fork?
@@ -78,7 +81,7 @@ module Resque
     end
 
     def release_fork
-      log "jobs processed by child: #{jobs_processed}"
+      log "jobs processed by child: #{jobs_processed}; rss: #{rss}"
       run_hook :before_child_exit, self
       Resque.after_fork, Resque.before_fork = *@suppressed_fork_hooks
       @release_fork_limit = @jobs_processed = @cant_fork = nil
@@ -91,7 +94,7 @@ module Resque
     end
 
     def fork_job_limit_reached?
-      fork_job_limit_remaining <= 0 ? true : false
+      fork_job_limit_remaining <= 0 || fork_job_over_memory_threshold?
     end
 
     def fork_job_limit_remaining
@@ -109,6 +112,16 @@ module Resque
     def jobs_per_fork
       @jobs_per_fork ||= ENV['JOBS_PER_FORK'].nil? ? nil : ENV['JOBS_PER_FORK'].to_i
     end
+
+    def fork_job_over_memory_threshold?
+      !!(memory_threshold && rss > memory_threshold)
+    end
+
+    def memory_threshold
+      @memory_threshold ||= ENV["RESQUE_MEM_THRESHOLD"].to_i
+      @memory_threshold > 0 && @memory_threshold
+    end
+
   end
 
   # the `before_child_exit` hook will run in the child process

data/lib/resque/plugins/multi_job_forks/rss_reader.rb

@@ -0,0 +1,44 @@
+module Resque
+  module Plugins
+    module MultiJobForks
+      module RssReader
+        extend self
+        LINUX = RbConfig::CONFIG['host_os'].start_with?('linux')
+        PS_CMD = "ps -o rss= -p %d".freeze
+        UNITS  = {
+          b:  1024**-1,
+          kb: 1024**0,
+          mb: 1024**1,
+          gb: 1024**2,
+          tb: 1024**3,
+        }.freeze
+
+        # Returns RSS in Kb; should work on Linux and Mac OS X
+        def rss(pid=Process.pid)
+          LINUX ? rss_linux(pid) : rss_posix(pid)
+        end
+
+        # Fork/exec ps and parse result.
+        # Should work on any system with POSIX ps.
+        def rss_posix(pid=Process.pid)
+          `#{PS_CMD % [pid]}`.to_i
+        end
+
+        # Read from /proc/$pid/status.  Linux only.
+        # MUCH faster and doesn't incur significant memory cost.
+        def rss_linux(pid=Process.pid)
+          File.open("/proc/#{pid}/status") do |file|
+            file.each_line do |line|
+              if line[/^VmRSS/]
+                _,c,u = line.split
+                return (c.to_i * UNITS[u.downcase.to_sym]).to_i
+              end
+            end
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/test/test_rss_reader.rb

@@ -0,0 +1,34 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '/helper')
+
+class TestRssReader < Test::Unit::TestCase
+
+  def setup
+    @object = Object.new
+    @object.extend Resque::Plugins::MultiJobForks::RssReader
+  end
+
+  def test_current_process_rss
+    rss = @object.rss
+    # not a very strict test, but have you ever seen a ruby process < 1Mb?
+    # the "real" test is manual verification via top/ps/htop
+    assert_equal Fixnum, rss.class
+    assert @object.rss > 1000
+  end
+
+  def test_rss_other_processes
+    rss = @object.rss(1) # init is guaranteed to exist
+    assert_equal Fixnum, rss.class
+    assert @object.rss > 1000
+  end
+
+  def test_rss_posix
+    assert @object.rss_posix > 1000
+  end
+
+  if Resque::Plugins::MultiJobForks::RssReader::LINUX
+    def test_rss_linux
+      assert @object.rss_linux > 1000
+    end
+  end
+
+end

metadata

@@ -1,8 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: resque-multi-job-forks
 version: !ruby/object:Gem::Version
-  version: 0.4.1
-  prerelease: 
+  version: 0.4.2
 platform: ruby
 authors:
 - Mick Staugaard
@@ -11,70 +10,62 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-07 00:00:00.000000000 Z
+date: 2014-04-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.24'
+        version: '1.22'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.24'
+        version: '1.22'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: When your resque jobs are frequent and fast, the overhead of forking
@@ -88,38 +79,34 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/resque-multi-job-forks.rb
+- lib/resque/plugins/multi_job_forks/rss_reader.rb
 - test/helper.rb
 - test/test_resque-multi-job-forks.rb
+- test/test_rss_reader.rb
 homepage: http://github.com/staugaard/resque-multi-job-forks
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 4214525788567926268
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 4214525788567926268
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.25
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Have your resque workers process more that one job
 test_files:
 - test/helper.rb
 - test/test_resque-multi-job-forks.rb
+- test/test_rss_reader.rb