resque-pool 0.6.0 → 0.7.0

data/lib/resque/pool/cli.rb

@@ -12,6 +12,7 @@ module Resque
 
       def run
         opts = parse_options
+        obtain_shared_lock opts[:lock_file]
         daemonize if opts[:daemon]
         manage_pidfile opts[:pidfile]
         redirect opts
@@ -20,9 +21,9 @@ module Resque
         start_pool
       end
 
-      def parse_options
+      def parse_options(argv=nil)
         opts = {}
-        OptionParser.new do |opt|
+        parser = OptionParser.new do |opt|
           opt.banner = <<-EOS.gsub(/^            /, '')
             resque-pool is the best way to manage a group (pool) of resque workers
 
@@ -36,11 +37,31 @@ module Resque
           EOS
           opt.on('-c', '--config PATH', "Alternate path to config file") { |c| opts[:config] = c }
           opt.on('-a', '--appname NAME', "Alternate appname") { |c| opts[:appname] = c }
-          opt.on("-d", '--daemon', "Run as a background daemon") { opts[:daemon] = true }
+          opt.on("-d", '--daemon', "Run as a background daemon") {
+            opts[:daemon] = true
+            opts[:stdout]  ||= "log/resque-pool.stdout.log"
+            opts[:stderr]  ||= "log/resque-pool.stderr.log"
+            opts[:pidfile] ||= "tmp/pids/resque-pool.pid" unless opts[:no_pidfile]
+          }
+          opt.on("-k", '--kill-others', "Shutdown any other Resque Pools on startup") { opts[:killothers] = true }
           opt.on('-o', '--stdout FILE', "Redirect stdout to logfile") { |c| opts[:stdout] = c }
           opt.on('-e', '--stderr FILE', "Redirect stderr to logfile") { |c| opts[:stderr] = c }
           opt.on('--nosync', "Don't sync logfiles on every write") { opts[:nosync] = true }
-          opt.on("-p", '--pidfile FILE', "PID file location") { |c| opts[:pidfile] = c }
+          opt.on("-p", '--pidfile FILE', "PID file location") { |c|
+            opts[:pidfile] = c
+            opts[:no_pidfile] = false
+          }
+          opt.on('--no-pidfile', "Force no pidfile, even if daemonized") {
+            opts[:pidfile] = nil
+            opts[:no_pidfile] = true
+          }
+          opt.on('-l', '--lock FILE' "Open a shared lock on a file") { |c| opts[:lock_file] = c }
+          opt.on("-H", "--hot-swap", "Set appropriate defaults to hot-swap a new pool for a running pool") {|c|
+            opts[:pidfile] = nil
+            opts[:no_pidfile] = true
+            opts[:lock_file] ||= "tmp/resque-pool.lock"
+            opts[:killothers] = true
+          }
           opt.on("-E", '--environment ENVIRONMENT', "Set RAILS_ENV/RACK_ENV/RESQUE_ENV") { |c| opts[:environment] = c }
           opt.on("-s", '--spawn-delay MS', Integer, "Delay in milliseconds between spawning missing workers") { |c| opts[:spawn_delay] = c }
           opt.on('--term-graceful-wait', "On TERM signal, wait for workers to shut down gracefully") { opts[:term_graceful_wait] = true }
@@ -49,12 +70,9 @@ module Resque
           opt.on('--single-process-group', "Workers remain in the same process group as the master") { opts[:single_process_group] = true }
           opt.on("-h", "--help", "Show this.") { puts opt; exit }
           opt.on("-v", "--version", "Show Version"){ puts "resque-pool #{VERSION} (c) nicholas a. evans"; exit}
-        end.parse!
-        if opts[:daemon]
-          opts[:stdout]  ||= "log/resque-pool.stdout.log"
-          opts[:stderr]  ||= "log/resque-pool.stderr.log"
-          opts[:pidfile] ||= "tmp/pids/resque-pool.pid"
         end
+        parser.parse!(argv || parser.default_argv)
+
         opts
       end
 
@@ -66,6 +84,18 @@ module Resque
         exit unless pid.nil?
       end
 
+      # Obtain a lock on a file that will be held for the lifetime of
+      # the process.  This aids in concurrent daemonized deployment with
+      # process managers like upstart since multiple pools can share a
+      # lock, but not a pidfile.
+      def obtain_shared_lock(lock_path)
+        return unless lock_path
+        @lock_file = File.open(lock_path, 'w')
+        unless @lock_file.flock(File::LOCK_SH)
+          fail "unable to obtain shared lock on #{@lock_file}"
+        end
+      end
+
       def manage_pidfile(pidfile)
         return unless pidfile
         pid = Process.pid
@@ -90,8 +120,7 @@ module Resque
 
       def process_still_running?(pidfile)
         old_pid = open(pidfile).read.strip.to_i
-        Process.kill 0, old_pid
-        true
+        old_pid > 0 && Process.kill(0, old_pid)
       rescue Errno::ESRCH
         false
       rescue Errno::EPERM
@@ -130,6 +159,7 @@ module Resque
         if opts[:spawn_delay]
           Resque::Pool.spawn_delay = opts[:spawn_delay] * 0.001
         end
+        Resque::Pool.kill_other_pools = !!opts[:killothers]
       end
 
       def setup_environment(opts)
@@ -142,7 +172,14 @@ module Resque
 
       def start_pool
         require 'rake'
+        self.const_set :RakeApp, Class.new(Rake::Application) {
+          def default_task_name # :nodoc:
+            "resque:pool"
+          end
+        }
+        Rake.application = RakeApp.new
         require 'resque/pool/tasks'
+
         Rake.application.init
         Rake.application.load_rakefile
         Rake.application["resque:pool"].invoke

data/lib/resque/pool/config_loaders/file_or_hash_loader.rb

@@ -0,0 +1,63 @@
+module Resque
+  class Pool
+    module ConfigLoaders
+
+      class FileOrHashLoader
+        def initialize(filename_or_hash=nil)
+          case filename_or_hash
+          when String, nil
+            @filename = filename_or_hash
+          when Hash
+            @static_config = filename_or_hash.dup
+          else
+            raise "#{self.class} cannot be initialized with #{filename_or_hash.inspect}"
+          end
+        end
+
+        def call(environment)
+          @config ||= load_config_from_file(environment)
+        end
+
+        def reset!
+          @config = nil
+        end
+
+        private
+
+        def load_config_from_file(environment)
+          if @static_config
+            new_config = @static_config
+          else
+            filename = config_filename
+            new_config = load_config filename
+          end
+          apply_environment new_config, environment
+        end
+
+        def apply_environment(config, environment)
+          environment and config[environment] and config.merge!(config[environment])
+          config.delete_if {|key, value| value.is_a? Hash }
+        end
+
+        def config_filename
+          @filename || choose_config_file
+        end
+
+        def load_config(filename)
+          return {} unless filename
+          YAML.load(ERB.new(IO.read(filename)).result)
+        end
+
+        CONFIG_FILES = ["resque-pool.yml", "config/resque-pool.yml"]
+        def choose_config_file
+          if ENV["RESQUE_POOL_CONFIG"]
+            ENV["RESQUE_POOL_CONFIG"]
+          else
+            CONFIG_FILES.detect { |f| File.exist?(f) }
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/resque/pool/config_loaders/throttled.rb

@@ -0,0 +1,44 @@
+require "delegate"
+
+module Resque
+  class Pool
+
+    module ConfigLoaders
+
+      # Throttle the frequency of loading pool configuration
+      # Defaults to call only once per 10 seconds.
+      class Throttled < SimpleDelegator
+
+        def initialize(config_loader, period: 10, time_source: Time)
+          super(config_loader)
+          @period = period
+          @resettable = config_loader.respond_to?(:reset!)
+          @last_check = 0
+          @time_source = time_source
+        end
+
+        def call(env)
+          # We do not need to cache per `env`, since the value of `env` will not
+          # change during the life of the process.
+          if (now > @last_check + @period)
+            @cache = super
+            @last_check = now
+          end
+          @cache
+        end
+
+        def reset!
+          @last_check = 0
+          super if @resettable
+        end
+
+        private
+
+        def now
+          @time_source.now.to_f
+        end
+      end
+
+    end
+  end
+end

data/lib/resque/pool/killer.rb

@@ -0,0 +1,40 @@
+module Resque
+  class Pool
+    class Killer
+      include Logging
+
+      GRACEFUL_SHUTDOWN_SIGNAL=:INT
+
+      def self.run
+        new.run
+      end
+
+      def run
+        my_pid = Process.pid
+        pool_pids = all_resque_pool_processes
+        pids_to_kill = pool_pids.reject{|pid| pid == my_pid}
+        pids_to_kill.each do |pid|
+          log "Pool (#{my_pid}) in kill-others mode: killing pool with pid (#{pid})"
+          Process.kill(GRACEFUL_SHUTDOWN_SIGNAL, pid)
+        end
+      end
+
+
+      def all_resque_pool_processes
+        out = `ps -e -o pid= -o command= 2>&1`
+        raise "Unable to identify other pools: #{out}" unless $?.success?
+        parse_pids_from_output out
+      end
+
+      RESQUE_POOL_PIDS = /
+        ^\s*(\d+)                         # PID digits, optional leading spaces
+        \s+                               # column divider
+        #{Regexp.escape(PROCLINE_PREFIX)} # exact match at start of command
+      /x
+
+      def parse_pids_from_output(output)
+        output.scan(RESQUE_POOL_PIDS).flatten.map(&:to_i)
+      end
+    end
+  end
+end

data/lib/resque/pool/logging.rb

@@ -75,11 +75,13 @@ module Resque
         reopened_count
       end
 
+      PROCLINE_PREFIX="resque-pool-master"
+
       # Given a string, sets the procline ($0)
       # Procline is always in the format of:
       #   resque-pool-master: STRING
       def procline(string)
-        $0 = "resque-pool-master#{app}: #{string}"
+        $0 = "#{PROCLINE_PREFIX}#{app}: #{string}"
       end
 
       # TODO: make this use an actual logger

data/lib/resque/pool/version.rb

@@ -1,5 +1,5 @@
 module Resque
   class Pool
-    VERSION = "0.6.0"
+    VERSION = "0.7.0"
   end
 end

data/resque-pool.gemspec

@@ -0,0 +1,42 @@
+require_relative "lib/resque/pool/version"
+
+Gem::Specification.new do |spec|
+  spec.name        = "resque-pool"
+  spec.version     = Resque::Pool::VERSION
+  spec.authors     = ["nicholas a. evans",]
+  spec.email       = ["nick@ekenosen.net"]
+
+  spec.summary     = "quickly and easily fork a pool of resque workers"
+  spec.description = <<-EOF
+    quickly and easily fork a pool of resque workers,
+    saving memory (w/REE) and monitoring their uptime
+  EOF
+  spec.homepage    = "http://github.com/nevans/resque-pool"
+  spec.license     = 'MIT'
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "http://github.com/nevans/resque-pool"
+  spec.metadata["changelog_uri"] = "https://github.com/resque/resque/blob/master/HISTORY.md"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.required_ruby_version = '>= 2.0'
+
+  spec.add_dependency "resque", ">= 1.22", "< 3"
+  spec.add_dependency "rake",   ">= 10.0", "< 13.0"
+
+  spec.add_development_dependency "bundler",  "~> 2.0"
+  spec.add_development_dependency "rspec",    "~> 3.8"
+  spec.add_development_dependency "cucumber", "~> 3.0"
+  spec.add_development_dependency "aruba",    "~> 0.14.0"
+  spec.add_development_dependency "ronn"
+  spec.add_development_dependency "mustache"
+
+end

metadata

@@ -1,111 +1,137 @@
 --- !ruby/object:Gem::Specification
 name: resque-pool
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - nicholas a. evans
 autorequire: 
-bindir: bin
+bindir: exe
 cert_chain: []
-date: 2015-10-14 00:00:00.000000000 Z
+date: 2019-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.22'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.22'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '13.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.10'
+        version: '3.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.10'
+        version: '3.8'
 - !ruby/object:Gem::Dependency
   name: cucumber
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: aruba
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.11
+        version: 0.14.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.11
+        version: 0.14.0
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: ronn
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: ronn
+  name: mustache
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: |2
@@ -118,43 +144,70 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitignore"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- CONTRIBUTING.md
+- Changelog.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
 - README.md
 - Rakefile
-- LICENSE.txt
-- Changelog.md
+- config/alternate.yml
+- config/cucumber.yml
+- examples/Gemfile
+- examples/Gemfile.lock
+- examples/Rakefile
+- examples/chef_cookbook/recipes/default.rb
+- examples/chef_cookbook/templates/default/initd.erb
+- examples/chef_cookbook/templates/default/monitrc.erb
+- examples/log/.gitignore
+- examples/rails-resque.rake
+- examples/resque-pool.god
+- examples/resque-pool.yml
+- examples/tmp/pids/.gitignore
+- examples/upstart-reload.conf
+- examples/upstart.conf
+- exe/resque-pool
 - lib/resque/pool.rb
-- lib/resque/pool/tasks.rb
 - lib/resque/pool/cli.rb
+- lib/resque/pool/config_loaders/file_or_hash_loader.rb
+- lib/resque/pool/config_loaders/throttled.rb
+- lib/resque/pool/killer.rb
+- lib/resque/pool/logging.rb
 - lib/resque/pool/pooled_worker.rb
+- lib/resque/pool/tasks.rb
 - lib/resque/pool/version.rb
-- lib/resque/pool/logging.rb
-- lib/resque/pool/file_or_hash_loader.rb
-- bin/resque-pool
-- man/resque-pool.1.ronn
 - man/resque-pool.1
-- man/resque-pool.yml.5.ronn
+- man/resque-pool.1.ronn
 - man/resque-pool.yml.5
+- man/resque-pool.yml.5.ronn
+- resque-pool.gemspec
 homepage: http://github.com/nevans/resque-pool
 licenses:
 - MIT
-metadata: {}
+metadata:
+  homepage_uri: http://github.com/nevans/resque-pool
+  source_code_uri: http://github.com/nevans/resque-pool
+  changelog_uri: https://github.com/resque/resque/blob/master/HISTORY.md
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.9.3
+      version: '2.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.6.14.3
 signing_key: 
 specification_version: 4
 summary: quickly and easily fork a pool of resque workers