resque-pool 0.5.0 → 0.6.0.rc1

data/lib/resque/pool/logging.rb

@@ -3,35 +3,76 @@ module Resque
     module Logging
       extend self
 
-      # more than a little bit complicated...
-      # copied this from Unicorn.
+      # This reopens ALL logfiles in the process that have been rotated
+      # using logrotate(8) (without copytruncate) or similar tools.
+      # A +File+ object is considered for reopening if it is:
+      #   1) opened with the O_APPEND and O_WRONLY flags
+      #   2) the current open file handle does not match its original open path
+      #   3) unbuffered (as far as userspace buffering goes, not O_SYNC)
+      # Returns the number of files reopened
+      #
+      # This was mostly copied from Unicorn 4.8.2 to simplify reopening
+      # logs in the same way that Unicorn does.  Original comments and 
+      # explanations are left intact.
       def self.reopen_logs!
-        log "Flushing logs"
-        [$stdout, $stderr].each do |fd|
-          if fd.instance_of? File
-            # skip if the file is the exact same inode and device
-            orig_st = fd.stat
-            begin
-              cur_st = File.stat(fd.path)
-              next if orig_st.ino == cur_st.ino && orig_st.dev == cur_st.dev
-            rescue Errno::ENOENT
+        to_reopen      = [ ]
+        reopened_count = 0
+
+        ObjectSpace.each_object(File) { |fp| is_log?(fp) and to_reopen << fp }
+        log "Flushing #{to_reopen.length} logs"
+
+        to_reopen.each do |fp|
+          orig_st = begin
+            fp.stat
+          rescue IOError, Errno::EBADF # race
+            next
+          end
+
+          begin
+            b = File.stat(fp.path)
+            # Skip if reopening wouldn't do anything
+            next if orig_st.ino == b.ino && orig_st.dev == b.dev
+          rescue Errno::ENOENT
+          end
+
+          begin
+            # stdin, stdout, stderr are special.  The following dance should
+            # guarantee there is no window where `fp' is unwritable in MRI
+            # (or any correct Ruby implementation).
+            #
+            # Fwiw, GVL has zero bearing here.  This is tricky because of
+            # the unavoidable existence of stdio FILE * pointers for
+            # std{in,out,err} in all programs which may use the standard C library
+            if fp.fileno <= 2
+              # We do not want to hit fclose(3)->dup(2) window for std{in,out,err}
+              # MRI will use freopen(3) here internally on std{in,out,err}
+              fp.reopen(fp.path, "a")
+            else
+              # We should not need this workaround, Ruby can be fixed:
+              #    http://bugs.ruby-lang.org/issues/9036
+              # MRI will not call call fclose(3) or freopen(3) here
+              # since there's no associated std{in,out,err} FILE * pointer
+              # This should atomically use dup3(2) (or dup2(2)) syscall
+              File.open(fp.path, "a") { |tmpfp| fp.reopen(tmpfp) }
             end
-            # match up the encoding
-            open_arg = 'a'
-            if fd.respond_to?(:external_encoding) && enc = fd.external_encoding
-              open_arg << ":#{enc.to_s}"
-              enc = fd.internal_encoding and open_arg << ":#{enc.to_s}"
+
+            fp.sync = true
+            fp.flush # IO#sync=true may not implicitly flush
+            new_st = fp.stat
+
+            # this should only happen in the master:
+            if orig_st.uid != new_st.uid || orig_st.gid != new_st.gid
+              fp.chown(orig_st.uid, orig_st.gid)
             end
-            # match up buffering (does reopen reset this?)
-            sync = fd.sync
-            # sync to disk
-            fd.fsync
-            # reopen, and set ruby buffering appropriately
-            fd.reopen fd.path, open_arg
-            fd.sync = sync
-            log "Reopened logfile: #{fd.path}"
+
+            log "Reopened logfile: #{fp.path}"
+            reopened_count += 1
+          rescue IOError, Errno::EBADF
+            # not much we can do...
           end
         end
+
+        reopened_count
       end
 
       # Given a string, sets the procline ($0)
@@ -43,12 +84,14 @@ module Resque
 
       # TODO: make this use an actual logger
       def log(message)
+        return if $skip_logging
         puts "resque-pool-manager#{app}[#{Process.pid}]: #{message}"
         #$stdout.fsync
       end
 
       # TODO: make this use an actual logger
       def log_worker(message)
+        return if $skip_logging
         puts "resque-pool-worker#{app}[#{Process.pid}]: #{message}"
         #$stdout.fsync
       end
@@ -60,6 +103,20 @@ module Resque
         app_name ? "[#{app_name}]" : ""
       end
 
+      private
+
+      # Used by reopen_logs, borrowed from Unicorn...
+      def self.is_log?(fp)
+        append_flags = File::WRONLY | File::APPEND
+
+        ! fp.closed? &&
+          fp.stat.file? &&
+          fp.sync &&
+          (fp.fcntl(Fcntl::F_GETFL) & append_flags) == append_flags
+        rescue IOError, Errno::EBADF
+          false
+      end
+
     end
   end
 end

data/lib/resque/pool/pooled_worker.rb

@@ -5,13 +5,17 @@ class Resque::Pool
     attr_accessor :pool_master_pid
     attr_accessor :worker_parent_pid
 
+    # We will return false if there are no potential_parent_pids, because that
+    # means we aren't even running inside resque-pool.
+    #
     # We can't just check if we've been re-parented to PID 1 (init) because we
     # want to support docker (which will make the pool master PID 1).
     #
     # We also check the worker_parent_pid, because resque-multi-jobs-fork calls
     # Worker#shutdown? from inside the worker child process.
     def pool_master_has_gone_away?
-      not potential_parent_pids.include?(Process.ppid)
+      pids = potential_parent_pids
+      pids.any? && !pids.include?(Process.ppid)
     end
 
     def potential_parent_pids

data/lib/resque/pool/version.rb

@@ -1,5 +1,5 @@
 module Resque
   class Pool
-    VERSION = "0.5.0"
+    VERSION = "0.6.0.rc1"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: resque-pool
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0.rc1
 platform: ruby
 authors:
 - nicholas a. evans
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-25 00:00:00.000000000 Z
+date: 2015-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
@@ -24,20 +24,6 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.22'
-- !ruby/object:Gem::Dependency
-  name: trollop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '2.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -136,27 +122,18 @@ files:
 - Rakefile
 - LICENSE.txt
 - Changelog.md
-- lib/resque/pool/logging.rb
-- lib/resque/pool/cli.rb
-- lib/resque/pool/version.rb
+- lib/resque/pool.rb
 - lib/resque/pool/tasks.rb
+- lib/resque/pool/cli.rb
 - lib/resque/pool/pooled_worker.rb
-- lib/resque/pool.rb
+- lib/resque/pool/version.rb
+- lib/resque/pool/logging.rb
+- lib/resque/pool/file_or_hash_loader.rb
 - bin/resque-pool
 - man/resque-pool.1.ronn
 - man/resque-pool.1
 - man/resque-pool.yml.5.ronn
 - man/resque-pool.yml.5
-- features/basic_daemon_config.feature
-- features/support/aruba_daemon_support.rb
-- features/support/env.rb
-- features/step_definitions/daemon_steps.rb
-- features/step_definitions/resque-pool_steps.rb
-- spec/resque-pool.yml
-- spec/mock_config.rb
-- spec/resque_pool_spec.rb
-- spec/resque-pool-custom.yml.erb
-- spec/spec_helper.rb
 homepage: http://github.com/nevans/resque-pool
 licenses:
 - MIT
@@ -169,26 +146,16 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - '>='
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
 summary: quickly and easily fork a pool of resque workers
-test_files:
-- spec/mock_config.rb
-- spec/resque_pool_spec.rb
-- spec/spec_helper.rb
-- spec/resque-pool.yml
-- features/support/aruba_daemon_support.rb
-- features/support/env.rb
-- features/step_definitions/daemon_steps.rb
-- features/step_definitions/resque-pool_steps.rb
-- features/basic_daemon_config.feature
-has_rdoc: 
+test_files: []

data/features/basic_daemon_config.feature

@@ -1,68 +0,0 @@
-Feature: Basic resque-pool daemon configuration and operation
-  To easily manage a pool of resque workers, resque-pool provides a daemon with
-  simple configuration.  Static configuration is handled in the
-  config/config.yml file and dynamic configuration is handled in the Rakefile.
-
-  Background:
-    Given a file named "Rakefile" with:
-    """
-    require 'resque/pool/tasks'
-    """
-
-  Scenario: no config file
-    When I run the pool manager as "resque-pool"
-    Then the pool manager should report that it has started up
-    And the pool manager should report that the pool is empty
-    And the pool manager should have no child processes
-    When I send the pool manager the "QUIT" signal
-    Then the pool manager should finish
-    And the pool manager should report that it is finished
-
-  @slow_exit
-  Scenario: basic config file
-    Given a file named "config/resque-pool.yml" with:
-    """
-    foo: 1
-    bar: 2
-    "bar,baz": 3
-    """
-    When I run the pool manager as "resque-pool"
-    Then the pool manager should report that it has started up
-    And the pool manager should report that 6 workers are in the pool
-    And the pool manager should have 1 "foo" worker child processes
-    And the pool manager should have 2 "bar" worker child processes
-    And the pool manager should have 3 "bar,baz" worker child processes
-    When I send the pool manager the "QUIT" signal
-    Then the resque workers should all shutdown
-    And the pool manager should finish
-    And the pool manager should report that a "foo" worker has been reaped
-    And the pool manager should report that a "bar" worker has been reaped
-    And the pool manager should report that a "bar,baz" worker has been reaped
-    And the pool manager should report that it is finished
-
-  Scenario: daemonized
-    Given a directory named "log"
-    And a directory named "tmp/pids"
-    And a file named "config/resque-pool.yml" with:
-    """
-    foo: 2
-    bar: 4
-    "baz,quux": 4
-    """
-    When I run the pool manager as "resque-pool -d"
-    Then the pool manager should record its pid in "tmp/pids/resque-pool.pid"
-    And the pool manager should daemonize
-    And a file named "log/resque-pool.stdout.log" should exist
-    And a file named "log/resque-pool.stderr.log" should exist
-    And the pool manager should log that it has started up
-    And the pool manager should log that 10 workers are in the pool
-    And the pool manager should have 2 "foo" worker child processes
-    And the pool manager should have 4 "bar" worker child processes
-    And the pool manager should have 4 "baz,quux" worker child processes
-    When I send the pool manager the "QUIT" signal
-    Then the resque workers should all shutdown
-    And the pool manager daemon should finish
-    And the pool manager should log that a "foo" worker has been reaped
-    And the pool manager should log that a "bar" worker has been reaped
-    And the pool manager should log that a "baz,quux" worker has been reaped
-    And the pool manager should log that it is finished

data/features/step_definitions/daemon_steps.rb

@@ -1,33 +0,0 @@
-# syntactic sugar, and separate ivar.  daemons aren't interactive
-When /^I run "([^"]*)" in the background$/ do |cmd|
-  run_background(unescape(cmd))
-end
-
-Then /^the (output|logfiles) should contain the following lines \(with interpolated \$PID\):$/ do |output_logfiles, partial_output|
-  interpolate_background_pid(partial_output).split("\n").each do |line|
-    output_or_log(output_logfiles).should include(line)
-  end
-end
-
-When /^I send "([^"]*)" the "([^"]*)" signal$/ do |cmd, signal|
-  send_signal(cmd, signal)
-end
-
-Then /^the "([^"]*)" process should finish$/ do |cmd|
-  # doesn't actually stop... just polls for exit
-  processes[cmd].stop
-end
-
-Before("@slow_exit") do
-  @aruba_timeout_seconds = 10
-end
-
-After do
-  kill_all_processes!
-  # now kill the daemon!
-  begin
-    Process.kill(9, @pid_from_pidfile) if @pid_from_pidfile
-  rescue Errno::ESRCH
-  end
-  #`pkill -9 resque-pool`
-end

data/features/step_definitions/resque-pool_steps.rb

@@ -1,159 +0,0 @@
-def process_should_exist(pid)
-  lambda { Process.kill(0, pid) }.should_not raise_error(Errno::ESRCH)
-end
-
-def process_should_not_exist(pid)
-  lambda { Process.kill(0, pid) }.should raise_error(Errno::ESRCH)
-end
-
-def grab_worker_pids(count, str)
-  puts "TODO: check output_or_log for #{count} worker started messages"
-  pid_regex = (1..count).map { '(\d+)' }.join ', '
-  full_regex = /resque-pool-manager\[aruba\]\[\d+\]: Pool contains worker PIDs: \[#{pid_regex}\]/m
-  str.should =~ full_regex
-  @worker_pids = full_regex.match(str).captures.map {|pid| pid.to_i }
-end
-
-def output_or_logfiles_string(report_log)
-  case report_log
-  when "report", "output"
-    "output"
-  when "log", "logfiles"
-    "logfiles"
-  else
-    raise ArgumentError
-  end
-end
-
-def output_or_log(report_log)
-  case report_log
-  when "report", "output"
-    interactive_output
-  when "log", "logfiles"
-    in_current_dir do
-      File.read("log/resque-pool.stdout.log") << File.read("log/resque-pool.stderr.log")
-    end
-  else
-    raise ArgumentError
-  end
-end
-
-class NotFinishedStarting < StandardError; end
-def worker_processes_for(queues)
-  children_of(background_pid).select do |pid, cmd|
-    raise NotFinishedStarting if cmd =~ /Starting$/
-    cmd =~ /^resque-\d+.\d+.\d+: Waiting for #{queues}$/
-  end
-rescue NotFinishedStarting
-  retry
-end
-
-def children_of(ppid)
-  if RUBY_PLATFORM =~ /darwin/i
-    ps = `ps -eo ppid,pid,comm | grep '^ *#{ppid} '`
-  else
-    ps = `ps -eo ppid,pid,cmd | grep '^ *#{ppid} '`
-  end
-  ps.split(/\s*\n/).map do |line|
-    _, pid, cmd = line.strip.split(/\s+/, 3)
-    [pid, cmd]
-  end
-end
-
-When /^I run the pool manager as "([^"]*)"$/ do |cmd|
-  @pool_manager_process = run_background(unescape(cmd))
-end
-
-When /^I send the pool manager the "([^"]*)" signal$/ do |signal|
-  Process.kill signal, background_pid
-  output_logfiles = @pid_from_pidfile ? "logfiles" : "output"
-  case signal
-  when "QUIT"
-    keep_trying do
-      step "the #{output_logfiles} should contain the following lines (with interpolated $PID):", <<-EOF
-resque-pool-manager[aruba][$PID]: QUIT: graceful shutdown, waiting for children
-      EOF
-    end
-  else
-    raise ArgumentError
-  end
-end
-
-Then /^the pool manager should record its pid in "([^"]*)"$/ do |pidfile|
-  in_current_dir do
-    keep_trying do
-      File.should be_file(pidfile)
-      @pid_from_pidfile = File.read(pidfile).to_i
-      @pid_from_pidfile.should_not == 0
-      process_should_exist(@pid_from_pidfile)
-    end
-  end
-end
-
-Then /^the pool manager should daemonize$/ do
-  stop_processes!
-end
-
-Then /^the pool manager daemon should finish$/ do
-  keep_trying do
-    process_should_not_exist(@pid_from_pidfile)
-  end
-end
-
-# nomenclature: "report" => output to stdout/stderr
-#               "log"    => output to default logfile
-
-Then /^the pool manager should (report|log) that it has started up$/ do |report_log|
-  keep_trying do
-    step "the #{output_or_logfiles_string(report_log)} should contain the following lines (with interpolated $PID):", <<-EOF
-resque-pool-manager[aruba][$PID]: Resque Pool running in test environment
-resque-pool-manager[aruba][$PID]: started manager
-    EOF
-  end
-end
-
-Then /^the pool manager should (report|log) that the pool is empty$/ do |report_log|
-  step "the #{output_or_logfiles_string(report_log)} should contain the following lines (with interpolated $PID):", <<-EOF
-resque-pool-manager[aruba][$PID]: Pool is empty
-  EOF
-end
-
-Then /^the pool manager should (report|log) that (\d+) workers are in the pool$/ do |report_log, count|
-  grab_worker_pids Integer(count), output_or_log(report_log)
-end
-
-Then /^the resque workers should all shutdown$/ do
-  @worker_pids.each do |pid|
-    keep_trying do
-      process_should_not_exist(pid)
-    end
-  end
-end
-
-Then "the pool manager should have no child processes" do
-  children_of(background_pid).should have(:no).keys
-end
-
-Then /^the pool manager should have (\d+) "([^"]*)" worker child processes$/ do |count, queues|
-  worker_processes_for(queues).should have(Integer(count)).members
-end
-
-Then "the pool manager should finish" do
-  # assuming there will not be multiple processes running
-  stop_processes!
-end
-
-Then /^the pool manager should (report|log) that it is finished$/ do |report_log|
-  step "the #{output_or_logfiles_string(report_log)} should contain the following lines (with interpolated $PID):", <<-EOF
-resque-pool-manager[aruba][$PID]: manager finished
-  EOF
-end
-
-Then /^the pool manager should (report|log) that a "([^"]*)" worker has been reaped$/ do |report_log, worker_type|
-  step 'the '+ output_or_logfiles_string(report_log) +' should match /Reaped resque worker\[\d+\] \(status: 0\) queues: '+ worker_type + '/'
-end
-
-Then /^the logfiles should match \/([^\/]*)\/$/ do |partial_output|
-  output_or_log("log").should =~ /#{partial_output}/
-end
-