pipemaster 0.4.2 → 0.5.0

data/CHANGELOG

@@ -1,3 +1,23 @@
+0.5.0 (2010-03-01)
+This release adds background processes.  The Pipefile can specify multiple
+background activities.  Pipemaster starts each activity by forking a new child
+process (with the same environment as all other workers).
+
+When Pipemaster terminates, it terminates all child processes (QUIT or TERM,
+depending on signal).  In addition, when reloading the configuration (HUP),
+Pipemaster asks all background processes to terminate gracefully and restarts
+new background processes based on the new configuration.
+
+To define a background process:
+
+background :resque do
+  resque = Resque::Worker.new(:tasks)
+	resque.work(5) # interval, will block
+end
+
+* Added: Background processes.
+* Changed: Configuration method app becomes setup.  Same semantics, just better name.
+
 0.4.2 (2010-02-19)
 * Fix: Pipe command exits with retcode on successful completion.
 

data/Gemfile

@@ -1,4 +1,3 @@
-source "http://gemcutter.org"
-gem "gemcutter"
+source "http://rubygems.org/"
 gem "rake"
 gem "yard"

data/lib/pipemaster/configurator.rb

@@ -21,6 +21,7 @@ module Pipemaster
           server.logger.info("forked child re-executing...")
         },
       :pid => nil,
+      :background => {},
       :commands => {}
     }
 
@@ -116,10 +117,10 @@ module Pipemaster
       Server::START_CTX[:cwd] = ENV["PWD"] = path
     end
 
-    # Application is a block we run at startup.  This is the heavy stuff (e.g.
-    # starting up ActiveRecord) we want to run once and fork from.
-    def app(*args, &block)
-      set_hook(:app, block_given? ? block : args[0], 0)
+    # Setup block runs on startup.  Put all the heavy stuff here (e.g. loading
+    # libraries, initializing state from database).
+    def setup(*args, &block)
+      set_hook(:setup, block_given? ? block : args[0], 0)
     end
 
     # Sets after_fork hook to a given block.  This block will be called by
@@ -144,6 +145,25 @@ module Pipemaster
       set_hook(:before_exec, block_given? ? block : args[0], 1)
     end
 
+    # Background process: the block is executed in a child process.  The master
+    # will tell the child process to stop/terminate using appropriate signals, 
+    # restart the process after a successful upgrade.  Block accepts two
+    # arguments: server and worker.
+    def background(name_or_hash, a_proc = nil, &block)
+      set[:background] = {} if set[:background] == :unset
+      if Hash === name_or_hash
+        name_or_hash.each_pair do |name, a_proc|
+          background name, a_proc
+        end
+      else
+        name = name_or_hash.to_sym
+        a_proc ||= block
+        arity = a_proc.arity
+        (arity == 0 || arity < 0) or raise ArgumentError, "background #{name}#{a_proc.inspect} has invalid arity: #{arity} (need 0)"
+        set[:background][name] = a_proc
+      end
+    end
+
     # Sets listeners to the given +addresses+, replacing or augmenting the
     # current set.  This is for internal API use only, do not use it in your
     # Pipemaster config file.  Use listen instead.

data/lib/pipemaster/server.rb

@@ -7,9 +7,11 @@ require "pipemaster/socket_helper"
 module Pipemaster
 
   class Server < Struct.new(:listener_opts, :timeout, :logger,
-                            :app, :before_fork, :after_fork, :before_exec,
-                            :pid, :reexec_pid, :init_listeners,
-                            :master_pid, :config, :ready_pipe)
+                            :setup, :before_fork, :after_fork, :before_exec,
+                            :background, :commands,
+                            :pid, :reexec_pid, :master_pid,
+                            :config, :ready_pipe, :init_listeners)
+                            
 
     include SocketHelper
 
@@ -22,6 +24,9 @@ module Pipemaster
     # This hash maps PIDs to Workers
     WORKERS = {}
 
+    # Background workers.
+    BACKGROUND = []
+
     # We populate this at startup so we can figure out how to reexecute
     # and upgrade the currently running instance of Pipemaster
     # This Hash is considered a stable interface and changing its contents
@@ -75,7 +80,6 @@ module Pipemaster
       config.commit!(self, :skip => [:listeners, :pid])
     end
 
-    attr_accessor :commands
 
     def start
       self.master_pid = $$
@@ -88,12 +92,12 @@ module Pipemaster
         Pipemaster::Util.reopen_logs
         logger.info "master done reopening logs"
       end
-      trap(:HUP)  { reloaded = true ; load_config! }
+      trap(:HUP)  { reloaded = true ; load_config! ; restart_background }
       trap(:USR2) { reexec }
 
       proc_name "pipemaster"
-      logger.info "loading application"
-      app.call if app
+      logger.info "running setup"
+      setup.call if setup
 
       logger.info "master process ready" # test_exec.rb relies on this message
       if ready_pipe
@@ -110,9 +114,9 @@ module Pipemaster
       end
       config_listeners.each { |addr| listen(addr) }
 
-      reloaded = nil
       begin
         reloaded = false
+        restart_background
         while selected = Kernel.select(LISTENERS)
           selected.first.each do |socket|
             client = socket.accept_nonblock
@@ -295,6 +299,7 @@ module Pipemaster
         Process.kill(signal, wpid)
       rescue Errno::ESRCH
         worker = WORKERS.delete(wpid)
+        BACKGROUND.delete(wpid)
       end
     end
 
@@ -309,7 +314,8 @@ module Pipemaster
             self.pid = pid.chomp('.oldbin') if pid
             proc_name 'master'
           else
-            worker = WORKERS.delete(wpid) rescue nil
+            WORKERS.delete(wpid) rescue nil
+            BACKGROUND.delete(wpid)
             logger.info "reaped #{status.inspect} "
           end
         end
@@ -414,10 +420,46 @@ module Pipemaster
       ensure
         socket.close_write
         socket.close
-        exit
+        exit!
+      end
+    end
+
+    def restart_background
+      # Gracefully shut down all backgroud processes.
+      BACKGROUND.delete_if { |wpid| Process.kill(:QUIT, wpid) rescue true }
+      # Start them again.
+      background.each do |name, block|
+        worker = Worker.new
+        before_fork.call self, worker
+        pid = fork { run_in_background name, worker, &block }
+        BACKGROUND << pid
+        WORKERS[pid] = worker
       end
     end
 
+    def run_in_background(name, worker, &block)
+      trap(:QUIT) { exit }
+      [:TERM, :INT].each { |sig| trap(sig) { exit! } }
+      [:USR1, :USR2].each { |sig| trap(sig, nil) }
+      trap(:CHLD, 'DEFAULT')
+
+      WORKERS.clear
+      LISTENERS.each { |sock| sock.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC) }
+      after_fork.call self, worker
+      proc_name "pipemaster: #{name}"
+      logger.info "#{Process.pid} background worker #{name}"
+      block.call
+      logger.info "#{Process.pid} finished worker #{name}"
+    rescue SystemExit => ex
+      logger.info "#{Process.pid} finished worker #{name}"
+    rescue =>ex
+      logger.info "#{Process.pid} failed: #{ex.message}" 
+      socket.write "#{ex.class.name}: #{ex.message}\n"
+      socket.write 127.chr
+    ensure
+      exit!
+    end
+
   end
 end
 

data/pipemaster.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name           = "pipemaster"
-  spec.version        = "0.4.2"
+  spec.version        = "0.5.0"
   spec.author         = "Assaf Arkin"
   spec.email          = "assaf@labnotes.org"
   spec.homepage       = "http://github.com/assaf/pipemaster"

data/test/test_helper.rb

@@ -41,7 +41,7 @@ def redirect_test_io
   orig_err = STDERR.dup
   orig_out = STDOUT.dup
   STDERR.reopen("test_stderr.#{$$}.log", "a")
-  STDOUT.reopen("test_stdout.#{$$}.log", "a")
+  #STDOUT.reopen("test_stdout.#{$$}.log", "a")
   STDERR.sync = STDOUT.sync = true
 
   at_exit do

data/test/unit/test_configurator.rb

@@ -5,7 +5,7 @@ require 'tempfile'
 require 'pipemaster'
 
 TestStruct = Struct.new(
-  *(Pipemaster::Configurator::DEFAULTS.keys + %w(listener_opts listeners)))
+  *(Pipemaster::Configurator::DEFAULTS.keys + %w(listener_opts listeners background commands)))
 class TestConfigurator < Test::Unit::TestCase
 
   def test_config_init
@@ -147,5 +147,20 @@ class TestConfigurator < Test::Unit::TestCase
     end
   end
 
-end
+  def test_background_proc
+    test_struct = TestStruct.new
+    [ proc { }, Proc.new { |*a| }, lambda { |*a| } ].each do |my_proc|
+      Pipemaster::Configurator.new(:background => { :foo => my_proc }).commit!(test_struct)
+      assert_equal my_proc, test_struct.background[:foo]
+    end
+  end
+
+  def test_background_wrong_arity
+    [ proc { |a| }, Proc.new { |a,b| }, lambda { |a,b,c| } ].each do |my_proc|
+      assert_raises(ArgumentError) do
+        Pipemaster::Configurator.new(:background => { :foo => my_proc })
+      end
+    end
+  end
 
+end

data/test/unit/test_server.rb

@@ -13,7 +13,7 @@ class ServerTest < Test::Unit::TestCase
       redirect_test_io do
         wait_master_ready("test_stderr.#$$.log")
         File.truncate("test_stderr.#$$.log", 0)
-        Process.kill 9, @pid
+        Process.kill :QUIT, @pid
       end
     end
   end
@@ -22,7 +22,7 @@ class ServerTest < Test::Unit::TestCase
     redirect_test_io do
       @server = Pipemaster::Server.new({:listeners => [ "127.0.0.1:#{@port}" ]}.merge(options || {}))
       @pid = fork { @server.start }
-      at_exit { Process.kill @pid, 9 }
+      at_exit { Process.kill :QUIT, @pid }
       wait_master_ready("test_stderr.#$$.log")
     end
   end
@@ -95,17 +95,57 @@ class ServerTest < Test::Unit::TestCase
     tmp.close!
   end
 
-  def test_loading_app
+  def test_running_setup
     iam = "sad"
-    start :app => lambda { iam.replace "happy" }, :commands => { :iam => lambda { $stdout << iam } }
+    start :setup => lambda { iam.replace "happy" }, :commands => { :iam => lambda { $stdout << iam } }
     assert_equal "happy", hit("127.0.0.1:#@port", :iam).last
   end
 
-  def test_reloading_app
+  def test_running_setup_again
     text = "foo"
-    start :app => lambda { text << "bar" }, :commands => { :text => lambda { $stdout << text } }
+    start :setup => lambda { text << "bar" }, :commands => { :text => lambda { $stdout << text } }
     assert_equal "foobar", hit("127.0.0.1:#@port", :text).last
-    Process.kill "HUP", @pid 
+    Process.kill :HUP, @pid 
     assert_equal "foobar", hit("127.0.0.1:#@port", :text).last
   end
+
+  def test_running_background
+    sync = Tempfile.new("sync")
+    start :background => { :chmod => lambda { |*_| sync.chmod 0 while sleep 0.01 } }
+    assert_equal 0, sync.stat.mode & 1
+    sync.chmod 1 ; sleep 0.1
+    assert_equal 0, sync.stat.mode & 1
+  ensure
+    sync.close!
+  end
+
+  def test_stopping_background
+    sync = Tempfile.new("sync")
+    start :background => { :chmod => lambda { |*_| sync.chmod 0 while sleep 0.01 } }
+    sync.chmod 1 ; sleep 0.1
+    assert_equal 0, sync.stat.mode & 1
+    Process.kill :QUIT, @pid 
+    sleep 0.1
+    sync.chmod 1 ; sleep 0.1
+    assert_equal 1, sync.stat.mode & 1
+  ensure
+    sync.close!
+  end
+
+  def test_restarting_background
+    config = Tempfile.new("config")
+    config.write "$flag = 0" ; config.flush
+    sync = Tempfile.new("sync")
+    start :config_file => config.path, :background => { :chmod => lambda { |*_| sync.chmod $flag while sleep 0.01 } }
+    sync.chmod 1 ; sleep 0.1
+    assert_equal 0, sync.stat.mode & 1
+    config.rewind ; config.write "$flag = 1" ; config.flush
+    Process.kill :HUP, @pid 
+    sleep 0.1
+    sync.chmod 0 ; sleep 0.2
+    assert_equal 1, sync.stat.mode & 1
+  ensure
+    sync.close!
+  end
+
 end

metadata

@@ -1,7 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: pipemaster
 version: !ruby/object:Gem::Version 
-  version: 0.4.2
+  prerelease: false
+  segments: 
+  - 0
+  - 5
+  - 0
+  version: 0.5.0
 platform: ruby
 authors: 
 - Assaf Arkin
@@ -9,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-02-20 00:00:00 -08:00
+date: 2010-03-01 00:00:00 -08:00
 default_executable: 
 dependencies: []
 
@@ -51,7 +56,7 @@ licenses: []
 post_install_message: To get started run pipemaster --help
 rdoc_options: 
 - --title
-- Pipemaster 0.4.2
+- Pipemaster 0.5.0
 - --main
 - README.rdoc
 - --webcvs
@@ -62,18 +67,20 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      segments: 
+      - 0
       version: "0"
-  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      segments: 
+      - 0
       version: "0"
-  version: 
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.5
+rubygems_version: 1.3.6
 signing_key: 
 specification_version: 3
 summary: Use the fork