sponges 0.2.0 → 0.3.0

data/README.md

@@ -22,7 +22,7 @@ Internally, `sponges` strongly relies on Unix signals.
 
 ## Installation
 
-Ruby 1.9.2 (or superior) and Redis are required.
+Ruby 1.9.2 (or superior).
 
 Install it with rubygems:
 
@@ -66,15 +66,14 @@ Sponges.configure do |config|
   config.size          = 3
   config.daemonize     = true
   config.after_fork do
-    puts "Execute some when a child process is created"
+    puts "Execute code when a child process is created"
   end
   config.on_chld do
     puts "Execute code when a child process is killed"
 end
 
 # Register a pool named "worker_name".
-# Options are optionnal. Consider it as a default, options given by command have
-# the precedence.
+#
 Sponges.start "worker_name" do
   Worker.new({some: args}).run
 end
@@ -99,9 +98,10 @@ Start 8 instances of the worker and daemonize them:
 ruby example.rb start -d -s 8 # By default, size equals cpu core's size.
 ```
 
-Retart gracefully 4 instances of the worker and daemonize them:
+Retart gracefully 4 instances of the worker, with a timeout of 3 seconds and
+daemonize them:
 ``` bash
-ruby example.rb restart -g -d -s 4
+ruby example.rb restart -g -d -s 4 -t 3
 ```
 
 Stop workers with a `QUIT` signal :
@@ -116,10 +116,12 @@ ruby example.rb kill
 
 Stop workers with a `HUP` signal :
 ``` bash
-ruby example.rb stop -g
+ruby example.rb stop -g -t 5
 ```
 In this case, you will have to trap the `HUP` signal, and handle a clean stop
-from each workers. The point is to wait for a task to be done before quitting.
+from each workers. The point is to wait for a task to be done before quitting. A
+timeout can be specify with the `-t` option. When this timeout is hited, the
+process is killed.
 
 Increment worker's pool size :
 ``` bash
@@ -136,6 +138,24 @@ Show a list of workers and their children.
 ruby example.rb list
 ```
 
+## Stores
+
+sponges can store pids in memory or in redis. Memory is the default store. The
+`list` command is not available with the memory store.
+
+To select the redis store, you need to add `nest` to your application's
+Gemfile, and do the following.
+
+``` ruby
+gem "sponges"
+```
+
+``` ruby
+Sponges.configure do |config|
+  config.store         = :redis
+end
+```
+
 ## Acknowledgements
 
 sponges would not have been the same without [Jesse

data/lib/sponges.rb

@@ -2,13 +2,15 @@
 require 'boson/runner'
 require 'socket'
 require 'logger'
-require 'nest'
 require 'machine'
 require_relative 'sponges/configuration'
 require_relative 'sponges/supervisor'
 require_relative 'sponges/runner'
 require_relative 'sponges/commander'
 require_relative 'sponges/cli'
+require_relative 'sponges/store'
+require_relative 'sponges/store/memory'
+require_relative 'sponges/store/redis'
 
 module Sponges
   SIGNALS = [:INT, :QUIT, :TERM]

data/lib/sponges/cli.rb

@@ -17,8 +17,13 @@ module Sponges
     end
 
     option :gracefully, type: :boolean
+    option :timeout,    type: :numeric
     desc "Stop workers"
     def stop(options = {})
+      options = {
+        timeout:    Sponges::Configuration.timeout,
+        gracefully: Sponges::Configuration.gracefully
+      }.reject{|k, v| v.nil?}.merge(options)
       Sponges::Commander.new(Sponges::Configuration.worker_name, options).stop
     end
 
@@ -30,6 +35,7 @@ module Sponges
     option :daemonize,  type: :boolean
     option :size,       type: :numeric
     option :gracefully, type: :boolean
+    option :timeout,    type: :numeric
     desc "Restart workers"
     def restart(options = {})
       stop(options)
@@ -50,6 +56,10 @@ module Sponges
 
     desc "Show running processes"
     def list
+      if Sponges::Configuration.store == :memory
+        puts "Command not available with the memory store"
+        exit
+      end
       redis = Nest.new('sponges')
       puts %q{
  ___ _ __   ___  _ __   __ _  ___  ___

data/lib/sponges/commander.rb

@@ -1,12 +1,16 @@
 # encoding: utf-8
+require 'timeout'
+
 module Sponges
   # This class concern is to send messages to supervisor. It's used to send
   # messages like 'stop' or 'restart'
   #
   class Commander
+    attr_reader :store
+
     def initialize(name, options = {})
       @name, @options = name, options
-      @redis = Nest.new('sponges', Configuration.redis || Redis.new)[Socket.gethostname]
+      @store = Sponges::Store.new(@name)
     end
 
     # Kills the supervisor, and then all workers.
@@ -14,8 +18,14 @@ module Sponges
     def kill
       Sponges.logger.info "Runner #{@name} kill message received."
       stop :KILL
-      Array(@redis[:worker][@name][:pids].smembers).each do|pid|
-        kill_process(:KILL, pid, "Worker")
+      children_pids.each do|pid|
+        begin
+          kill_process(:KILL, pid, "Worker")
+        rescue Errno::ESRCH => e
+          # Don't panic
+        ensure
+          store.delete_children pid
+        end
       end
     end
 
@@ -24,8 +34,18 @@ module Sponges
     def stop(signal = nil)
       signal ||= gracefully? ? :HUP : :QUIT
       Sponges.logger.info "Runner #{@name} stop message received."
-      if pid = @redis[:worker][@name][:supervisor].get
-        kill_process(signal, pid)
+      if pid = supervisor_pid
+        if @options[:timeout]
+          begin
+            Timeout::timeout(@options[:timeout]) do
+              kill_process(signal, pid)
+            end
+          rescue Timeout::Error
+            kill
+          end
+        else
+          kill_process(signal, pid)
+        end
       else
         Sponges.logger.info "No supervisor found."
       end
@@ -35,9 +55,9 @@ module Sponges
     #
     def increment
       Sponges.logger.info "Runner #{@name} increment message received."
-      if pid = @redis[:worker][@name][:supervisor].get
+      if pid = supervisor_pid
         begin
-          Process.kill :TTIN, pid.to_i
+          Process.kill :TTIN, supervisor.to_i
         rescue Errno::ESRCH => e
           Sponges.logger.error e
         end
@@ -50,9 +70,9 @@ module Sponges
     #
     def decrement
       Sponges.logger.info "Runner #{@name} decrement message received."
-      if pid = @redis[:worker][@name][:supervisor].get
+      if supervisor_pid
         begin
-          Process.kill :TTOU, pid.to_i
+          Process.kill :TTOU, supervisor_pid.to_i
         rescue Errno::ESRCH => e
           Sponges.logger.error e
         end
@@ -63,6 +83,14 @@ module Sponges
 
     private
 
+    def supervisor_pid
+      @store.supervisor_pid
+    end
+
+    def children_pids
+      @store.children_pids
+    end
+
     def kill_process(signal, pid, type = "Supervisor")
       pid = pid.to_i
       begin
@@ -73,7 +101,7 @@ module Sponges
         end
         Sponges.logger.info "#{type} #{pid} has stopped."
       rescue Errno::ESRCH => e
-        Sponges.logger.error e
+        # Don't panic
       end
     end
 

data/lib/sponges/configuration.rb

@@ -5,7 +5,7 @@ module Sponges
   class Configuration
     class << self
       ACCESSOR = [:worker_name, :worker, :logger, :redis, :size,
-        :daemonize, :after_fork
+        :daemonize, :after_fork, :timeout, :gracefully, :store
       ]
       attr_accessor *ACCESSOR
 
@@ -27,6 +27,14 @@ module Sponges
       def on_chld(&block)
         Hook._on_chld = block
       end
+
+      def store
+        @store || :memory
+      end
+
+      def redis
+        @redis ||= Redis.new
+      end
     end
   end
 

data/lib/sponges/runner.rb

@@ -4,15 +4,17 @@ module Sponges
   # watch over the supervisor.
   #
   class Runner
+    attr_reader :store
+
     def initialize(name, options = {}, block)
       @name, @block = name, block
       @options = default_options.merge options
-      @redis = Nest.new('sponges', Configuration.redis || Redis.new)
-      if running?
+      @store = Sponges::Store.new(@name)
+      if store.running?
         Sponges.logger.error "Runner #{@name} already started."
         exit
       end
-      @redis[:hostnames].sadd Socket.gethostname
+      store.register_hostname Socket.gethostname
     end
 
     def start
@@ -29,20 +31,6 @@ module Sponges
 
     private
 
-    def running?
-      if pid = @redis[Socket.gethostname][:worker][@name][:supervisor].get
-        begin
-          Process.kill 0, pid.to_i
-          true
-        rescue Errno::ESRCH => e
-          @redis[Socket.gethostname][:worker][@name][:supervisor].del
-          false
-        end
-      else
-        false
-      end
-    end
-
     def trap_signals
       Sponges::SIGNALS.each do |signal|
         trap(signal) {|signal| kill_supervisor(signal) }
@@ -63,7 +51,7 @@ module Sponges
     def fork_supervisor
       fork do
         $PROGRAM_NAME = "#{@name}_supervisor"
-        Supervisor.new(@name, @options, @block).start
+        Supervisor.new(@name, @options, store, @block).start
       end
     end
 

data/lib/sponges/store.rb

@@ -0,0 +1,15 @@
+# encoding: utf-8
+module Sponges
+  class Store
+    class << self
+      def new(name)
+        case Sponges::Configuration.store
+        when :memory
+          Sponges::Store::Memory.new(name)
+        when :redis
+          Sponges::Store::Redis.new(name)
+        end
+      end
+    end
+  end
+end

data/lib/sponges/store/memory.rb

@@ -0,0 +1,65 @@
+# encoding: utf-8
+require 'forwardable'
+
+module Sponges
+  class Store
+    class Memory
+      extend Forwardable
+
+      attr_writer :supervisor_pid
+      attr_reader :pids, :name
+
+      def initialize(name)
+        @pids, @name = [], name
+      end
+
+      def_delegator :@pids, :<<, :add_children
+      def_delegator :@pids, :delete, :delete_children
+
+      def supervisor_pid
+        return @supervisor_pid if @supervisor_pid
+        s = find_supervisor
+        @supervisor_pid = s.pid if s
+      end
+
+      def children_pids
+        @pids.any? ? @pids : find_childs.map(&:pid)
+      end
+
+      def running?
+        !!find_supervisor
+      end
+
+      def register(supervisor_pid)
+        @supervisor_pid = supervisor_pid
+      end
+
+      def register_hostname(hostname)
+      end
+
+      def on_fork
+      end
+
+      def clear
+      end
+
+      private
+
+      def supervisor_name
+        "#{name}_supervisor"
+      end
+
+      def childs_name
+        "#{name}_child"
+      end
+
+      def find_supervisor
+        Sys::ProcTable.ps.select {|f| f.cmdline == supervisor_name }.first
+      end
+
+      def find_childs
+        Sys::ProcTable.ps.select {|f| f.cmdline =~ /^#{childs_name}/ }
+      end
+    end
+  end
+end

data/lib/sponges/store/redis.rb

@@ -0,0 +1,74 @@
+# encoding: utf-8
+module Sponges
+  class Store
+    class Redis
+      attr_reader :redis, :hostname_store, :name
+      private     :redis, :hostname_store
+
+      def initialize(name)
+        @name = name
+        @redis ||= Nest.new('sponges', Configuration.redis)
+        @hostname_store = @redis[Socket.gethostname]
+      end
+
+      def supervisor_pid
+        hostname_store[:worker][name][:supervisor].get
+      end
+
+      def children_pids
+        Array(hostname_store[:worker][name][:pids].smembers)
+      end
+
+      def running?
+        if pid = hostname_store[:worker][name][:supervisor].get
+          begin
+            Process.kill 0, pid.to_i
+            true
+          rescue Errno::ESRCH => e
+            hostname_store[:worker][name][:supervisor].del
+            false
+          end
+        else
+          false
+        end
+      end
+
+      def register_hostname(hostname)
+        redis[:hostnames].sadd hostname
+      end
+
+      def register(supervisor_pid)
+        hostname_store[:workers].sadd name
+        hostname_store[:worker][name][:supervisor].set supervisor_pid
+      end
+
+      def add_children(pid)
+        hostname_store[:worker][name][:pids].sadd pid
+      end
+
+      def delete_children(pid)
+        hostname_store[:worker][name][:pids].srem pid
+      end
+
+      def clear(name)
+        hostname_store[:worker][name][:supervisor].del
+        hostname_store[:workers].srem name
+      end
+
+      def on_fork
+        Sponges::Configuration.redis.client.reconnect
+      end
+
+      private
+
+      def find_supervisor
+        Sys::ProcTable.ps.select {|f| f.cmdline == supervisor_name }.first
+      end
+
+      def find_childs
+        Sys::ProcTable.ps.select {|f| f.cmdline =~ /^#{childs_name}/ }
+      end
+    end
+  end
+end
+

data/lib/sponges/supervisor.rb

@@ -1,15 +1,17 @@
 # encoding: utf-8
 module Sponges
   class Supervisor
-    def initialize(name, options, block)
-      @name, @options, @block = name, options, block
-      set_up_redis
-      @pids = @redis[:worker][name][:pids]
+    attr_reader :store, :name, :options
+
+    def initialize(name, options, store, block)
+      @name, @options, @store, @block = name, options, store, block
+      store.on_fork
+      store.register Process.pid
       @children_seen = 0
     end
 
     def start
-      @options[:size].times do
+      options[:size].times do
         fork_children
       end
       trap_signals
@@ -22,16 +24,6 @@ module Sponges
 
     private
 
-    def set_up_redis
-      if Configuration.redis
-        redis_client = Configuration.redis
-        redis_client.client.reconnect
-      end
-      @redis = Nest.new('sponges', redis_client || Redis.new)[Socket.gethostname]
-      @redis[:workers].sadd @name
-      @redis[:worker][@name][:supervisor].set Process.pid
-    end
-
     def fork_children
       name = children_name
       pid = fork do
@@ -40,11 +32,11 @@ module Sponges
         @block.call
       end
       Sponges.logger.info "Supervisor create a child with #{pid} pid."
-      @pids.sadd pid
+      store.add_children pid
     end
 
     def children_name
-      "#{@name}_child_#{@children_seen +=1}"
+      "#{name}_child_#{@children_seen +=1}"
     end
 
     def trap_signals
@@ -59,19 +51,19 @@ module Sponges
       end
       trap(:TTOU) do
         Sponges.logger.warn "Supervisor decrement child's pool by one."
-        if pids.first
-          kill_one(pids.first, :HUP)
+        if store.children_pids.first
+          kill_one(store.children_pids.first, :HUP)
         else
           Sponges.logger.warn "No more child to kill."
         end
       end
       trap(:CHLD) do
-        pids.each do |pid|
+        store.children_pids.each do |pid|
           begin
             dead = Process.waitpid(pid.to_i, Process::WNOHANG)
             if dead
               Sponges.logger.warn "Child #{dead} died. Restarting a new one..."
-              @pids.srem dead
+                store.delete_children dead
               Sponges::Hook.on_chld
               fork_children
             end
@@ -88,30 +80,26 @@ module Sponges
       Process.waitall
       Sponges.logger.info "Children shutdown complete."
       Sponges.logger.info "Supervisor shutdown. Exiting..."
-      pid = @redis[:worker][@name][:supervisor]
-      @redis[:worker][@name][:supervisor].del
-      @redis[:workers].srem @name
+      pid = store.supervisor_pid
+      store.clear(name)
       Process.kill :USR1, pid.to_i
     end
 
     def kill_them_all(signal)
-      pids.each do |pid|
+      store.children_pids.each do |pid|
         kill_one(pid, signal)
       end
     end
 
     def kill_one(pid, signal)
       begin
-        @pids.srem pid
         Process.kill signal, pid.to_i
+        Process.waitpid pid.to_i
+        store.delete_children pid
         Sponges.logger.info "Child #{pid} receive a #{signal} signal."
       rescue Errno::ESRCH => e
         # Don't panic
       end
     end
-
-    def pids
-      Array(@pids.smembers)
-    end
   end
 end

data/lib/sponges/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module Sponges
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sponges
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-23 00:00:00.000000000 Z
+date: 2012-12-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: boson
@@ -27,22 +27,6 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: nest
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: machine
   requirement: !ruby/object:Gem::Requirement
@@ -92,6 +76,9 @@ files:
 - lib/sponges/commander.rb
 - lib/sponges/configuration.rb
 - lib/sponges/runner.rb
+- lib/sponges/store.rb
+- lib/sponges/store/memory.rb
+- lib/sponges/store/redis.rb
 - lib/sponges/supervisor.rb
 - lib/sponges/version.rb
 homepage: https://github.com/AF83/sponges