resque-pool-dynamic 0.1.0

data/lib/resque/pool/dynamic/shell.rb

@@ -0,0 +1,106 @@
+require 'pathname'
+require 'json'
+
+require 'ripl'
+
+require 'resque/pool/dynamic'
+
+module Resque
+  class Pool
+    # Add-on to resque-pool to allow manually controlled, dynamic sessions
+    class Dynamic
+      # Run a dynamic resque pool session.
+      # @param [Hash] options Options passed to the Shell instance
+      # @option opts [String] :no_start Don't start resque-pool automatically
+      def self.shell(options={})
+        myself = self.new
+        myself.start unless options[:no_start]
+        myself.status
+        Shell.run(myself, options)
+        puts "\nBye!"
+      ensure
+        myself.stop if myself
+      end
+
+      # Ripl shell subclassed for nicer CLI experience
+      class Shell < Ripl::Shell
+        # When printing error, cut backtrace at (ripl)
+        def print_eval_error(e)
+          i = e.backtrace.index { |ln| ln =~ /\(ripl\):\d+:in `run'/ }
+          if not i
+            bt = e.backtrace
+          elsif i > 0
+            bt = e.backtrace[0..i-1]
+          else
+            bt = []
+          end
+          bt = bt.map { |ln| "  " << ln }.join("\n")
+          puts "ERROR: #{e.to_s}\n#{bt}"
+        end
+
+        # Don't print nil return values
+        def print_result(result)
+          super if @error_raised or !result.nil?
+        end
+
+        # Show help
+        # @param [String] subject Topic to display
+        def help(subject=nil)
+          @help ||= File.open(
+            Pathname.new(__FILE__).dirname.join('help.json')
+            ) { |f| JSON::load(f.read) }
+          if subject
+            if @help.has_key?(subject)
+              puts @help[subject]['full']
+              return
+            end
+            puts "I don't know anything about #{subject}."
+          end
+          puts "Known commands:"
+          @help.keys.sort.each do |cmd|
+            puts "  #{cmd} - #{@help[cmd]['summary']}"
+          end
+          puts "For details, type: help command"
+        end
+
+        # Handle help as special command
+        # @param [String] input Input string
+        def eval_input(input)
+          splut = input.split(/\s+/,2)
+          if splut.first == 'help'
+            help(splut[1])
+          else
+            super
+          end
+        end
+
+        # Run the Ripl shell in context of specified instance
+        # @param [Object] instance Instance to get bindings from
+        # @param [Hash] opts Options passed to Ripl::Shell
+        def self.run(instance, opts={})
+          options = {
+            :readline => true, 
+            :riplrc => ENV['RIPL_RC'] || '~/.riplrc',
+            :completion => {},
+            :binding => instance.instance_eval { binding }
+          }
+          options.update(opts)
+          Ripl::Runner::load_rc(options[:riplrc]) if options[:riplrc]!=''
+          create(options).loop
+        end
+      end
+
+      ### Utility functions
+
+      # Loop, providing index to the block
+      # @yield [i] Ever increasing index
+      def loop_with_index
+        index = 0
+        loop do
+          yield(index)
+          index += 1
+        end
+      end
+    end
+  end
+end

data/lib/resque/pool/dynamic/tasks.rb

@@ -0,0 +1,29 @@
+
+desc <<EOF
+Starts a dynamic pool of Resque worker processes.
+
+Variables:
+WORKERS=worker spec - specify initial numbers of workers
+                      (look below for format description)
+NO_START=1 - don't start resque-pool master automatically
+             (default is to start)
+
+Workers spec format:
+queue_name=process_number[:queue_name=process_number[:...]]
+
+queue_name can be whatever resque:work will accept as QUEUE=
+parameter.
+
+Example:
+$ rake resque:pool:dynamic WORKERS=foo=1:bar=1:baz,quux,xyzzy=5:*=2
+will start:
+ - one worker for queue 'foo'
+ - one worker for queue 'bar'
+ - five workers for queues baz,quux,xyzzy
+ - two workers for all queues
+EOF
+task "resque:pool:dynamic" do
+  require 'resque/pool/dynamic/shell'
+
+  Resque::Pool::Dynamic.shell(:no_start => ENV['NO_START'])
+end

data/lib/resque/pool/dynamic/version.rb

@@ -0,0 +1,7 @@
+module Resque
+  class Pool
+    class Dynamic
+      VERSION = "0.1.0"
+    end
+  end
+end

data/lib/resque/pool/dynamic.rb

@@ -0,0 +1,217 @@
+require 'tempfile'
+require 'yaml'
+
+require "resque/pool/dynamic/version"
+require "resque/pool/dynamic/logfile"
+
+module Resque
+  class Pool
+    class Dynamic
+      # PID of the resque-pool process, or nil if it's not running
+      # @return [Integer, NilClass]
+      # @api cli
+      attr_reader :pid
+
+      def initialize
+        # Make myself a project group leader
+        Process.setpgid(0, 0)
+      end
+
+      # Tempfile instance for configuration file
+      # @return [Tempfile]
+      def config_tempfile
+        @config_tempfile ||= Tempfile.new(%w(resque-pool-dynamic .yaml))
+      end
+
+      # Path to the temporary config file
+      # @return [String]
+      # @api cli
+      def config_path
+        config_tempfile.path
+      end
+
+      # Parse workers configuration
+      # @param [String] spec Workers specification string
+      #   Format: queue2=number:queue2=number2:queue3,queue5=number4:*=number5
+      # @return [Hash] Configuration dictionary
+      # @example
+      #   parse_config_string('foo=1:bar=2:baz,quux=4')
+      #   #=> {"baz,quux"=>4, "foo"=>1, "bar"=>2}
+      def parse_config_string(spec)
+        return {} unless spec
+        Hash[ 
+          spec.split(':').map { |w|
+            k, v = w.split('=') ; [ k, v.to_i ] } 
+        ]
+      end
+
+      # @overload config
+      #   Show current workers configuration
+      #   @return [Hash]
+      # @overload config(opts)
+      #   Update workers configuration.
+      #   If configuration has change and resque-pool is running, it is reloaded.
+      #   @param [Hash] opts Dictionary of worker process counts to update
+      #     (pass 0 or nil as value to delete all workers for a queue from pool)
+      #   @return [Hash] Updated config
+      # @note Default configuration is taken from environment variable
+      #   "WORKERS", as interpreted by the #parse_config_string method.
+      # @example config
+      #   #=> {"foo"=>1, "bar"=>2}
+      #   config :foo => 2, :baz => 7
+      #   #=> {"foo"=>2, "baz"=>7, "bar"=>2}
+      #   config :bar => 0
+      #   #=> {"foo"=>2, "baz"=>7}
+      # @api cli
+      def config(args=nil)
+        @config ||= parse_config_string(ENV['WORKERS'])
+
+        if args
+          oldconfig = config.clone
+
+          args.each do |w, n|
+            w = w.to_s
+            if n.nil? || n==0
+              @config.delete w
+            else
+              @config[w] = n
+            end
+          end
+
+          if pid && @config != oldconfig
+            write_config
+            reload
+          end
+        end
+
+        @config
+      end
+
+      # Write temporary configuration file
+      # @api cli
+      def write_config
+        File.open(config_path, 'w') do |cf|
+          cf.write(YAML::dump(config))
+        end
+      end
+
+      # Start resque-pool master
+      # This is executed in a forked child process.
+      def run_pool
+        @pid = $$
+        ENV["RESQUE_POOL_CONFIG"] = config_path
+
+        $stdin.reopen '/dev/null'
+        log = File.new(log_path, "a")
+        $stdout.reopen log
+        $stderr.reopen log
+        $stdout.sync = $stderr.sync = true
+
+        require 'rake'
+        Rake::Task['resque:pool'].invoke 
+      end
+
+      # Fork a child process for resque-pool master
+      # @return [Integer] PID of the child process
+      # @api cli
+      def start!
+        raise "Already running: #{pid}" if pid
+        write_config
+        @pid = fork { self.run_pool }
+        Process.setpgid(pid, 0) # don't broadcast ^C to child
+
+        if 30.times { |i| sleep 1 ; break if File.exist? log_path }
+          # Loop will return nil if broken. If value is returned, file does not exist.
+          raise "Log file #{log_path} still not present after #{i} seconds, giving up"
+          stop!
+        end
+
+        pid
+      end
+
+      # Start resque-pool, showing startup logs
+      # @api cli
+      def start
+        unless pid
+          start!
+          log.rewind
+          log.tail_until /started manager/
+        else
+          warn "Already started as #{pid}"
+        end
+      end
+
+      # Return pid of running resque-pool or raise an exception
+      # @return [Integer]
+      # @raise [RuntimeError]
+      def pid!
+        pid or raise "Not started!"
+      end
+
+      # Send signal to a running resque-pool
+      # @param [String, Integer] sig Signal name or number
+      # @api cli
+      def kill!(sig)
+        Process.kill(sig, pid!)
+      end
+
+      # Stop running resque-pool
+      # @api cli
+      def stop!
+        if pid
+          kill! "INT"
+          wpid, status = Process.wait2(pid)
+          @pid = nil
+          status
+        end
+      end
+
+      # Stop running resque-pool, show shutdown logs
+      # @api cli
+      def stop
+        puts "Shutting down resque-pool-master #{pid}"
+        log.ff if has_log?
+        status = stop!
+        log.tail if has_log?
+        return status
+      end
+
+      # Reload resque-pool configuration
+      # @api cli
+      def reload
+        puts "Reloading resque-pool-master #{pid!} configuration"
+        write_config
+        kill!('HUP')
+      end
+
+      # Show child process tree by calling `pstree` system command
+      # @api cli
+      def pstree
+        system case RUBY_PLATFORM
+               when /darwin/
+                 "pstree -w #{pid!}"
+               when /linux/
+                 "pstree -l -a -p #{pid!}"
+               else
+                 "pstree #{pid!}"
+               end
+        nil
+      end
+
+      # Show current status
+      # @api cli
+      def status
+        puts( '',
+          "Status: " << ( pid ? "running, pid: #{pid}" : "not running" ),
+          "Configuration:",
+          YAML::dump(config).grep(/^(?!---\s*$)/).map { |v| "  " << v }
+          )
+        if pid
+          puts "Process tree:"
+          pstree
+        end
+        puts
+      end
+    end
+  end
+end

data/lib/resque/pool/help.json

@@ -0,0 +1 @@
+{"pid":{"summary":"PID of the resque-pool process, or nil if it's not running.","full":"----------------------------------- Method: #pid (Resque::Pool::Dynamic)\n                                (Defined in: lib/resque/pool/dynamic.rb)\n    dynamic.pid -> (Integer, NilClass) (readonly)\n------------------------------------------------------------------------\n    PID of the resque-pool process, or nil if it's not running \nReturns:\n--------\n    (Integer, NilClass) - \n"},"config_path":{"summary":"Path to the temporary config file.","full":"--------------------------- Method: #config_path (Resque::Pool::Dynamic)\n                                (Defined in: lib/resque/pool/dynamic.rb)\n    dynamic.config_path -> String \n------------------------------------------------------------------------\n    Path to the temporary config file \nReturns:\n--------\n    (String) - \n"},"config":{"summary":"","full":"-------------------------------- Method: #config (Resque::Pool::Dynamic)\n                                (Defined in: lib/resque/pool/dynamic.rb)\n    dynamic.config -> Hash \n    dynamic.config(opts) -> Hash \n------------------------------------------------------------------------\n    Note: Default configuration is taken from environment variable\n    \"WORKERS\", as interpreted by the #parse_config_string method.\nExamples:\n---------\n    # config\n    #=> {\"foo\"=>1, \"bar\"=>2}\n    config :foo => 2, :baz => 7\n    #=> {\"foo\"=>2, \"baz\"=>7, \"bar\"=>2}\n    config :bar => 0\n    #=> {\"foo\"=>2, \"baz\"=>7}\nOverloads:\n----------\n------------------------------------------------------------------------\n    dynamic.config -> Hash \n------------------------------------------------------------------------\n        Show current workers configuration \n    Returns:\n    --------\n        (Hash) - \n------------------------------------------------------------------------\n    dynamic.config(opts) -> Hash \n------------------------------------------------------------------------\n        Update workers configuration. If configuration has change and\n        resque-pool is running, it is reloaded. \n    Parameters:\n    -----------\n        (Hash) opts - Dictionary of worker process counts to update\n        (pass 0 or nil as value to delete all workers for a queue from pool)\n    Returns:\n    --------\n        (Hash) - Updated config\n"},"write_config":{"summary":"Write temporary configuration file.","full":"-------------------------- Method: #write_config (Resque::Pool::Dynamic)\n                                (Defined in: lib/resque/pool/dynamic.rb)\n    dynamic.write_config \n------------------------------------------------------------------------\n    Write temporary configuration file \n"},"start!":{"summary":"Fork a child process for resque-pool master.","full":"-------------------------------- Method: #start! (Resque::Pool::Dynamic)\n                                (Defined in: lib/resque/pool/dynamic.rb)\n    dynamic.start! -> Integer \n------------------------------------------------------------------------\n    Fork a child process for resque-pool master \nReturns:\n--------\n    (Integer) - PID of the child process\n"},"start":{"summary":"Start resque-pool, showing startup logs.","full":"--------------------------------- Method: #start (Resque::Pool::Dynamic)\n                                (Defined in: lib/resque/pool/dynamic.rb)\n    dynamic.start \n------------------------------------------------------------------------\n    Start resque-pool, showing startup logs \n"},"kill!":{"summary":"Send signal to a running resque-pool.","full":"--------------------------------- Method: #kill! (Resque::Pool::Dynamic)\n                                (Defined in: lib/resque/pool/dynamic.rb)\n    dynamic.kill!(sig) \n------------------------------------------------------------------------\n    Send signal to a running resque-pool \nParameters:\n-----------\n    (String, Integer) sig - Signal name or number\n"},"stop!":{"summary":"Stop running resque-pool.","full":"--------------------------------- Method: #stop! (Resque::Pool::Dynamic)\n                                (Defined in: lib/resque/pool/dynamic.rb)\n    dynamic.stop! \n------------------------------------------------------------------------\n    Stop running resque-pool \n"},"stop":{"summary":"Stop running resque-pool, show shutdown logs.","full":"---------------------------------- Method: #stop (Resque::Pool::Dynamic)\n                                (Defined in: lib/resque/pool/dynamic.rb)\n    dynamic.stop \n------------------------------------------------------------------------\n    Stop running resque-pool, show shutdown logs \n"},"reload":{"summary":"Reload resque-pool configuration.","full":"-------------------------------- Method: #reload (Resque::Pool::Dynamic)\n                                (Defined in: lib/resque/pool/dynamic.rb)\n    dynamic.reload \n------------------------------------------------------------------------\n    Reload resque-pool configuration \n"},"pstree":{"summary":"Show child process tree by calling `pstree` system command.","full":"-------------------------------- Method: #pstree (Resque::Pool::Dynamic)\n                                (Defined in: lib/resque/pool/dynamic.rb)\n    dynamic.pstree \n------------------------------------------------------------------------\n    Show child process tree by calling `pstree` system command \n"},"status":{"summary":"Show current status.","full":"-------------------------------- Method: #status (Resque::Pool::Dynamic)\n                                (Defined in: lib/resque/pool/dynamic.rb)\n    dynamic.status \n------------------------------------------------------------------------\n    Show current status \n"},"log_path":{"summary":"Logfile path.","full":"------------------------------ Method: #log_path (Resque::Pool::Dynamic)\n                        (Defined in: lib/resque/pool/dynamic/logfile.rb)\n    dynamic.log_path -> String \n------------------------------------------------------------------------\n    Logfile path \nReturns:\n--------\n    (String) - Path to the log file\n"},"log":{"summary":"Open log of resque-pool process.","full":"----------------------------------- Method: #log (Resque::Pool::Dynamic)\n                        (Defined in: lib/resque/pool/dynamic/logfile.rb)\n    dynamic.log -> Logfile \n------------------------------------------------------------------------\n    Open log of resque-pool process \nReturns:\n--------\n    (Logfile) - Logfile instance\n"},"has_log?":{"summary":"True if we have an open log file.","full":"------------------------------ Method: #has_log? (Resque::Pool::Dynamic)\n                        (Defined in: lib/resque/pool/dynamic/logfile.rb)\n    dynamic.has_log? -> Boolean \n------------------------------------------------------------------------\n    True if we have an open log file \nReturns:\n--------\n    (Boolean) - \n"},"log.tail":{"summary":"Show last n lines of the file, or follow the file.","full":"------------------------- Method: #tail (Resque::Pool::Dynamic::Logfile)\n                        (Defined in: lib/resque/pool/dynamic/logfile.rb)\n    logfile.tail(n = nil) {|ln| ... } \n------------------------------------------------------------------------\n    Show last n lines of the file, or follow the file \nParameters:\n-----------\n    (Integer, NilClass) n - Number of lines or nil\nYields:\n-------\n    (ln) - Called for every line of file\nYield Parameters:\n-----------------\n    (String) ln - Input line\n"},"log.tail_until":{"summary":"Follow the file until a line matches regexp.","full":"------------------- Method: #tail_until (Resque::Pool::Dynamic::Logfile)\n                        (Defined in: lib/resque/pool/dynamic/logfile.rb)\n    logfile.tail_until(rx) \n------------------------------------------------------------------------\n    Follow the file until a line matches regexp \n"},"log.tail_to_eof":{"summary":"Print contents of the file until current end of file.","full":"------------------ Method: #tail_to_eof (Resque::Pool::Dynamic::Logfile)\n                        (Defined in: lib/resque/pool/dynamic/logfile.rb)\n    logfile.tail_to_eof \n------------------------------------------------------------------------\n    Print contents of the file until current end of file \n"},"log.ff":{"summary":"Fast forward until end of file (see IO::Tail#backward).","full":"--------------------------- Method: #ff (Resque::Pool::Dynamic::Logfile)\n                        (Defined in: lib/resque/pool/dynamic/logfile.rb)\n    logfile.ff \n------------------------------------------------------------------------\n    Fast forward until end of file (see IO::Tail#backward) \n"},"log.rew":{"summary":"Rewind until beginning of file (see IO::Tail#forward).","full":"-------------------------- Method: #rew (Resque::Pool::Dynamic::Logfile)\n                        (Defined in: lib/resque/pool/dynamic/logfile.rb)\n    logfile.rew \n------------------------------------------------------------------------\n    Rewind until beginning of file (see IO::Tail#forward) \n"},"log.rewind":{"summary":"Rewind until beginning of file (see IO::Tail#forward).","full":"----------------------- Method: #rewind (Resque::Pool::Dynamic::Logfile)\n                        (Defined in: lib/resque/pool/dynamic/logfile.rb)\n    logfile.rewind \n------------------------------------------------------------------------\n    Rewind until beginning of file (see IO::Tail#forward) \n"},"exit":{"summary":"Finish work","full":"Finish work (AKA quit)"}}

data/resque-pool-dynamic.gemspec

@@ -0,0 +1,29 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "resque/pool/dynamic/version"
+
+Gem::Specification.new do |s|
+  s.name        = "resque-pool-dynamic"
+  s.version     = Resque::Pool::Dynamic::VERSION
+  s.authors     = ["Maciej Pasternacki"]
+  s.email       = ["maciej@pasternacki.net"]
+  s.homepage    = "https://github.com/mpasternacki/resque-pool-dynamic"
+  s.summary     = "A dynamic manager for resque pool"
+  s.description = <<EOF
+A class to dynamically manage number of processes and status in the
+resque pool with a command-line user interface to interacively manage
+and supervise the worker pool.
+EOF
+
+  s.rubyforge_project = "resque-pool-dynamic"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency "yard"
+  s.add_runtime_dependency "io-tail"
+  s.add_runtime_dependency "rake"
+  s.add_runtime_dependency "ripl"
+end

metadata

@@ -0,0 +1,127 @@
+--- !ruby/object:Gem::Specification
+name: resque-pool-dynamic
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Maciej Pasternacki
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-04-25 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: &70134237409700 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70134237409700
+- !ruby/object:Gem::Dependency
+  name: io-tail
+  requirement: &70134237409220 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70134237409220
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &70134237408800 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70134237408800
+- !ruby/object:Gem::Dependency
+  name: ripl
+  requirement: &70134237408380 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70134237408380
+description: ! 'A class to dynamically manage number of processes and status in the
+
+  resque pool with a command-line user interface to interacively manage
+
+  and supervise the worker pool.
+
+'
+email:
+- maciej@pasternacki.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README.md
+- Rakefile
+- doc/Resque.html
+- doc/Resque/Pool.html
+- doc/Resque/Pool/Dynamic.html
+- doc/Resque/Pool/Dynamic/Logfile.html
+- doc/Resque/Pool/Dynamic/Shell.html
+- doc/_index.html
+- doc/class_list.html
+- doc/css/common.css
+- doc/css/full_list.css
+- doc/css/style.css
+- doc/file.README.html
+- doc/file_list.html
+- doc/frames.html
+- doc/index.html
+- doc/js/app.js
+- doc/js/full_list.js
+- doc/js/jquery.js
+- doc/method_list.html
+- doc/top-level-namespace.html
+- help.yml
+- lib/resque/pool/dynamic.rb
+- lib/resque/pool/dynamic/logfile.rb
+- lib/resque/pool/dynamic/shell.rb
+- lib/resque/pool/dynamic/tasks.rb
+- lib/resque/pool/dynamic/version.rb
+- lib/resque/pool/help.json
+- resque-pool-dynamic.gemspec
+homepage: https://github.com/mpasternacki/resque-pool-dynamic
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: resque-pool-dynamic
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: A dynamic manager for resque pool
+test_files: []
+has_rdoc: