sidekiq-runner 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ca0010af09c70a83fb3d9a7e19bb1fab9ab8c2d7
-  data.tar.gz: 2a56f515a2abc4174eca0e222161facc262d45bc
+  metadata.gz: a964d456a2836681b258bc3cc4052fbc26dce0dd
+  data.tar.gz: e931c16b0328eff55bcd3a09cba5de69e73e6e55
 SHA512:
-  metadata.gz: 30074809519b45508efeaa338146c1e601065f1e77c4d7df375d19e1ef67dfc4af440475f2643eb53146b3185092bd0ea361b944aa1683717471cfa2fa03c23f
-  data.tar.gz: c03238275cb23d3a27628abca14a087a22a0a9a519355525103392e05aed0f4039e87a3f7078fe8fdbb26d6c09e5057a434f1eae3d02d73ef0f5ea6d1c441eb8
+  metadata.gz: 38bbf8146a5b13e4ab5c22891c5ccf49a206e3f148273c8d813ae6e87d3f290096fb2df3c4af9877e7af2e25c7f8e2891bf34e3b077a3aad16b2295858428476
+  data.tar.gz: b58ed792d28ce63e3ece4538a53a832b0cbef7ff9e2cc85c3f41ade78fdf3531df9e63f743e7e52301189f00c6254e9901c0f69141424a5de1c02616a7be53b4

data/lib/sidekiq-runner.rb

@@ -1,6 +1,6 @@
+require 'sidekiq-runner/sidekiq_instance'
 require 'sidekiq-runner/sidekiq_configuration'
 require 'sidekiq-runner/god_configuration'
-require 'sidekiq-runner/sidekiq_instance'
 require 'sidekiq-runner/version'
 
 module SidekiqRunner
@@ -13,11 +13,9 @@ module SidekiqRunner
   end
 
   def self.start
-    sidekiq_config, god_config = get_all_settings
-
-    fail 'No sidekiq instances defined. There is nothing to run.' if sidekiq_config.empty?
+    sidekiq_config, god_config = SidekiqConfiguration.get, GodConfiguration.get
 
-    abort 'God is running. I found an instance of a running god process. Please stop it manually and try again.' if god_alive?(god_config)
+    abort 'God is already running.' if god_alive?(god_config)
 
     run(:start, sidekiq_config) do
       puts 'Starting god.'
@@ -26,37 +24,27 @@ module SidekiqRunner
   end
 
   def self.stop
-    sidekiq_config, god_config = get_all_settings
+    sidekiq_config, god_config = SidekiqConfiguration.get, GodConfiguration.get
 
     run(:stop, sidekiq_config) do
       God::EventHandler.load
 
       if god_alive?(god_config)
-        puts "Stopping process #{god_config.process_name}."
-
-        # Stop all the processes
         sidekiq_config.each_key do |name|
+          puts "Stopping Sidekiq instance #{name}..."
           God::CLI::Command.new('stop', god_config.options, ['', name])
         end
 
-        puts 'Terminating god.'
+        puts "Terminating god process #{god_config.process_name}..."
         God::CLI::Command.new('terminate', god_config.options, [])
       else
-        puts 'God is not running, so no need to stop it.'
+        abort 'God is not running, so no need to stop it.'
       end
     end
   end
 
-  def self.settings
-    SidekiqConfiguration.get.to_hash
-  end
-
   private
 
-  def self.get_all_settings
-    [SidekiqConfiguration.get, GodConfiguration.get]
-  end
-
   def self.god_alive?(god_config)
     puts 'Checking whether god is alive...'
 
@@ -65,7 +53,7 @@ module SidekiqRunner
     DRb.start_service('druby://127.0.0.1:0')
     server = DRbObject.new(nil, God::Socket.socket(god_config.port))
 
-    # ping server to ensure that it is responsive
+    # Ping server to ensure that it is responsive.
     begin
       server.ping
     rescue DRb::DRbConnError
@@ -76,14 +64,15 @@ module SidekiqRunner
 
   def self.run(action, sidekiq_config)
     begin
-      # Use this flag to actually load all of the god infrastructure
+
+      # Use this flag to actually load all of the god infrastructure.
       $load_god = true
       require 'god'
       require 'god/cli/run'
 
-      # Peform the action
+      # Peform the action.
       yield if block_given?
-      cb = nil
+
     rescue SystemExit => e
       cb = e.success? ? "#{action}_success_cb" : "#{action}_error_cb"
     ensure
@@ -94,5 +83,4 @@ module SidekiqRunner
       end
     end
   end
-
 end

data/lib/sidekiq-runner/god_configuration.rb

@@ -1,12 +1,18 @@
-require 'sidekiq-runner/common_configuration'
+require 'yaml'
 
 module SidekiqRunner
-  class GodConfiguration < CommonConfiguration
+  class GodConfiguration
     def self.default
       @default ||= GodConfiguration.new
     end
 
-    RUNNER_ATTRIBUTES = [:config_file, :daemonize, :port, :syslog, :events, :options]
+    def self.get
+      config = default.dup
+      config.send :merge_config_file!
+      config
+    end
+
+    RUNNER_ATTRIBUTES = [:config_file, :daemonize, :port, :syslog, :events]
     RUNNER_ATTRIBUTES.each { |att| attr_accessor att }
 
     CONFIG_FILE_ATTRIBUTES = [:process_name, :interval, :stop_timeout, :log_file]
@@ -16,24 +22,40 @@ module SidekiqRunner
       @process_name = 'sidekiq'
       @interval  = 30
       @stop_timeout = 30
-      @log_file   = File.join(Dir.pwd, 'log', 'god.log')
 
+      @log_file   = File.join(Dir.pwd, 'log', 'god.log')
       @config_file  = File.join(Dir.pwd, 'config', 'god.yml')
+
       @daemonize = true
-      # This is going to be a part of the .sock file name e.g. "/tmp/god.17165.sock"
-      # Change this in the configuration file to be able to run multiple instances of god
-      @port = 17165
       @syslog = true
       @events = true
-      @options = {
+
+      # This is going to be a part of the .sock file name e.g. "/tmp/god.17165.sock"
+      # Change this in the configuration file to be able to run multiple instances of god.
+      @port = 17165
+    end
+
+    def options
+      {
         daemonize: @daemonize,
         port: @port,
         syslog: @syslog,
         events: @events,
-        config: File.expand_path("../#{@process_name}.god", __FILE__),
+        config: File.expand_path("../sidekiq.god", __FILE__),
         log: @log_file
       }
     end
 
+    private
+
+    def merge_config_file!
+      if File.exist?(config_file)
+        yml = YAML.load_file(config_file)
+        CONFIG_FILE_ATTRIBUTES.each do |k|
+          v = yml[k] || yml[k.to_s]
+          send("#{k}=", v) unless v.nil?
+        end
+      end
+    end
   end
 end

data/lib/sidekiq-runner/sidekiq.god

@@ -0,0 +1,76 @@
+$LOAD_PATH.unshift "#{File.expand_path('../..', __FILE__)}"
+require 'sidekiq-runner'
+
+sidekiq_config = SidekiqRunner::SidekiqConfiguration.get
+god_config = SidekiqRunner::GodConfiguration.get
+
+God.terminate_timeout = god_config.stop_timeout + 10
+
+sidekiq_config.each do |name, skiq|
+  God.watch do |w|
+    w.name = name
+
+    # Set start command.
+    w.start = skiq.build_start_command
+
+    # Set stop command.
+    w.stop = skiq.build_stop_command(god_config.stop_timeout)
+    w.stop_timeout = god_config.stop_timeout
+
+    # Make sure the pidfile is deleted as sidekiqctl does not delete stale pidfiles.
+    w.pid_file = skiq.pidfile
+    w.behavior(:clean_pid_file)
+
+    # Working directory has to be set properly.
+    # Be aware that by default, God sets the working directory to / (root dir).
+    w.dir = skiq.chdir || Rails.root || Dir.pwd
+
+    # Determine the state on startup.
+    # If process is running move to 'up' state, otherwise move to 'start' state.
+    # States transitions: http://godrb.com/
+    w.transition(:init, { true => :up, false => :start }) do |on|
+      on.condition(:process_running) do |c|
+        c.running = true
+        c.interval = 5
+      end
+    end
+
+    # Determine when process has finished starting.
+    # If process is running move to 'up' state.
+    w.transition([:start, :restart], :up) do |on|
+      on.condition(:process_running) do |c|
+        c.running = true
+        c.interval = 5
+      end
+
+      # If the process went down during state change, try 5 times, on failure move to 'start' state again.
+      on.condition(:tries) do |c|
+        c.times = 3
+        c.interval = 5
+        c.transition = :start
+      end
+    end
+
+    # Start if process is not running.
+    w.transition(:up, :start) do |on|
+      on.condition(:process_running) do |c|
+        c.running = false
+        # Set poll interval in case kqueue/netlink events won't work (you have to
+        # trigger god with root privileges for events to work).
+        c.interval = god_config.interval
+      end
+    end
+
+    w.lifecycle do |on|
+      on.condition(:flapping) do |c|
+        c.to_state = [:start, :restart] # If this watch is started or restarted...
+        c.times = 5                     # 5 times
+        c.within = 5.minutes            # within 5 minutes
+        c.transition = :unmonitored     # then unmonitor it.
+        c.retry_in = 10.minutes         # Then after 10 minutes monitor it again to see if it was just a temporary problem.
+        c.retry_times = 5               # If the process is seen to be flapping 5 times
+        c.retry_within = 2.hours        # within 2 hours, then give up completely.
+      end
+    end
+  end
+end

data/lib/sidekiq-runner/sidekiq_configuration.rb

@@ -1,18 +1,21 @@
-require 'sidekiq-runner/common_configuration'
-require 'fileutils'
+require 'yaml'
 
 module SidekiqRunner
-  class SidekiqConfiguration < CommonConfiguration
+  class SidekiqConfiguration
     include Enumerable
 
-    RUNNER_ATTRIBUTES = [:config_file, :daemonize]
+    RUNNER_ATTRIBUTES = [:config_file]
     RUNNER_ATTRIBUTES.each { |att| attr_accessor att }
 
     attr_reader :sidekiqs
 
     def initialize
-      @config_file = File.join(Dir.pwd, 'config', 'sidekiq.yml')
-      @daemonize   = true
+      @config_file =
+        if defined?(Rails)
+          File.join(Rails.root, 'config', 'sidekiq.yml')  
+        else
+          File.join(Dir.pwd, 'config', 'sidekiq.yml')
+        end
 
       @sidekiqs    = {}
     end
@@ -21,6 +24,13 @@ module SidekiqRunner
       @default ||= SidekiqConfiguration.new
     end
 
+    def self.get
+      config = default.dup
+      config.send :merge_config_file!
+      config.send :sane?
+      config
+    end
+
     def each(&block)
       @sidekiqs.each(&block)
     end
@@ -33,33 +43,34 @@ module SidekiqRunner
       @sidekiqs.empty?
     end
 
-    # DELETEME in the future
-    # Redefined and kept for bakward compatibility
-    def queue(name, weight = 1)
-      # Create a default sidekiq instance when no instances were created
-      fail 'Sidekiq instances hash does not seem to be empty. 
-      It means you are using the newer syntax in the initializer to create at least one instance. 
-      Therefore you should not be using the old queue() function.' unless @sidekiqs.empty?
+    def add_instance(name)
+      fail "Sidekiq instance with the name '#{name}' already exists! No duplicates please." if @sidekiqs.key?(name)
 
-      add_instance('sidekiq_default') do |skiq|
-        skiq.add_queue('default')
-      end
+      @sidekiqs[name] = SidekiqInstance.new(name)
+      yield @sidekiqs[name] if block_given?
     end
 
-    def add_instance(name)
-      skiq = SidekiqInstance.new(name)
-      yield skiq if block_given?
-
-      fail "Sidekick instance with the name of #{name} already exists! No duplicates please." if @sidekiqs.has_key?(name)
+    # Top-level single instance configuration methods, partially for backward compatibility.
 
-      FileUtils.mkdir_p(File.dirname(skiq.pidfile))
-      FileUtils.mkdir_p(File.dirname(skiq.logfile))
+    SidekiqInstance::CONFIG_FILE_ATTRIBUTES.each do |meth|
+      define_method("#{meth}=") do |val|
+        ensure_default_sidekiq!
+        @sidekiqs.each { |_, skiq| skiq.send("#{meth}=", val) }
+      end
+    end
 
-      @sidekiqs[name] = skiq
+    def queue(name, weight = 1)
+      fail 'Multiple Sidekiq instances defined and queue() outside of instance block called.' if @sidekiqs.size > 1
 
-      skiq
+      ensure_default_sidekiq!
+      @sidekiqs.values.first.add_queue(name, weight)
     end
 
+    alias_method :add_queue, :queue
+    alias_method :configfile=, :config_file=
+
+    # Callbacks.
+
     %w(start stop).each do |action|
       %w(success error).each do |state|
         attr_reader "#{action}_#{state}_cb".to_sym
@@ -72,44 +83,25 @@ module SidekiqRunner
 
     private
 
+    def ensure_default_sidekiq!
+      add_instance('sidekiq_default') if empty?
+    end
+
     def merge_config_file!
       sidekiqs_common_config = {}
-      yml = {}
-
-      if File.exist?(config_file)
-        yml = YAML.load_file(config_file)
-        # Get sidekiq config common for all instances
-        SidekiqInstance::CONFIG_FILE_ATTRIBUTES.each do |k|
-          v = yml[k] || yml[k.to_s]
-          sidekiqs_common_config[k] = v if v
-        end
-      end
 
-      # Backwards compatibility
-      # Get sidekiq pidfile and logfile if no sidekiq instance was specified
-      SidekiqInstance::INDIVIDUAL_FILE_ATTRIBUTES.each do |k|
-        v = yml[k] || yml[k.to_s]
-        sidekiqs_common_config[k] = v if v
-      end
+      yml = File.exist?(config_file) ? YAML.load_file(config_file) : {}
+      yml = Hash[yml.map { |k, v| [k.to_sym, v] }]
 
-      @sidekiqs.each do |name, skiq|
-        # Symbolize keys in yml hash
-        if yml[name] && yml[name].is_a?(Hash)
-          yml_config = Hash[yml[name].map { |k, v| [k.to_sym, v] }]
-        end
-        # Merge common and specific sidekiq instance configs
-        # Sidekiq instances not defined in the initializer but present in yml will be ignored
-        final_skiq_config = yml_config ? sidekiqs_common_config.merge(yml_config) : sidekiqs_common_config
-        skiq.merge_config!(final_skiq_config)
-        skiq.sane?
-      end
-
-      self
+      @sidekiqs.each_value { |skiq| skiq.merge_config_file!(yml) }
     end
 
     def sane?
-      fail 'No requirefile given and not in Rails env.' if !defined?(Rails) && !requirefile
       fail 'No sidekiq instances defined. Nothing to run.' if @sidekiqs.empty?
+      fail 'Sidekiq instances with the same pidfile found.' if @sidekiqs.values.map(&:pidfile).uniq.size < @sidekiqs.size
+      fail 'Sidekiq instances with the same logfile found.' if @sidekiqs.values.map(&:logfile).uniq.size < @sidekiqs.size
+
+      @sidekiqs.each_value { |skiq| skiq.sane? }
     end
   end
 end

data/lib/sidekiq-runner/sidekiq_instance.rb

@@ -1,9 +1,8 @@
 module SidekiqRunner
   class SidekiqInstance
 
-    CONFIG_FILE_ATTRIBUTES = [:bundle_env, :chdir, :requirefile, :concurrency, :verbose]
-    INDIVIDUAL_FILE_ATTRIBUTES = [:pidfile, :logfile]
-    CONFIG_FILE_ATTRIBUTES.concat(INDIVIDUAL_FILE_ATTRIBUTES).each { |att| attr_accessor att }
+    CONFIG_FILE_ATTRIBUTES = [:bundle_env, :chdir, :requirefile, :concurrency, :verbose, :pidfile, :logfile]
+    CONFIG_FILE_ATTRIBUTES.each { |att| attr_accessor att }
 
     attr_reader :name, :queues
 
@@ -14,35 +13,68 @@ module SidekiqRunner
       @queues = []
 
       @bundle_env   = true
-      @daemonize    = true
       @chdir        = nil
       @requirefile  = nil
       @pidfile      = File.join(Dir.pwd, 'tmp', 'pids', "#{@name}.pid")
       @logfile      = File.join(Dir.pwd, 'log', "#{@name}.log")
       @concurrency  = 4
       @verbose      = false
-
     end
 
     def add_queue(queue_name, weight = 1)
       fail "Cannot add the queue. The name is empty!" if queue_name.empty?
       fail "Cannot add the queue. The weight is not an integer!" unless weight.is_a? Integer
-      fail "Cannot add the queue. The queue with \"#{queue_name}\" name already exist" if @queues.any?{|arr| arr.first == queue_name}
+      fail "Cannot add the queue. The queue with \"#{queue_name}\" name already exist" if @queues.any? { |q| q.first == queue_name }
       @queues << [queue_name, weight]
     end
 
+    def merge_config_file!(yml)
+      # Get global configuration options.
+      SidekiqInstance::CONFIG_FILE_ATTRIBUTES.each do |k|
+        send("#{k}=", yml[k]) unless yml[k].nil?
+      end
+
+      # Override with instance-specific options.
+      if (syml = yml[@name.to_sym]) && (syml.is_a?(Hash))
+        syml = Hash[syml.map { |k, v| [k.to_sym, v] }]
+
+        SidekiqInstance::CONFIG_FILE_ATTRIBUTES.each do |k|
+          send("#{k}=", syml[k]) unless syml[k].nil?
+        end
+      end
+    end
+
     def sane?
       fail "No queues given for #{@name}!" if @queues.empty?  
+      fail "No requirefile given for #{@name} and not in Rails environment!" if !defined?(Rails) && !requirefile
     end
 
-    def merge_config!(skiq)
-        CONFIG_FILE_ATTRIBUTES.each do |k|
-          v = nil || skiq[k]
-          send("#{k}=", v) unless v.nil?
-        end
+    def build_start_command
+      cmd = []
+      cmd << (bundle_env ? 'bundle exec sidekiq' : 'sidekiq')
+      cmd << '-d'
+      cmd << "-c #{concurrency}"
+      cmd << '-v' if verbose
+      cmd << "-L #{logfile}"
+      cmd << "-P #{pidfile}"
+      cmd << "-e #{Rails.env}" if defined?(Rails)
+      cmd << "-r #{requirefile}" if requirefile
+
+      queues.each do |q, w|
+        cmd << "-q #{q},#{w.to_s}"
+      end
 
-      self
+      cmd.join(' ')
     end
 
+    def build_stop_command(timeout)
+      cmd = []
+      cmd << (bundle_env ? 'bundle exec sidekiqctl' : 'sidekiqctl')
+      cmd << 'stop'
+      cmd << pidfile
+      cmd << timeout
+
+      cmd.join(' ')
+    end
   end
-end
+end

data/lib/sidekiq-runner/tasks.rb

@@ -3,18 +3,18 @@ require 'sidekiq-runner'
 rails_env = Rake::Task.task_defined?(:environment) ? [:environment] : []
 
 namespace :sidekiqrunner do
-  desc 'Start sidekiq-runner instances'
+  desc 'Start Sidekiq instances'
   task start: rails_env do
     puts 'Starting sidekiq instances...'
     SidekiqRunner.start
   end
 
-  desc 'Stop sidekiq-runner instances'
+  desc 'Stop Sidekiq instances'
   task stop: rails_env do
     puts 'Gracefully shutting down sidekiq instances...'
     SidekiqRunner.stop
   end
 
-  desc 'Restart sidekiq-runner instances'
-  task restart: ['sidekiq:stop', 'sidekiq:start']
+  desc 'Restart Sidekiq instances'
+  task restart: ['sidekiqrunner:stop', 'sidekiqrunner:start']
 end

data/lib/sidekiq-runner/version.rb

@@ -1,3 +1,3 @@
 module SidekiqRunner
-  VERSION = '0.0.3'
+  VERSION = '0.0.4'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-runner
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - FlavourSys Technology GmbH
@@ -38,22 +38,21 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.13'
-description: A little collection of Sidekiq start/stop rake tasks and configuration
-  framework
+description: Provide an easy way to configure, start, and monitor all your Sidekiq
+  processes
 email: technology@flavoursys.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - lib/sidekiq-runner.rb
-- lib/sidekiq-runner/common_configuration.rb
 - lib/sidekiq-runner/god_configuration.rb
-- lib/sidekiq-runner/sidekiq_command_builder.rb
+- lib/sidekiq-runner/sidekiq.god
 - lib/sidekiq-runner/sidekiq_configuration.rb
 - lib/sidekiq-runner/sidekiq_instance.rb
 - lib/sidekiq-runner/tasks.rb
 - lib/sidekiq-runner/version.rb
-homepage: http://flavoursys.com
+homepage: https://github.com/FlavourSys/sidekiq-runner
 licenses:
 - MIT
 metadata: {}
@@ -76,5 +75,5 @@ rubyforge_project:
 rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
-summary: Sidekiq rake task collection
+summary: Sidekiq configuration and rake tasks
 test_files: []

data/lib/sidekiq-runner/common_configuration.rb

@@ -1,46 +0,0 @@
-require 'yaml'
-
-module SidekiqRunner
-
-  # common functions for GodConfiguration and Configuration classes
-  class CommonConfiguration
-    # this method MUST be reimplemented
-    def self.default
-      fail(NotImplementedError, "Method \"#{__method__}\" is not implemented in \"#{name}\" class. Please DO implement this method to be able to derive from CommonConfiguration class.")
-    end
-
-    RUNNER_ATTRIBUTES = []
-    CONFIG_FILE_ATTRIBUTES = []
-
-    @config_file = ''
-
-    def to_hash
-      Hash[CONFIG_FILE_ATTRIBUTES.map { |att| [att, send(att)] }]
-    end
-
-    def self.get
-      config = default.dup
-      config.send :merge_config_file!
-      config.send :sane?
-      config
-    end
-
-    private
-
-    def merge_config_file!
-      if File.exist?(config_file)
-        yml = YAML.load_file(config_file)
-        CONFIG_FILE_ATTRIBUTES.each do |k|
-          v = yml[k] || yml[k.to_s]
-          send("#{k}=", v) unless v.nil?
-        end
-      end
-
-      self
-    end
-
-    def sane?
-    end
-
-  end # class
-end # module

data/lib/sidekiq-runner/sidekiq_command_builder.rb

@@ -1,31 +0,0 @@
-module SidekiqCommandBuilder
-
-  def self.build_start_command(sidekiq_config, skiq)
-    cmd = []
-    cmd << (skiq.bundle_env ? 'bundle exec sidekiq' : 'sidekiq')
-    cmd << '-d' if sidekiq_config.daemonize
-    cmd << "-c #{skiq.concurrency}"
-    cmd << "-e #{Rails.env}" if defined?(Rails)
-    cmd << '-v' if skiq.verbose
-    cmd << "-L #{skiq.logfile}"
-    cmd << "-P #{skiq.pidfile}"
-    cmd << "-r #{skiq.requirefile}" if skiq.requirefile
-
-    skiq.queues.each do |q, w|
-      cmd << "-q #{q},#{w.to_s}"
-    end
-
-    cmd.join(' ')
-  end
-
-  def self.build_stop_command(skiq, timeout)
-    cmd = []
-    cmd << (skiq.bundle_env ? 'bundle exec sidekiqctl' : 'sidekiqctl')
-    cmd << 'stop'
-    cmd << skiq.pidfile
-    cmd << timeout
-
-    cmd.join(' ')
-  end
-
-end