resque-integration 3.4.1

data/lib/resque/integration/configuration.rb

@@ -0,0 +1,238 @@
+# coding: utf-8
+
+require 'yaml'
+require 'ostruct'
+require 'erb'
+
+require 'active_support/core_ext/hash/keys'
+require 'active_support/core_ext/hash/deep_merge'
+
+module Resque
+  module Integration
+    class Configuration
+      # Worker entity
+      class Worker < OpenStruct
+        def initialize(queue, config)
+          data = {:queue => queue}
+
+          if config.is_a?(Hash)
+            data.merge!(config.symbolize_keys)
+          else
+            data[:count] = config
+          end
+
+          super(data)
+        end
+
+        # Returns workers count for given queue
+        def count
+          [super || 1, 1].max
+        end
+
+        # Returns hash of ENV variables that should be associated with this worker
+        def env
+          env = super || {}
+
+          env[:QUEUE] ||= queue
+          env[:JOBS_PER_FORK] ||= jobs_per_fork if jobs_per_fork
+          env[:MINUTES_PER_FORK] ||= minutes_per_fork if minutes_per_fork
+          env[:SHUFFLE] ||= 1 if shuffle
+
+          Hash[env.map { |k, v| [k, v.to_s] }]
+        end
+      end
+
+      # Failure notifier configuration
+      class Notifier < OpenStruct
+        def initialize(config)
+          super(config || {})
+        end
+
+        # Is notifier enabled
+        def enabled?
+          to.any? && enabled.nil? ? true : enabled
+        end
+
+        # Returns true if payload should be included into reports
+        def include_payload?
+          include_payload.nil? ?
+            true :
+            include_payload
+        end
+
+        # Returns recipients list
+        def to
+          super || []
+        end
+
+        # Returns sender address
+        def from
+          super || 'no_reply@gmail.com'
+        end
+
+        # Returns mailer method
+        def mail
+          (super || :alert).to_sym
+        end
+
+        # Returns mailer class
+        def mailer
+          super || 'ResqueFailedJobMailer::Mailer'
+        end
+      end
+
+      # Create configuration from given +paths+
+      def initialize(*paths)
+        @configuration = {}
+        paths.each { |f| load f }
+      end
+
+      # Returns Resque redis configuration
+      #
+      # @return [OpenStruct]
+      def redis
+        @redis ||= (self['redis'] || {}).symbolize_keys
+      end
+
+      # Returns workers configuration
+      #
+      # @return [Array<Worker>]
+      def workers
+        @workers ||= (self[:workers] || {}).map { |k, v| Worker.new(k, v) }
+      end
+
+      # Returns failure notifier config
+      def failure_notifier
+        @notifier ||= Notifier.new(self['failure.notifier'])
+      end
+
+      # Returns flag for cleaning on shutdown see https://github.com/resque/resque/issues/1167
+      def run_at_exit_hooks?
+        value = self['resque.run_at_exit_hooks']
+
+        if value.is_a?(String) && %w(n no false off disabled).include?(value)
+          value = false
+        end
+
+        value.nil? ? true : value
+      end
+
+      # Returns Resque polling interval
+      def interval
+        (self['resque.interval'] || 5).to_i
+      end
+
+      # Returns Resque verbosity level
+      def verbosity
+        (self['resque.verbosity'] || 0).to_i
+      end
+
+      # Returns Resque log level
+      def log_level
+        (self['resque.log_level'] || 1).to_i
+      end
+
+      # Returns path to resque log file
+      def log_file
+        self['resque.log_file'] || ::Rails.root.join('log/resque.log').to_s
+      end
+
+      def config_file
+        self['resque.config_file'] || ::Rails.root.join('config/resque.god').to_s
+      end
+
+      def pid_file
+        "#{pids}/resque-god.pid"
+      end
+
+      def pids
+        self['resque.pids'] || ::Rails.root.join('tmp/pids').to_s
+      end
+
+      def root
+        self['resque.root'] || ::Rails.root.to_s
+      end
+
+      # Путь до файла с расписание resque schedule
+      #
+      # Returns String
+      def schedule_file
+        self['resque.schedule_file'] || ::Rails.root.join('config', 'resque_schedule.yml')
+      end
+
+      # Есть ли расписание у приложения?
+      #
+      # Returns boolean
+      def schedule_exists?
+        return @schedule_exists if defined?(@schedule_exists)
+        @schedule_exists = File.exist?(schedule_file)
+      end
+
+      # Используется ли resque scheduler
+      #
+      # Returns boolean
+      def resque_scheduler?
+        value = self['resque.scheduler'] || true
+
+        if value.is_a?(String) && %w(n no false off disabled).include?(value)
+          value = false
+        end
+
+        value
+      end
+
+      # Returns maximum terminate timeout
+      def terminate_timeout
+        workers.map(&:stop_timeout).compact.max.to_i + 10
+      end
+
+      # Returns environment variables that should be associated with this configuration
+      def env
+        env = self['env'] || {}
+
+        env[:INTERVAL] ||= interval
+
+        Hash[env.map { |k, v| [k, v.to_s] }]
+      end
+
+      # Generate GOD configuration file
+      def to_god
+        template = ERB.new(File.read(File.join(File.dirname(__FILE__), 'god.erb')))
+        template.result(binding)
+      end
+
+      def god_log_file
+        self['resque.god_log_file'] || ::Rails.root.join('log/god.log').to_s
+      end
+
+      def god_log_level
+        self['resque.god_log_level'] || 'info'
+      end
+
+      private
+      def load(path)
+        if File.exists?(path)
+          input = File.read(path)
+          input = ERB.new(input).result if defined?(ERB)
+          config = YAML.load(input)
+
+          @configuration.merge!(config)
+        end
+      end
+
+      # get value from configuration
+      def [](path)
+        parts = path.to_s.split('.')
+        result = @configuration
+
+        parts.each do |k|
+          result = result[k]
+
+          break if result.nil?
+        end
+
+        result
+      end
+    end # class Configuration
+  end # module Integration
+end # module Resque

data/lib/resque/integration/continuous.rb

@@ -0,0 +1,75 @@
+module Resque
+  module Integration
+    # Continuous job can re-enqueue self with respect to resque-lock and resque-meta.
+    #
+    # @example
+    #   class ResqueJob
+    #     include Resque::Integration
+    #
+    #     unique
+    #     continuous
+    #
+    #     def self.execute(id)
+    #       chunk = Company.find(id).products.limit(1000)
+    #
+    #       if chunk.size > 0
+    #         heavy_work
+    #         continue # it will re-enqueue the job with the same arguments. Avoid infinite loops!
+    #       end
+    #     end
+    #   end
+    module Continuous
+      # Remove any locks if needed and re-enqueue job with the same arguments
+      def continue(*args)
+        @continued = args
+      end
+      private :continue # one should not call it from outside
+
+      # This callback resets Meta's finish flags
+      def after_perform_reset_meta(*)
+        if should_reset_meta?
+          meta_obj = meta
+
+          meta_obj.data.delete('succeeded')
+          meta_obj.data.delete('finished_at')
+
+          meta_obj.save
+        end
+      end
+
+      # Just to ensure that previous jobs won't affect current
+      def before_perform_continue(*)
+        @continued = nil
+      end
+
+      # `after` callbacks are executed after `around` callbacks
+      # so here we can re-enqueue the job, because lock (from resque-lock) already released
+      def after_perform_continue(*args)
+        if continued?
+          args = if @continued.any?
+            if unique?
+              meta_id = args.first # we should keep meta_id as first argument
+              [meta_id] + @continued
+            else
+              @continued
+            end
+          else
+            args
+          end
+
+          ::Resque.enqueue(self, *args)
+        end
+      end
+
+      private
+
+      def continued?
+        !@continued.nil?
+      end
+
+      def should_reset_meta?
+        continued? && unique? && meta
+      end
+    end # module Continuous
+  end # module Integration
+end # module Resque

data/lib/resque/integration/engine.rb

@@ -0,0 +1,103 @@
+require 'redis'
+require 'redis/version'
+require 'rails/engine'
+require 'active_support/core_ext/string/inflections'
+
+module Resque::Integration
+  # Rails engine
+  # @see http://guides.rubyonrails.org/engines.html
+  class Engine < Rails::Engine
+    rake_tasks do
+      load 'resque/integration/tasks/hooks.rake'
+      load 'resque/integration/tasks/resque.rake'
+      load 'resque/integration/tasks/lock.rake'
+    end
+
+    # Читает конфиг-файлы config/resque.yml и config/resque.local.yml,
+    # мерджит результаты и сохраняет в Resque.config
+    initializer 'resque-integration.config' do
+      paths = Rails.root.join('config', 'resque.yml'),
+              Rails.root.join('config', 'resque.local.yml')
+
+      Resque.config = Resque::Integration::Configuration.new(*paths.map(&:to_s))
+    end
+
+    # Читает конфиг-файл config/resque_schedule.yml
+    initializer 'resque-integration.schedule_config' do
+      if Resque.config.resque_scheduler? && Resque.config.schedule_exists?
+        config = ERB.new(File.read(Resque.config.schedule_file)).result
+        Resque.schedule = YAML.load(config)
+      end
+    end
+
+    # Устанавливает для Resque соединение с Редисом,
+    # данные берутся из конфига (см. выше)
+    initializer 'resque-integration.redis' do
+      redis = Resque.config.redis
+
+      if redis.any?
+        Resque.redis = Redis.new(redis)
+        Resque.redis.namespace = redis[:namespace] if redis[:namespace]
+      end
+    end
+
+    initializer 'resque-integration.logger' do
+      Resque.logger.level = Resque.config.log_level
+
+      case Resque.config.verbosity
+      when 1
+        Resque.logger.formatter = Resque::VerboseFormatter.new
+      when 2
+        Resque.logger.formatter = Resque::VeryVerboseFormatter.new
+      else
+        Resque.logger.formatter = Resque::QuietFormatter.new
+      end
+    end
+
+    # Конфигурирование плагина resque-failed-job-mailer.
+    # Данные берутся из конфига (см. выше)
+    initializer 'resque-integration.failure_notifier' do
+      notifier = Resque.config.failure_notifier
+
+      if notifier.enabled?
+        require 'resque_failed_job_mailer'
+
+        Resque::Failure::Notifier.configure do |config|
+          config.to = notifier.to
+          config.from = notifier.from
+          config.include_payload = notifier.include_payload?
+          config.mail = notifier.mail
+          config.mailer = notifier.mailer.constantize
+        end
+      end
+    end
+
+    # Глушим ошибки, по которым происходит автоматический перезапуск
+    initializer 'resque-integration.retrys' do
+      require 'resque/failure'
+      require 'resque/failure/redis'
+
+      Resque::Failure::MultipleWithRetrySuppression.classes = [
+        Resque::Failure::Redis,
+        Resque::Integration::FailureBackends::QueuesTotals
+      ]
+
+      if Resque.config.failure_notifier.enabled?
+        require 'resque_failed_job_mailer'
+
+        Resque::Failure::MultipleWithRetrySuppression.classes << Resque::Failure::Notifier
+      end
+
+      Resque::Failure.backend = Resque::Failure::MultipleWithRetrySuppression
+    end
+
+    initializer "resque-integration.extensions" do
+      ::Resque::Worker.prepend ::Resque::Integration::Extensions::Worker
+      ::Resque::Job.singleton_class.prepend ::Resque::Integration::Extensions::Job
+    end
+
+    initializer 'resque-integration.application', before: :load_config_initializers do |app|
+      app.config.resque_job_status = {route_constraints: {domain: :current}, route_path: '/_job_'}
+    end
+  end # class Engine
+end # module Resque::Integration

data/lib/resque/integration/extensions/job.rb

@@ -0,0 +1,17 @@
+module Resque
+  module Integration
+    module Extensions
+      # Public: extension for proper determine queue
+      # when destroy job with priority
+      module Job
+        def destroy(queue, klass, *args)
+          if klass.respond_to?(:priority?) && klass.priority?
+            queue = klass.priority_queue(args.last)
+          end
+
+          super
+        end
+      end
+    end
+  end
+end

data/lib/resque/integration/extensions/worker.rb

@@ -0,0 +1,17 @@
+module Resque
+  module Integration
+    module Extensions
+      module Worker
+        def queues
+          queues = super
+          shuffle? ? queues.shuffle : queues
+        end
+
+        def shuffle?
+          return @shuffle if defined?(@shuffle)
+          @shuffle = !ENV['SHUFFLE'].to_s.empty?
+        end
+      end
+    end
+  end
+end

data/lib/resque/integration/extensions.rb

@@ -0,0 +1,8 @@
+module Resque
+  module Integration
+    module Extensions
+      autoload :Worker, "resque/integration/extensions/worker"
+      autoload :Job, "resque/integration/extensions/job"
+    end
+  end
+end

data/lib/resque/integration/failure_backends/queues_totals.rb

@@ -0,0 +1,37 @@
+module Resque
+  module Integration
+    module FailureBackends
+      class QueuesTotals < ::Resque::Failure::Base
+        REDIS_COUNTER_KEY = 'resque:integration:failure_backends:queues_totals'.freeze
+        MAX_COUNTER_VALUE = 10_000_000
+
+        private_constant :REDIS_COUNTER_KEY
+
+        def save
+          current_value = Resque.redis.hincrby(REDIS_COUNTER_KEY, queue, 1)
+          Resque.redis.hset(REDIS_COUNTER_KEY, queue, 1) if current_value >= MAX_COUNTER_VALUE
+        end
+
+        def self.queues
+          Resque.redis.hkeys(REDIS_COUNTER_KEY)
+        end
+
+        def self.count(queue = nil, _class_name = nil)
+          if queue.nil?
+            Resque.redis.hvals(REDIS_COUNTER_KEY).map(&:to_i).sum
+          else
+            Resque.redis.hget(REDIS_COUNTER_KEY, queue).to_i
+          end
+        end
+
+        def self.clear(queue = nil)
+          if queue.nil?
+            Resque.redis.del(REDIS_COUNTER_KEY)
+          else
+            Resque.redis.hdel(REDIS_COUNTER_KEY, queue)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/resque/integration/failure_backends.rb

@@ -0,0 +1,7 @@
+module Resque
+  module Integration
+    module FailureBackends
+      autoload :QueuesTotals, 'resque/integration/failure_backends/queues_totals'
+    end
+  end
+end

data/lib/resque/integration/god.erb

@@ -0,0 +1,99 @@
+# THIS FILE IS AUTO-GENERATED. PLEASE DO NOT CHANGE IT!
+
+bundle = '<%= `which bundle`.strip %>'
+rails_root = '<%= root %>'
+God.pid_file_directory = '<%= pids %>'
+God.terminate_timeout = <%= terminate_timeout.to_i %>
+
+<% workers.each do |worker| %>
+  <% worker.count.times do |i| %>
+    God.watch do |w|
+      w.dir = rails_root
+      w.name = 'resque-<%= worker.name || worker.queue %>-<%= i %>'
+      w.group = 'resque'
+      w.interval = 30.seconds
+      w.env = <%= Resque.config.env.merge(worker.env).merge(:RAILS_ENV => ::Rails.env, :BUNDLE_GEMFILE => "#{root}/Gemfile").stringify_keys! %>
+      w.start = "#{bundle} exec rake resque:work"
+      w.log = '<%= log_file %>'
+      w.stop_signal = 'QUIT'
+
+      <% if worker.stop_timeout %>
+        w.stop_timeout = <%= worker.stop_timeout %>
+      <% end %>
+
+      # determine the state on startup
+      w.transition(:init, { true => :up, false => :start }) do |on|
+        on.condition(:process_running) do |c|
+          c.running = true
+        end
+      end
+
+      # determine when process has finished starting
+      w.transition([:start, :restart], :up) do |on|
+        on.condition(:process_running) do |c|
+          c.running = true
+          c.interval = 5.seconds
+        end
+
+        # failsafe
+        on.condition(:tries) do |c|
+          c.times = 5
+          c.transition = :start
+          c.interval = 5.seconds
+        end
+      end
+
+      # start if process is not running
+      w.transition(:up, :start) do |on|
+        on.condition(:process_running) do |c|
+          c.running = false
+        end
+      end
+      # END OF WORKER CONFIGURATION
+    end
+  <% end %>
+<% end %>
+
+<% if resque_scheduler? %>
+  God.watch do |w|
+    w.dir = rails_root
+    w.name = 'resque-scheduler'
+    w.group = 'resque'
+    w.interval = 30.seconds
+    w.env = <%= Resque.config.env.merge(:RAILS_ENV => ::Rails.env, :BUNDLE_GEMFILE => "#{root}/Gemfile").stringify_keys! %>
+    w.start = "#{bundle} exec rake environment resque:scheduler"
+    w.log = '<%= log_file %>'
+    w.stop_signal = 'QUIT'
+    w.stop_timeout = 60.seconds
+
+    # determine the state on startup
+    w.transition(:init, { true => :up, false => :start }) do |on|
+      on.condition(:process_running) do |c|
+        c.running = true
+      end
+    end
+
+    # determine when process has finished starting
+    w.transition([:start, :restart], :up) do |on|
+      on.condition(:process_running) do |c|
+        c.running = true
+        c.interval = 5.seconds
+      end
+
+      # failsafe
+      on.condition(:tries) do |c|
+        c.times = 5
+        c.transition = :start
+        c.interval = 5.seconds
+      end
+    end
+
+    # start if process is not running
+    w.transition(:up, :start) do |on|
+      on.condition(:process_running) do |c|
+        c.running = false
+      end
+    end
+    # END OF SCHEDULE CONFIGURATION
+  end
+<% end %>

data/lib/resque/integration/hooks.rb

@@ -0,0 +1,72 @@
+# coding: utf-8
+
+module Resque
+  module Integration
+    # Backport of resque-1.24.x hooks
+    # @see https://github.com/resque/resque/pull/680/files
+    module Hooks
+      # We do not want to patch Worker, so we declare callable Array
+      class CallableArray < Array
+        def call(*args)
+          each { |hook| hook.call(*args) }
+        end
+      end
+      # Call with a block to register a hook.
+      # Call with no arguments to return all registered hooks.
+      def before_first_fork(&block)
+        block ? register_hook(:before_first_fork, block) : hooks(:before_first_fork)
+      end
+
+      # Register a before_first_fork proc.
+      def before_first_fork=(block)
+        register_hook(:before_first_fork, block)
+      end
+
+      # Call with a block to register a hook.
+      # Call with no arguments to return all registered hooks.
+      def before_fork(&block)
+        block ? register_hook(:before_fork, block) : hooks(:before_fork)
+      end
+
+      # Register a before_fork proc.
+      def before_fork=(block)
+        register_hook(:before_fork, block)
+      end
+
+      # Call with a block to register a hook.
+      # Call with no arguments to return all registered hooks.
+      def after_fork(&block)
+        block ? register_hook(:after_fork, block) : hooks(:after_fork)
+      end
+
+      # Register an after_fork proc.
+      def after_fork=(block)
+        register_hook(:after_fork, block)
+      end
+
+      private
+
+      # Register a new proc as a hook. If the block is nil this is the
+      # equivalent of removing all hooks of the given name.
+      #
+      # `name` is the hook that the block should be registered with.
+      def register_hook(name, block)
+        return clear_hooks(name) if block.nil?
+
+        @hooks ||= {}
+        @hooks[name] ||= CallableArray.new
+        @hooks[name] << block
+      end
+
+      # Clear all hooks given a hook name.
+      def clear_hooks(name)
+        @hooks && @hooks[name] = CallableArray.new
+      end
+
+      # Retrieve all hooks of a given name.
+      def hooks(name)
+        (@hooks && @hooks[name]) || CallableArray.new
+      end
+    end # module Hooks
+  end # module Integration
+end # module Resque