resque-integration 3.4.1

data/lib/resque/integration/tasks/resque.rake

@@ -0,0 +1,101 @@
+namespace :resque do
+  desc 'Generate God configuration file'
+  task :conf => :environment do
+    File.write(Resque.config.config_file, Resque.config.to_god)
+
+    puts "God configuration file generated to #{Resque.config.config_file}"
+  end
+
+  desc 'Start God server and watch for Resque workers'
+  task :start => :conf do
+    if god_running?
+      puts `#{god} start resque`
+    else
+      puts `#{god} -c #{Resque.config.config_file} -P #{Resque.config.pid_file} --log #{Resque.config.god_log_file} --log-level #{Resque.config.god_log_level} --no-syslog`
+    end
+  end
+
+  desc 'Restart Resque workers'
+  task :restart => :conf do
+    if god_stopped?
+      Rake::Task['resque:start'].invoke
+    else
+      puts `#{god} load #{Resque.config.config_file} stop && #{god} restart resque`
+    end
+  end
+
+  desc 'Stop Resque workers'
+  task :stop do
+    puts `#{god} stop resque`
+  end
+
+  desc 'Stop Resque workers and quit God'
+  task :terminate do
+    puts `#{god} terminate`
+  end
+
+  desc 'Stop processing any new jobs'
+  task :pause do
+    puts `#{god} signal resque USR2`
+  end
+
+  desc 'Resume jobs processing after pause'
+  task :resume do
+    puts `#{god} signal resque CONT`
+  end
+
+  desc 'Shows Resque status'
+  task :status do
+    puts `#{god} status resque`
+  end
+
+  desc 'Установить время истечения resque:resque-retry:*'
+  task :expire => :environment do
+    cursor = 0
+    redis = Redis.current
+
+    loop do
+      cursor, keys = redis.scan(cursor, count: 10_000, match: 'resque:resque-retry:*')
+      cursor = cursor.to_i
+
+      unless keys.empty?
+        redis.pipelined do
+          keys.each { |key| redis.expire(key, 1.hour.seconds) }
+        end
+      end
+
+      break if cursor.zero?
+    end
+  end
+
+  namespace :logs do
+    desc 'Rotate resque logs'
+    task :rotate => :environment do
+      if god_running?
+        Process.kill('USR1', File.read(Resque.config.pid_file).to_i)
+        sleep 3
+        puts `#{god} signal resque HUP`
+      else
+        puts 'god is not running'
+      end
+    end
+  end
+
+  private
+
+  def god
+    `which god`.strip
+  end
+
+  def god_running?
+    File.exists?(Resque.config.pid_file) && Process.kill(0, File.read(Resque.config.pid_file).to_i)
+  rescue Errno::ESRCH
+    false
+  rescue Errno::EPERM
+    true
+  end
+
+  def god_stopped?
+    !god_running?
+  end
+end

data/lib/resque/integration/unique.rb

@@ -0,0 +1,218 @@
+require 'digest/sha1'
+require 'active_support/core_ext/module/aliasing'
+require 'active_support/core_ext/hash'
+require 'resque/plugins/progress'
+
+module Resque
+  module Integration
+    # Unique job
+    #
+    # @example
+    #   class MyJob
+    #     include Resque::Integration
+    #
+    #     # jobs are considered as equal if their first argument is the same
+    #     unique { |*args| args.first }
+    #
+    #     def self.execute(image_id)
+    #       # do it
+    #     end
+    #   end
+    #
+    #   MyJob.enqueue(11)
+    module Unique
+      LOCK_TIMEOUT = 259_200 # 3 days
+
+      def self.extended(base)
+        if base.singleton_class.include?(::Resque::Integration::Priority)
+          raise 'Uniqueness should be enabled before Prioritness'
+        end
+
+        base.extend(::Resque::Plugins::Progress)
+        base.singleton_class.prepend(Overrides)
+      end
+
+      module Overrides
+        # Overriding +enqueue+ method here so now it returns existing metadata if job already queued
+        def enqueue(*args) #:nodoc:
+          meta = enqueued?(*args)
+          return meta if meta
+
+          # enqueue job and retrieve its meta
+          super
+        end
+
+        # Overriding +meta_id+ here so now it generates the same MetaID for Jobs with same args
+        def meta_id(*args)
+          ::Digest::SHA1.hexdigest([secret_token, self, lock_on.call(*args)].join)
+        end
+      end
+
+      # Returns true because job is unique now
+      def unique?
+        true
+      end
+
+      # Метод вызывает resque-scheduler чтобы поставить задание в текущую очередь
+      def scheduled(queue, klass, *args)
+        klass.constantize.enqueue_to(queue, *args)
+      end
+
+      # Метод вызывает resque-retry когда ставить отложенное задание
+      # здесь мы убираем meta_id из аргументов
+      def retry_args(meta_id, *args)
+        args
+      end
+
+      # Метод вызывает resque-retry, когда записывает/читает число перезапусков
+      #  - во время работы воркера первым аргументом передается meta_id;
+      #  - во время чтения из вебинтерфейса, meta_id не передается, т.к. она выкидывается во время перепостановки
+      #  джоба(см retry_args);
+      #  - если метод вызывается в пользовательском коде(и @meta_id отсутствует), то meta_id нельзя передавать.
+      def retry_identifier(*args)
+        return if args.empty?
+        args.shift if @meta_id.is_a?(String) && !@meta_id.empty? && @meta_id == args.first
+        lock_id(*args)
+      end
+
+      # Get or set proc returning unique arguments
+      def lock_on(&block)
+        if block_given?
+          @unique = block
+        else
+          @unique ||= proc { |*args| args }
+        end
+      end
+
+      # LockID should be independent from MetaID
+      # @api private
+      def lock_id(*args)
+        args = args.map { |i| i.is_a?(Hash) ? i.with_indifferent_access : i }
+        locked_args = lock_on.call(*args)
+        encoded_args = ::Digest::SHA1.hexdigest(obj_to_string(locked_args))
+        "lock:#{name}-#{encoded_args}"
+      end
+
+      # get meta object associated with job
+      def meta
+        get_meta(@meta_id)
+      end
+
+      # default `perform` method override
+      def perform(meta_id, *args)
+        execute(*args)
+      end
+
+      def execute(*)
+        raise NotImplementedError, "You should implement `execute' method"
+      end
+
+      # When job is failed we should remove lock
+      def on_failure_lock(_e, _meta_id, *args)
+        unlock(*args)
+      end
+
+      # Before dequeue check if job is running
+      def before_dequeue_lock(*args)
+        (meta_id = args.first) &&
+        (meta = get_meta(meta_id)) &&
+        !meta.working?
+      end
+
+      def on_failure_retry(exception, *args)
+        return unless defined?(super)
+
+        # Keep meta_id if kill -9 (or ABRT)
+        @meta_id = args.first if exception.is_a?(::Resque::DirtyExit)
+
+        super
+      end
+
+      # Before enqueue acquire a lock
+      #
+      # Returns boolean
+      def before_enqueue_lock(_meta_id, *args)
+        ::Resque.redis.set(lock_id(*args), 1, ex: lock_timeout, nx: true)
+      end
+
+      def around_perform_lock(_meta_id, *args)
+        yield
+      ensure
+        # Always clear the lock when we're done, even if there is an error.
+        unlock(*args)
+      end
+
+      # When job is dequeued we should remove lock
+      def after_dequeue_lock(_meta_id, *args)
+        unlock(*args) unless args.empty?
+      end
+
+      # Fail metadata if dequeue succeed
+      def after_dequeue_meta(*args)
+        if (meta_id = args.first) && (meta = get_meta(meta_id))
+          meta.fail!
+        end
+      end
+
+      # Is job already in queue or in process?
+      def enqueued?(*args)
+        # if lock exists and timeout not exceeded
+        get_meta(meta_id(*args)) if locked?(*args)
+      end
+
+      def lock_timeout
+        LOCK_TIMEOUT
+      end
+
+      # Returns true if resque job is in locked state
+      def locked?(*args)
+        ::Resque.redis.exists(lock_id(*args))
+      end
+
+      # Dequeue unique job
+      def dequeue(*args)
+        ::Resque.dequeue(self, meta_id(*args), *args)
+      end
+
+      def enqueue_to(queue, *args)
+        meta = enqueued?(*args)
+        return meta if meta.present?
+
+        meta = ::Resque::Plugins::Meta::Metadata.new('meta_id' => meta_id(args), 'job_class' => to_s)
+        meta.save
+
+        ::Resque.enqueue_to(queue, self, meta.meta_id, *args)
+        meta
+      end
+
+      private
+
+      # Remove lock for job with given +args+
+      def unlock(*args)
+        ::Resque.redis.del(lock_id(*args))
+      end
+
+      def secret_token
+        ::Rails.respond_to?(:application) &&
+        ::Rails.application &&
+        ::Rails.application.config.secret_token
+      end
+
+      def obj_to_string(obj)
+        case obj
+        when Hash
+          s = []
+          obj.keys.sort.each do |k|
+            s << obj_to_string(k)
+            s << obj_to_string(obj[k])
+          end
+          s.to_s
+        when Array
+          obj.map { |a| obj_to_string(a) }.to_s
+        else
+          obj.to_s
+        end
+      end
+    end # module Unique
+  end # module Integration
+end # module Resque

data/lib/resque/integration/version.rb

@@ -0,0 +1,5 @@
+module Resque
+  module Integration
+    VERSION = '3.4.1'.freeze
+  end
+end

data/lib/resque/integration.rb

@@ -0,0 +1,146 @@
+require 'resque/integration/version'
+
+require 'active_support/all'
+require 'rails/railtie'
+require 'active_record'
+require 'action_pack'
+
+require 'rake'
+require 'multi_json'
+
+require 'resque'
+silence_warnings { require 'resque/plugins/meta' }
+
+require 'resque/integration/monkey_patch/verbose_formatter'
+require 'resque/integration/hooks'
+
+require 'resque/scheduler'
+require 'resque/scheduler/tasks'
+require 'resque-retry'
+
+module Resque
+  include Integration::Hooks
+  extend Integration::Hooks
+
+  # Resque.config is available now
+  mattr_accessor :config
+
+  def queues_info
+    return @queues_info if defined?(@queues_info)
+
+    queues_info_config = Rails.root.join('config', 'resque_queues.yml')
+
+    @queues_info = Resque::Integration::QueuesInfo.new(config: queues_info_config)
+  end
+
+  # Seamless resque integration with all necessary plugins
+  # You should define an +execute+ method (not +perform+)
+  #
+  # Usage:
+  #   class MyJob
+  #     include Resque::Integration
+  #
+  #     queue :my_queue
+  #     unique ->(*args) { args.first }
+
+  #     def self.execute(*args)
+  #     end
+  #   end
+  module Integration
+    autoload :Backtrace, 'resque/integration/backtrace'
+    autoload :CLI, 'resque/integration/cli'
+    autoload :Configuration, 'resque/integration/configuration'
+    autoload :Continuous, 'resque/integration/continuous'
+    autoload :Unique, 'resque/integration/unique'
+    autoload :Ordered, 'resque/integration/ordered'
+    autoload :LogsRotator, 'resque/integration/logs_rotator'
+    autoload :QueuesInfo, 'resque/integration/queues_info'
+    autoload :Extensions, 'resque/integration/extensions'
+    autoload :FailureBackends, 'resque/integration/failure_backends'
+    autoload :Priority, 'resque/integration/priority'
+
+    extend ActiveSupport::Concern
+
+    included do
+      extend Backtrace
+
+      @queue ||= :default
+    end
+
+    module ClassMethods
+      # Get or set queue name (just a synonym to resque native methodology)
+      def queue(name = nil)
+        if name
+          @queue = name
+        else
+          @queue
+        end
+      end
+
+      # Mark Job as unique and set given +callback+ or +block+ as Unique Arguments procedure
+      def unique(callback = nil, &block)
+        extend Unique unless unique?
+
+        lock_on(&(callback || block))
+      end
+
+      # Extend job with 'continuous' functionality so you can re-enqueue job with +continue+ method.
+      def continuous
+        extend Continuous
+      end
+
+      def unique?
+        false
+      end
+
+      # Public: job used priority queues
+      def priority?
+        false
+      end
+
+      # Extend resque-retry.
+      #
+      # options - Hash
+      #
+      #   :limit - Integer (default: 2)
+      #   :delay - Integer (default: 60)
+      #   :expire_retry_key_after - Integer (default: 3200), истечение ключа в секундах. Если
+      #
+      #     t - среднее время выполнения одного джоба
+      #     n - текущее кол-во джобов в очереди
+      #     k - кол-во воркеров
+      #
+      #     то expire_retry_key_after >= t * n / k
+      #
+      #     Иначе ключ истечет, прежде чем джоб отработает.
+      #
+      #
+      # Returns nothing
+      def retrys(options = {})
+        if unique?
+          raise '`retrys` should be declared higher in code than `unique`'
+        end
+
+        extend Resque::Plugins::Retry
+
+        @retry_limit = options.fetch(:limit, 2)
+        @retry_delay = options.fetch(:delay, 60)
+        @expire_retry_key_after = options.fetch(:expire_retry_key_after, 1.hour.seconds)
+      end
+
+      # Mark Job as ordered
+      def ordered(options = {})
+        extend Ordered
+
+        self.max_iterations = options.fetch(:max_iterations, 20)
+        self.uniqueness = Ordered::Uniqueness.new(&options[:unique]) if options.key?(:unique)
+      end
+
+      def prioritized
+        extend Priority
+      end
+    end
+  end # module Integration
+end # module Resque
+
+require 'resque/integration/engine'

data/lib/resque-integration.rb

@@ -0,0 +1 @@
+require 'resque/integration'

data/resque-integration.gemspec

@@ -0,0 +1,40 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'resque/integration/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = 'resque-integration'
+  gem.version       = Resque::Integration::VERSION
+  gem.authors       = ['Alexei Mikhailov', 'Michail Merkushin']
+  gem.email         = %w(amikhailov83@gmail.com merkushin.m.s@gmail.com)
+  gem.summary       = %q{Seamless integration of resque with resque-progress and resque-lock}
+  gem.homepage      = 'https://github.com/abak-press/resque-integration'
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = %w(lib)
+
+  gem.add_runtime_dependency 'resque', '>= 1.25.2'
+  gem.add_runtime_dependency 'rails', '>= 3.0.0'
+  gem.add_runtime_dependency 'resque-meta', '>= 2.0.0'
+  gem.add_runtime_dependency 'resque-progress', '~> 1.0.1'
+  gem.add_runtime_dependency 'resque-multi-job-forks', '~> 0.4.2'
+  gem.add_runtime_dependency 'resque-failed-job-mailer', '~> 0.0.3'
+  gem.add_runtime_dependency 'resque-scheduler', '~> 4.0'
+  gem.add_runtime_dependency 'resque-retry', '~> 1.5'
+  gem.add_runtime_dependency 'god', '~> 0.13.4'
+
+  gem.add_runtime_dependency 'multi_json'
+  gem.add_runtime_dependency 'rake'
+
+  gem.add_development_dependency 'bundler'
+  gem.add_development_dependency 'rspec', '~> 2.14'
+  gem.add_development_dependency 'rspec-its'
+  gem.add_development_dependency 'simplecov'
+  gem.add_development_dependency 'appraisal', '>= 1.0.2'
+  gem.add_development_dependency 'combustion', '>= 0.5.5'
+  gem.add_development_dependency 'mock_redis'
+  gem.add_development_dependency 'timecop'
+  gem.add_development_dependency 'pry-byebug'
+end

data/spec/fixtures/resque_queues.yml

@@ -0,0 +1,45 @@
+defaults:
+  max_age: 10
+  max_size: 10
+  max_failures_count_per_5m: 5
+  max_failures_count_per_1h: 60
+  channel: default
+
+  warn_age: 8
+  warn_size: 7
+  warn_failures_count_per_5m: 3
+  warn_failures_count_per_1h: 30
+  warn_channel: default_warnings
+
+queues:
+  ?
+  >
+    first,
+    third
+  :
+    max_age: 20
+    warn_age: 10
+
+    max_size: 100
+    warn_size: 50
+
+    max_failures_count_per_5m: 15
+    warn_failures_count_per_5m: 7
+
+    max_failures_count_per_1h: 90
+    warn_failures_count_per_1h: 45
+
+    channel: first
+    warn_channel: first_warnings
+
+  third:
+    max_age: 30
+    warn_age: 15
+    max_failures_count_per_1h: 70
+    warn_failures_count_per_1h: 35
+
+  second_queue:
+    channel: [first, second]
+    warn_channel: [first_warnings, second_warnings]
+
+  without_channel: {}

data/spec/resque/controllers/jobs_controller_spec.rb

@@ -0,0 +1,65 @@
+require 'spec_helper'
+
+describe Resque::JobsController do
+  include Rack::Test::Methods
+
+  class MetaJob
+    extend Resque::Plugins::Meta
+
+    @queue = 'test'
+
+    def self.perform(_meta_id); end
+  end
+
+  describe '#show' do
+    let(:app) { described_class.action(:show) }
+
+    context 'when id is missing' do
+      before do
+        get '/'
+      end
+
+      it do
+        expect(last_response.status).to eq 404
+        expect(last_response.body).to eq '{"message":"not found"}'
+      end
+    end
+
+    context 'when id is invalid' do
+      before do
+        get '/', id: 'xx'
+      end
+
+      it do
+        expect(last_response.status).to eq 404
+        expect(last_response.body).to eq '{"message":"not found"}'
+      end
+    end
+
+    context 'when id is correct' do
+      let(:meta) { MetaJob.enqueue }
+      let(:meta_id) { meta.meta_id }
+      let(:body) do
+        {
+          enqueued_at: meta.enqueued_at,
+          started_at: nil,
+          finished_at: nil,
+          succeeded: nil,
+          failed: nil,
+          progress: {num: 0, total: 1, percent: 0.0, message: nil},
+          payload: nil
+        }.to_json
+      end
+
+      before do
+        get '/', id: meta_id
+      end
+
+      it do
+        expect(last_response.status).to eq 200
+        expect(last_response.body).to eq body
+        expect(last_response['Cache-Control']).to eq 'no-cache'
+      end
+    end
+  end
+end